diff --git a/CHANGELOG.md b/CHANGELOG.md index fad86c89da3de..59ce6b7690d59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,59 +1,74 @@ -- [v1.7.4](#v174) - - [Downloads for v1.7.4](#downloads-for-v174) +- [v1.8.0-alpha.3](#v180-alpha3) + - [Downloads for v1.8.0-alpha.3](#downloads-for-v180-alpha3) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - - [Changelog since v1.7.3](#changelog-since-v173) + - [Changelog since v1.8.0-alpha.2](#changelog-since-v180-alpha2) + - [Action Required](#action-required) - [Other notable changes](#other-notable-changes) -- [v1.6.8](#v168) - - [Downloads for v1.6.8](#downloads-for-v168) +- [v1.6.9](#v169) + - [Downloads for v1.6.9](#downloads-for-v169) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - - [Changelog since v1.6.7](#changelog-since-v167) + - [Changelog since v1.6.8](#changelog-since-v168) - [Other notable changes](#other-notable-changes-1) -- [v1.7.3](#v173) - - [Downloads for v1.7.3](#downloads-for-v173) +- [v1.7.4](#v174) + - [Downloads for v1.7.4](#downloads-for-v174) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) - - [Changelog since v1.7.2](#changelog-since-v172) + - [Changelog since v1.7.3](#changelog-since-v173) - [Other notable changes](#other-notable-changes-2) -- [v1.7.2](#v172) - - [Downloads for v1.7.2](#downloads-for-v172) +- [v1.6.8](#v168) + - [Downloads for v1.6.8](#downloads-for-v168) - [Client Binaries](#client-binaries-3) - [Server Binaries](#server-binaries-3) - [Node Binaries](#node-binaries-3) - - [Changelog since v1.7.1](#changelog-since-v171) + - [Changelog since v1.6.7](#changelog-since-v167) - [Other notable changes](#other-notable-changes-3) -- [v1.7.1](#v171) - - [Downloads for v1.7.1](#downloads-for-v171) +- [v1.7.3](#v173) + - [Downloads for v1.7.3](#downloads-for-v173) - [Client Binaries](#client-binaries-4) - [Server Binaries](#server-binaries-4) - [Node Binaries](#node-binaries-4) - - [Changelog since v1.7.0](#changelog-since-v170) + - [Changelog since v1.7.2](#changelog-since-v172) - [Other notable changes](#other-notable-changes-4) -- [v1.8.0-alpha.2](#v180-alpha2) - - [Downloads for v1.8.0-alpha.2](#downloads-for-v180-alpha2) +- [v1.7.2](#v172) + - [Downloads for v1.7.2](#downloads-for-v172) - [Client Binaries](#client-binaries-5) - [Server Binaries](#server-binaries-5) - [Node Binaries](#node-binaries-5) - - [Changelog since v1.7.0](#changelog-since-v170-1) - - [Action Required](#action-required) + - [Changelog since v1.7.1](#changelog-since-v171) - [Other notable changes](#other-notable-changes-5) -- [v1.6.7](#v167) - - [Downloads for v1.6.7](#downloads-for-v167) +- [v1.7.1](#v171) + - [Downloads for v1.7.1](#downloads-for-v171) - [Client Binaries](#client-binaries-6) - [Server Binaries](#server-binaries-6) - [Node Binaries](#node-binaries-6) - - [Changelog since v1.6.6](#changelog-since-v166) + - [Changelog since v1.7.0](#changelog-since-v170) - [Other notable changes](#other-notable-changes-6) -- [v1.7.0](#v170) - - [Downloads for v1.7.0](#downloads-for-v170) +- [v1.8.0-alpha.2](#v180-alpha2) + - [Downloads for v1.8.0-alpha.2](#downloads-for-v180-alpha2) - [Client Binaries](#client-binaries-7) - [Server Binaries](#server-binaries-7) - [Node Binaries](#node-binaries-7) + - [Changelog since v1.7.0](#changelog-since-v170-1) + - [Action Required](#action-required-1) + - [Other notable changes](#other-notable-changes-7) +- [v1.6.7](#v167) + - [Downloads for v1.6.7](#downloads-for-v167) + - [Client Binaries](#client-binaries-8) + - [Server Binaries](#server-binaries-8) + - [Node Binaries](#node-binaries-8) + - [Changelog since v1.6.6](#changelog-since-v166) + - [Other notable changes](#other-notable-changes-8) +- [v1.7.0](#v170) + - [Downloads for v1.7.0](#downloads-for-v170) + - [Client Binaries](#client-binaries-9) + - [Server Binaries](#server-binaries-9) + - [Node Binaries](#node-binaries-9) - [**Major Themes**](#major-themes) - [**Action Required Before Upgrading**](#action-required-before-upgrading) - [Network](#network) @@ -109,7 +124,7 @@ - [Local Storage](#local-storage) - [Volume Plugins](#volume-plugins) - [Metrics](#metrics) - - [**Other notable changes**](#other-notable-changes-7) + - [**Other notable changes**](#other-notable-changes-9) - [Admission plugin](#admission-plugin) - [API Machinery](#api-machinery-1) - [Application autoscaling](#application-autoscaling-1) @@ -137,130 +152,130 @@ - [Previous Releases Included in v1.7.0](#previous-releases-included-in-v170) - [v1.7.0-rc.1](#v170-rc1) - [Downloads for v1.7.0-rc.1](#downloads-for-v170-rc1) - - [Client Binaries](#client-binaries-8) - - [Server Binaries](#server-binaries-8) - - [Node Binaries](#node-binaries-8) - - [Changelog since v1.7.0-beta.2](#changelog-since-v170-beta2) - - [Action Required](#action-required-1) - - [Other notable changes](#other-notable-changes-8) -- [v1.8.0-alpha.1](#v180-alpha1) - - [Downloads for v1.8.0-alpha.1](#downloads-for-v180-alpha1) - - [Client Binaries](#client-binaries-9) - - [Server Binaries](#server-binaries-9) - - [Node Binaries](#node-binaries-9) - - [Changelog since v1.7.0-alpha.4](#changelog-since-v170-alpha4) - - [Action Required](#action-required-2) - - [Other notable changes](#other-notable-changes-9) -- [v1.6.6](#v166) - - [Downloads for v1.6.6](#downloads-for-v166) - [Client Binaries](#client-binaries-10) - [Server Binaries](#server-binaries-10) - [Node Binaries](#node-binaries-10) - - [Changelog since v1.6.5](#changelog-since-v165) - - [Action Required](#action-required-3) + - [Changelog since v1.7.0-beta.2](#changelog-since-v170-beta2) + - [Action Required](#action-required-2) - [Other notable changes](#other-notable-changes-10) -- [v1.7.0-beta.2](#v170-beta2) - - [Downloads for v1.7.0-beta.2](#downloads-for-v170-beta2) +- [v1.8.0-alpha.1](#v180-alpha1) + - [Downloads for v1.8.0-alpha.1](#downloads-for-v180-alpha1) - [Client Binaries](#client-binaries-11) - [Server Binaries](#server-binaries-11) - [Node Binaries](#node-binaries-11) - - [Changelog since v1.7.0-beta.1](#changelog-since-v170-beta1) - - [Action Required](#action-required-4) + - [Changelog since v1.7.0-alpha.4](#changelog-since-v170-alpha4) + - [Action Required](#action-required-3) - [Other notable changes](#other-notable-changes-11) -- [v1.6.5](#v165) - - [Known Issues for v1.6.5](#known-issues-for-v165) - - [Downloads for v1.6.5](#downloads-for-v165) +- [v1.6.6](#v166) + - [Downloads for v1.6.6](#downloads-for-v166) - [Client Binaries](#client-binaries-12) - [Server Binaries](#server-binaries-12) - [Node Binaries](#node-binaries-12) - - [Changelog since v1.6.4](#changelog-since-v164) + - [Changelog since v1.6.5](#changelog-since-v165) + - [Action Required](#action-required-4) - [Other notable changes](#other-notable-changes-12) -- [v1.7.0-beta.1](#v170-beta1) - - [Downloads for v1.7.0-beta.1](#downloads-for-v170-beta1) +- [v1.7.0-beta.2](#v170-beta2) + - [Downloads for v1.7.0-beta.2](#downloads-for-v170-beta2) - [Client Binaries](#client-binaries-13) - [Server Binaries](#server-binaries-13) - [Node Binaries](#node-binaries-13) - - [Changelog since v1.7.0-alpha.4](#changelog-since-v170-alpha4-1) + - [Changelog since v1.7.0-beta.1](#changelog-since-v170-beta1) - [Action Required](#action-required-5) - [Other notable changes](#other-notable-changes-13) -- [v1.6.4](#v164) - - [Known Issues for v1.6.4](#known-issues-for-v164) - - [Downloads for v1.6.4](#downloads-for-v164) +- [v1.6.5](#v165) + - [Known Issues for v1.6.5](#known-issues-for-v165) + - [Downloads for v1.6.5](#downloads-for-v165) - [Client Binaries](#client-binaries-14) - [Server Binaries](#server-binaries-14) - [Node Binaries](#node-binaries-14) - - [Changelog since v1.6.3](#changelog-since-v163) + - [Changelog since v1.6.4](#changelog-since-v164) - [Other notable changes](#other-notable-changes-14) -- [v1.7.0-alpha.4](#v170-alpha4) - - [Downloads for v1.7.0-alpha.4](#downloads-for-v170-alpha4) +- [v1.7.0-beta.1](#v170-beta1) + - [Downloads for v1.7.0-beta.1](#downloads-for-v170-beta1) - [Client Binaries](#client-binaries-15) - [Server Binaries](#server-binaries-15) - [Node Binaries](#node-binaries-15) - - [Changelog since v1.7.0-alpha.3](#changelog-since-v170-alpha3) + - [Changelog since v1.7.0-alpha.4](#changelog-since-v170-alpha4-1) - [Action Required](#action-required-6) - [Other notable changes](#other-notable-changes-15) -- [v1.6.3](#v163) - - [Known Issues for v1.6.3](#known-issues-for-v163) - - [Downloads for v1.6.3](#downloads-for-v163) +- [v1.6.4](#v164) + - [Known Issues for v1.6.4](#known-issues-for-v164) + - [Downloads for v1.6.4](#downloads-for-v164) - [Client Binaries](#client-binaries-16) - [Server Binaries](#server-binaries-16) - [Node Binaries](#node-binaries-16) - - [Changelog since v1.6.2](#changelog-since-v162) + - [Changelog since v1.6.3](#changelog-since-v163) - [Other notable changes](#other-notable-changes-16) -- [v1.7.0-alpha.3](#v170-alpha3) - - [Downloads for v1.7.0-alpha.3](#downloads-for-v170-alpha3) +- [v1.7.0-alpha.4](#v170-alpha4) + - [Downloads for v1.7.0-alpha.4](#downloads-for-v170-alpha4) - [Client Binaries](#client-binaries-17) - [Server Binaries](#server-binaries-17) - [Node Binaries](#node-binaries-17) - - [Changelog since v1.7.0-alpha.2](#changelog-since-v170-alpha2) + - [Changelog since v1.7.0-alpha.3](#changelog-since-v170-alpha3) - [Action Required](#action-required-7) - [Other notable changes](#other-notable-changes-17) -- [v1.5.7](#v157) - - [Downloads for v1.5.7](#downloads-for-v157) +- [v1.6.3](#v163) + - [Known Issues for v1.6.3](#known-issues-for-v163) + - [Downloads for v1.6.3](#downloads-for-v163) - [Client Binaries](#client-binaries-18) - [Server Binaries](#server-binaries-18) - [Node Binaries](#node-binaries-18) - - [Changelog since v1.5.6](#changelog-since-v156) + - [Changelog since v1.6.2](#changelog-since-v162) - [Other notable changes](#other-notable-changes-18) -- [v1.4.12](#v1412) - - [Downloads for v1.4.12](#downloads-for-v1412) +- [v1.7.0-alpha.3](#v170-alpha3) + - [Downloads for v1.7.0-alpha.3](#downloads-for-v170-alpha3) - [Client Binaries](#client-binaries-19) - [Server Binaries](#server-binaries-19) - [Node Binaries](#node-binaries-19) - - [Changelog since v1.4.9](#changelog-since-v149) + - [Changelog since v1.7.0-alpha.2](#changelog-since-v170-alpha2) + - [Action Required](#action-required-8) - [Other notable changes](#other-notable-changes-19) -- [v1.7.0-alpha.2](#v170-alpha2) - - [Downloads for v1.7.0-alpha.2](#downloads-for-v170-alpha2) +- [v1.5.7](#v157) + - [Downloads for v1.5.7](#downloads-for-v157) - [Client Binaries](#client-binaries-20) - [Server Binaries](#server-binaries-20) - - [Changelog since v1.7.0-alpha.1](#changelog-since-v170-alpha1) - - [Action Required](#action-required-8) + - [Node Binaries](#node-binaries-20) + - [Changelog since v1.5.6](#changelog-since-v156) - [Other notable changes](#other-notable-changes-20) -- [v1.6.2](#v162) - - [Downloads for v1.6.2](#downloads-for-v162) +- [v1.4.12](#v1412) + - [Downloads for v1.4.12](#downloads-for-v1412) - [Client Binaries](#client-binaries-21) - [Server Binaries](#server-binaries-21) - - [Changelog since v1.6.1](#changelog-since-v161) + - [Node Binaries](#node-binaries-21) + - [Changelog since v1.4.9](#changelog-since-v149) - [Other notable changes](#other-notable-changes-21) -- [v1.7.0-alpha.1](#v170-alpha1) - - [Downloads for v1.7.0-alpha.1](#downloads-for-v170-alpha1) +- [v1.7.0-alpha.2](#v170-alpha2) + - [Downloads for v1.7.0-alpha.2](#downloads-for-v170-alpha2) - [Client Binaries](#client-binaries-22) - [Server Binaries](#server-binaries-22) - - [Changelog since v1.6.0](#changelog-since-v160) + - [Changelog since v1.7.0-alpha.1](#changelog-since-v170-alpha1) + - [Action Required](#action-required-9) - [Other notable changes](#other-notable-changes-22) -- [v1.6.1](#v161) - - [Downloads for v1.6.1](#downloads-for-v161) +- [v1.6.2](#v162) + - [Downloads for v1.6.2](#downloads-for-v162) - [Client Binaries](#client-binaries-23) - [Server Binaries](#server-binaries-23) - - [Changelog since v1.6.0](#changelog-since-v160-1) + - [Changelog since v1.6.1](#changelog-since-v161) - [Other notable changes](#other-notable-changes-23) -- [v1.6.0](#v160) - - [Downloads for v1.6.0](#downloads-for-v160) +- [v1.7.0-alpha.1](#v170-alpha1) + - [Downloads for v1.7.0-alpha.1](#downloads-for-v170-alpha1) - [Client Binaries](#client-binaries-24) - [Server Binaries](#server-binaries-24) + - [Changelog since v1.6.0](#changelog-since-v160) + - [Other notable changes](#other-notable-changes-24) +- [v1.6.1](#v161) + - [Downloads for v1.6.1](#downloads-for-v161) + - [Client Binaries](#client-binaries-25) + - [Server Binaries](#server-binaries-25) + - [Changelog since v1.6.0](#changelog-since-v160-1) + - [Other notable changes](#other-notable-changes-25) +- [v1.6.0](#v160) + - [Downloads for v1.6.0](#downloads-for-v160) + - [Client Binaries](#client-binaries-26) + - [Server Binaries](#server-binaries-26) - [WARNING: etcd backup strongly recommended](#warning:-etcd-backup-strongly-recommended) - [Major updates and release themes](#major-updates-and-release-themes) - - [Action Required](#action-required-9) + - [Action Required](#action-required-10) - [Certificates API](#certificates-api) - [Cluster Autoscaler](#cluster-autoscaler) - [Deployment](#deployment) @@ -324,7 +339,7 @@ - [vSphere](#vsphere) - [Federation](#federation-2) - [kubefed](#kubefed-1) - - [Other Notable Changes](#other-notable-changes-24) + - [Other Notable Changes](#other-notable-changes-26) - [Garbage Collector](#garbage-collector) - [kubeadm](#kubeadm-4) - [kubectl](#kubectl-1) @@ -334,7 +349,7 @@ - [Updates to apply](#updates-to-apply) - [Updates to edit](#updates-to-edit) - [Bug fixes](#bug-fixes) - - [Other Notable Changes](#other-notable-changes-25) + - [Other Notable Changes](#other-notable-changes-27) - [Node Components](#node-components-4) - [Bug fixes](#bug-fixes-1) - [kube-controller-manager](#kube-controller-manager) @@ -347,7 +362,7 @@ - [Photon](#photon) - [rbd](#rbd) - [vSphere](#vsphere-1) - - [Other Notable Changes](#other-notable-changes-26) + - [Other Notable Changes](#other-notable-changes-28) - [Changes to Cluster Provisioning Scripts](#changes-to-cluster-provisioning-scripts) - [AWS](#aws-1) - [Juju](#juju) @@ -355,7 +370,7 @@ - [GCE](#gce-1) - [OpenStack](#openstack) - [Container Images](#container-images) - - [Other Notable Changes](#other-notable-changes-27) + - [Other Notable Changes](#other-notable-changes-29) - [Changes to Addons](#changes-to-addons) - [Dashboard](#dashboard) - [DNS](#dns) @@ -371,108 +386,108 @@ - [Previous Releases Included in v1.6.0](#previous-releases-included-in-v160) - [v1.5.6](#v156) - [Downloads for v1.5.6](#downloads-for-v156) - - [Client Binaries](#client-binaries-25) - - [Server Binaries](#server-binaries-25) + - [Client Binaries](#client-binaries-27) + - [Server Binaries](#server-binaries-27) - [Changelog since v1.5.5](#changelog-since-v155) - - [Other notable changes](#other-notable-changes-28) + - [Other notable changes](#other-notable-changes-30) - [v1.6.0-rc.1](#v160-rc1) - [Downloads for v1.6.0-rc.1](#downloads-for-v160-rc1) - - [Client Binaries](#client-binaries-26) - - [Server Binaries](#server-binaries-26) + - [Client Binaries](#client-binaries-28) + - [Server Binaries](#server-binaries-28) - [Changelog since v1.6.0-beta.4](#changelog-since-v160-beta4) - - [Other notable changes](#other-notable-changes-29) + - [Other notable changes](#other-notable-changes-31) - [v1.5.5](#v155) - [Downloads for v1.5.5](#downloads-for-v155) - - [Client Binaries](#client-binaries-27) - - [Server Binaries](#server-binaries-27) + - [Client Binaries](#client-binaries-29) + - [Server Binaries](#server-binaries-29) - [Changelog since v1.5.4](#changelog-since-v154) - [v1.6.0-beta.4](#v160-beta4) - [Downloads for v1.6.0-beta.4](#downloads-for-v160-beta4) - - [Client Binaries](#client-binaries-28) - - [Server Binaries](#server-binaries-28) + - [Client Binaries](#client-binaries-30) + - [Server Binaries](#server-binaries-30) - [Changelog since v1.6.0-beta.3](#changelog-since-v160-beta3) - - [Other notable changes](#other-notable-changes-30) + - [Other notable changes](#other-notable-changes-32) - [v1.6.0-beta.3](#v160-beta3) - [Downloads for v1.6.0-beta.3](#downloads-for-v160-beta3) - - [Client Binaries](#client-binaries-29) - - [Server Binaries](#server-binaries-29) + - [Client Binaries](#client-binaries-31) + - [Server Binaries](#server-binaries-31) - [Changelog since v1.6.0-beta.2](#changelog-since-v160-beta2) - - [Other notable changes](#other-notable-changes-31) + - [Other notable changes](#other-notable-changes-33) - [v1.6.0-beta.2](#v160-beta2) - [Downloads for v1.6.0-beta.2](#downloads-for-v160-beta2) - - [Client Binaries](#client-binaries-30) - - [Server Binaries](#server-binaries-30) + - [Client Binaries](#client-binaries-32) + - [Server Binaries](#server-binaries-32) - [Changelog since v1.6.0-beta.1](#changelog-since-v160-beta1) - - [Action Required](#action-required-10) - - [Other notable changes](#other-notable-changes-32) + - [Action Required](#action-required-11) + - [Other notable changes](#other-notable-changes-34) - [v1.5.4](#v154) - [Downloads for v1.5.4](#downloads-for-v154) - - [Client Binaries](#client-binaries-31) - - [Server Binaries](#server-binaries-31) + - [Client Binaries](#client-binaries-33) + - [Server Binaries](#server-binaries-33) - [Changelog since v1.5.3](#changelog-since-v153) - - [Other notable changes](#other-notable-changes-33) + - [Other notable changes](#other-notable-changes-35) - [v1.6.0-beta.1](#v160-beta1) - [Downloads for v1.6.0-beta.1](#downloads-for-v160-beta1) - - [Client Binaries](#client-binaries-32) - - [Server Binaries](#server-binaries-32) + - [Client Binaries](#client-binaries-34) + - [Server Binaries](#server-binaries-34) - [Changelog since v1.6.0-alpha.3](#changelog-since-v160-alpha3) - - [Action Required](#action-required-11) - - [Other notable changes](#other-notable-changes-34) + - [Action Required](#action-required-12) + - [Other notable changes](#other-notable-changes-36) - [v1.6.0-alpha.3](#v160-alpha3) - [Downloads for v1.6.0-alpha.3](#downloads-for-v160-alpha3) - - [Client Binaries](#client-binaries-33) - - [Server Binaries](#server-binaries-33) + - [Client Binaries](#client-binaries-35) + - [Server Binaries](#server-binaries-35) - [Changelog since v1.6.0-alpha.2](#changelog-since-v160-alpha2) - - [Other notable changes](#other-notable-changes-35) + - [Other notable changes](#other-notable-changes-37) - [v1.4.9](#v149) - [Downloads for v1.4.9](#downloads-for-v149) - - [Client Binaries](#client-binaries-34) - - [Server Binaries](#server-binaries-34) + - [Client Binaries](#client-binaries-36) + - [Server Binaries](#server-binaries-36) - [Changelog since v1.4.8](#changelog-since-v148) - - [Other notable changes](#other-notable-changes-36) + - [Other notable changes](#other-notable-changes-38) - [v1.5.3](#v153) - [Downloads for v1.5.3](#downloads-for-v153) - - [Client Binaries](#client-binaries-35) - - [Server Binaries](#server-binaries-35) - - [Node Binaries](#node-binaries-20) + - [Client Binaries](#client-binaries-37) + - [Server Binaries](#server-binaries-37) + - [Node Binaries](#node-binaries-22) - [Changelog since v1.5.2](#changelog-since-v152) - - [Other notable changes](#other-notable-changes-37) + - [Other notable changes](#other-notable-changes-39) - [v1.6.0-alpha.2](#v160-alpha2) - [Downloads for v1.6.0-alpha.2](#downloads-for-v160-alpha2) - - [Client Binaries](#client-binaries-36) - - [Server Binaries](#server-binaries-36) + - [Client Binaries](#client-binaries-38) + - [Server Binaries](#server-binaries-38) - [Changelog since v1.6.0-alpha.1](#changelog-since-v160-alpha1) - - [Other notable changes](#other-notable-changes-38) + - [Other notable changes](#other-notable-changes-40) - [v1.6.0-alpha.1](#v160-alpha1) - [Downloads for v1.6.0-alpha.1](#downloads-for-v160-alpha1) - - [Client Binaries](#client-binaries-37) - - [Server Binaries](#server-binaries-37) + - [Client Binaries](#client-binaries-39) + - [Server Binaries](#server-binaries-39) - [Changelog since v1.5.0](#changelog-since-v150) - - [Action Required](#action-required-12) - - [Other notable changes](#other-notable-changes-39) + - [Action Required](#action-required-13) + - [Other notable changes](#other-notable-changes-41) - [v1.5.2](#v152) - [Downloads for v1.5.2](#downloads-for-v152) - - [Client Binaries](#client-binaries-38) - - [Server Binaries](#server-binaries-38) + - [Client Binaries](#client-binaries-40) + - [Server Binaries](#server-binaries-40) - [Changelog since v1.5.1](#changelog-since-v151) - - [Other notable changes](#other-notable-changes-40) + - [Other notable changes](#other-notable-changes-42) - [v1.4.8](#v148) - [Downloads for v1.4.8](#downloads-for-v148) - - [Client Binaries](#client-binaries-39) - - [Server Binaries](#server-binaries-39) + - [Client Binaries](#client-binaries-41) + - [Server Binaries](#server-binaries-41) - [Changelog since v1.4.7](#changelog-since-v147) - - [Other notable changes](#other-notable-changes-41) + - [Other notable changes](#other-notable-changes-43) - [v1.5.1](#v151) - [Downloads for v1.5.1](#downloads-for-v151) - - [Client Binaries](#client-binaries-40) - - [Server Binaries](#server-binaries-40) + - [Client Binaries](#client-binaries-42) + - [Server Binaries](#server-binaries-42) - [Changelog since v1.5.0](#changelog-since-v150-1) - - [Other notable changes](#other-notable-changes-42) + - [Other notable changes](#other-notable-changes-44) - [Known Issues for v1.5.1](#known-issues-for-v151) - [v1.5.0](#v150) - [Downloads for v1.5.0](#downloads-for-v150) - - [Client Binaries](#client-binaries-41) - - [Server Binaries](#server-binaries-41) + - [Client Binaries](#client-binaries-43) + - [Server Binaries](#server-binaries-43) - [Major Themes](#major-themes-1) - [Features](#features) - [Known Issues](#known-issues-1) @@ -481,103 +496,103 @@ - [Action Required Before Upgrading](#action-required-before-upgrading-1) - [External Dependency Version Information](#external-dependency-version-information-2) - [Changelog since v1.5.0-beta.3](#changelog-since-v150-beta3) - - [Other notable changes](#other-notable-changes-43) + - [Other notable changes](#other-notable-changes-45) - [Previous Releases Included in v1.5.0](#previous-releases-included-in-v150) - [v1.4.7](#v147) - [Downloads for v1.4.7](#downloads-for-v147) - - [Client Binaries](#client-binaries-42) - - [Server Binaries](#server-binaries-42) + - [Client Binaries](#client-binaries-44) + - [Server Binaries](#server-binaries-44) - [Changelog since v1.4.6](#changelog-since-v146) - - [Other notable changes](#other-notable-changes-44) + - [Other notable changes](#other-notable-changes-46) - [v1.5.0-beta.3](#v150-beta3) - [Downloads for v1.5.0-beta.3](#downloads-for-v150-beta3) - - [Client Binaries](#client-binaries-43) - - [Server Binaries](#server-binaries-43) + - [Client Binaries](#client-binaries-45) + - [Server Binaries](#server-binaries-45) - [Changelog since v1.5.0-beta.2](#changelog-since-v150-beta2) - - [Other notable changes](#other-notable-changes-45) + - [Other notable changes](#other-notable-changes-47) - [v1.5.0-beta.2](#v150-beta2) - [Downloads for v1.5.0-beta.2](#downloads-for-v150-beta2) - - [Client Binaries](#client-binaries-44) - - [Server Binaries](#server-binaries-44) + - [Client Binaries](#client-binaries-46) + - [Server Binaries](#server-binaries-46) - [Changelog since v1.5.0-beta.1](#changelog-since-v150-beta1) - - [Other notable changes](#other-notable-changes-46) + - [Other notable changes](#other-notable-changes-48) - [v1.5.0-beta.1](#v150-beta1) - [Downloads for v1.5.0-beta.1](#downloads-for-v150-beta1) - - [Client Binaries](#client-binaries-45) - - [Server Binaries](#server-binaries-45) + - [Client Binaries](#client-binaries-47) + - [Server Binaries](#server-binaries-47) - [Changelog since v1.5.0-alpha.2](#changelog-since-v150-alpha2) - - [Action Required](#action-required-13) - - [Other notable changes](#other-notable-changes-47) + - [Action Required](#action-required-14) + - [Other notable changes](#other-notable-changes-49) - [v1.4.6](#v146) - [Downloads for v1.4.6](#downloads-for-v146) - - [Client Binaries](#client-binaries-46) - - [Server Binaries](#server-binaries-46) + - [Client Binaries](#client-binaries-48) + - [Server Binaries](#server-binaries-48) - [Changelog since v1.4.5](#changelog-since-v145) - - [Other notable changes](#other-notable-changes-48) + - [Other notable changes](#other-notable-changes-50) - [v1.3.10](#v1310) - [Downloads for v1.3.10](#downloads-for-v1310) - - [Client Binaries](#client-binaries-47) - - [Server Binaries](#server-binaries-47) + - [Client Binaries](#client-binaries-49) + - [Server Binaries](#server-binaries-49) - [Changelog since v1.3.9](#changelog-since-v139) - - [Other notable changes](#other-notable-changes-49) + - [Other notable changes](#other-notable-changes-51) - [v1.4.5](#v145) - [Downloads for v1.4.5](#downloads-for-v145) - - [Client Binaries](#client-binaries-48) - - [Server Binaries](#server-binaries-48) + - [Client Binaries](#client-binaries-50) + - [Server Binaries](#server-binaries-50) - [Changelog since v1.4.4](#changelog-since-v144) - - [Other notable changes](#other-notable-changes-50) + - [Other notable changes](#other-notable-changes-52) - [v1.5.0-alpha.2](#v150-alpha2) - [Downloads for v1.5.0-alpha.2](#downloads-for-v150-alpha2) - - [Client Binaries](#client-binaries-49) - - [Server Binaries](#server-binaries-49) + - [Client Binaries](#client-binaries-51) + - [Server Binaries](#server-binaries-51) - [Changelog since v1.5.0-alpha.1](#changelog-since-v150-alpha1) - - [Action Required](#action-required-14) - - [Other notable changes](#other-notable-changes-51) + - [Action Required](#action-required-15) + - [Other notable changes](#other-notable-changes-53) - [v1.2.7](#v127) - [Downloads for v1.2.7](#downloads-for-v127) - - [Client Binaries](#client-binaries-50) - - [Server Binaries](#server-binaries-50) + - [Client Binaries](#client-binaries-52) + - [Server Binaries](#server-binaries-52) - [Changelog since v1.2.6](#changelog-since-v126) - - [Other notable changes](#other-notable-changes-52) + - [Other notable changes](#other-notable-changes-54) - [v1.4.4](#v144) - [Downloads for v1.4.4](#downloads-for-v144) - - [Client Binaries](#client-binaries-51) - - [Server Binaries](#server-binaries-51) + - [Client Binaries](#client-binaries-53) + - [Server Binaries](#server-binaries-53) - [Changelog since v1.4.3](#changelog-since-v143) - - [Other notable changes](#other-notable-changes-53) + - [Other notable changes](#other-notable-changes-55) - [v1.3.9](#v139) - [Downloads](#downloads) - [Changelog since v1.3.8](#changelog-since-v138) - - [Other notable changes](#other-notable-changes-54) + - [Other notable changes](#other-notable-changes-56) - [v1.4.3](#v143) - [Downloads](#downloads-1) - [Changelog since v1.4.2-beta.1](#changelog-since-v142-beta1) - - [Other notable changes](#other-notable-changes-55) + - [Other notable changes](#other-notable-changes-57) - [v1.4.2](#v142) - [Downloads](#downloads-2) - [Changelog since v1.4.2-beta.1](#changelog-since-v142-beta1-1) - - [Other notable changes](#other-notable-changes-56) + - [Other notable changes](#other-notable-changes-58) - [v1.5.0-alpha.1](#v150-alpha1) - [Downloads](#downloads-3) - [Changelog since v1.4.0-alpha.3](#changelog-since-v140-alpha3) - [Experimental Features](#experimental-features) - - [Action Required](#action-required-15) - - [Other notable changes](#other-notable-changes-57) + - [Action Required](#action-required-16) + - [Other notable changes](#other-notable-changes-59) - [v1.4.2-beta.1](#v142-beta1) - [Downloads](#downloads-4) - [Changelog since v1.4.1](#changelog-since-v141) - - [Other notable changes](#other-notable-changes-58) + - [Other notable changes](#other-notable-changes-60) - [v1.4.1](#v141) - [Downloads](#downloads-5) - [Changelog since v1.4.1-beta.2](#changelog-since-v141-beta2) - [v1.4.1-beta.2](#v141-beta2) - [Downloads](#downloads-6) - [Changelog since v1.4.0](#changelog-since-v140) - - [Other notable changes](#other-notable-changes-59) + - [Other notable changes](#other-notable-changes-61) - [v1.3.8](#v138) - [Downloads](#downloads-7) - [Changelog since v1.3.7](#changelog-since-v137) - - [Other notable changes](#other-notable-changes-60) + - [Other notable changes](#other-notable-changes-62) - [v1.4.0](#v140) - [Downloads](#downloads-8) - [Major Themes](#major-themes-2) @@ -597,26 +612,26 @@ - [v1.4.0-beta.10](#v140-beta10) - [Downloads](#downloads-10) - [Changelog since v1.4.0-beta.8](#changelog-since-v140-beta8) - - [Other notable changes](#other-notable-changes-61) + - [Other notable changes](#other-notable-changes-63) - [v1.4.0-beta.8](#v140-beta8) - [Downloads](#downloads-11) - [Changelog since v1.4.0-beta.7](#changelog-since-v140-beta7) - [v1.4.0-beta.7](#v140-beta7) - [Downloads](#downloads-12) - [Changelog since v1.4.0-beta.6](#changelog-since-v140-beta6) - - [Other notable changes](#other-notable-changes-62) + - [Other notable changes](#other-notable-changes-64) - [v1.4.0-beta.6](#v140-beta6) - [Downloads](#downloads-13) - [Changelog since v1.4.0-beta.5](#changelog-since-v140-beta5) - - [Other notable changes](#other-notable-changes-63) + - [Other notable changes](#other-notable-changes-65) - [v1.4.0-beta.5](#v140-beta5) - [Downloads](#downloads-14) - [Changelog since v1.4.0-beta.3](#changelog-since-v140-beta3) - - [Other notable changes](#other-notable-changes-64) + - [Other notable changes](#other-notable-changes-66) - [v1.3.7](#v137) - [Downloads](#downloads-15) - [Changelog since v1.3.6](#changelog-since-v136) - - [Other notable changes](#other-notable-changes-65) + - [Other notable changes](#other-notable-changes-67) - [v1.4.0-beta.3](#v140-beta3) - [Downloads](#downloads-16) - [Changelog since v1.4.0-beta.2](#changelog-since-v140-beta2) @@ -627,57 +642,57 @@ - [v1.4.0-beta.2](#v140-beta2) - [Downloads](#downloads-17) - [Changelog since v1.4.0-beta.1](#changelog-since-v140-beta1) - - [Other notable changes](#other-notable-changes-66) + - [Other notable changes](#other-notable-changes-68) - [v1.4.0-beta.1](#v140-beta1) - [Downloads](#downloads-18) - [Changelog since v1.4.0-alpha.3](#changelog-since-v140-alpha3-1) - - [Action Required](#action-required-16) - - [Other notable changes](#other-notable-changes-67) + - [Action Required](#action-required-17) + - [Other notable changes](#other-notable-changes-69) - [v1.3.6](#v136) - [Downloads](#downloads-19) - [Changelog since v1.3.5](#changelog-since-v135) - - [Other notable changes](#other-notable-changes-68) + - [Other notable changes](#other-notable-changes-70) - [v1.4.0-alpha.3](#v140-alpha3) - [Downloads](#downloads-20) - [Changelog since v1.4.0-alpha.2](#changelog-since-v140-alpha2) - - [Action Required](#action-required-17) - - [Other notable changes](#other-notable-changes-69) + - [Action Required](#action-required-18) + - [Other notable changes](#other-notable-changes-71) - [v1.3.5](#v135) - [Downloads](#downloads-21) - [Changelog since v1.3.4](#changelog-since-v134) - - [Other notable changes](#other-notable-changes-70) + - [Other notable changes](#other-notable-changes-72) - [v1.3.4](#v134) - [Downloads](#downloads-22) - [Changelog since v1.3.3](#changelog-since-v133) - - [Other notable changes](#other-notable-changes-71) + - [Other notable changes](#other-notable-changes-73) - [v1.4.0-alpha.2](#v140-alpha2) - [Downloads](#downloads-23) - [Changelog since v1.4.0-alpha.1](#changelog-since-v140-alpha1) - - [Action Required](#action-required-18) - - [Other notable changes](#other-notable-changes-72) + - [Action Required](#action-required-19) + - [Other notable changes](#other-notable-changes-74) - [v1.3.3](#v133) - [Downloads](#downloads-24) - [Changelog since v1.3.2](#changelog-since-v132) - - [Other notable changes](#other-notable-changes-73) + - [Other notable changes](#other-notable-changes-75) - [Known Issues](#known-issues-3) - [v1.3.2](#v132) - [Downloads](#downloads-25) - [Changelog since v1.3.1](#changelog-since-v131) - - [Other notable changes](#other-notable-changes-74) + - [Other notable changes](#other-notable-changes-76) - [v1.3.1](#v131) - [Downloads](#downloads-26) - [Changelog since v1.3.0](#changelog-since-v130) - - [Other notable changes](#other-notable-changes-75) + - [Other notable changes](#other-notable-changes-77) - [v1.2.6](#v126) - [Downloads](#downloads-27) - [Changelog since v1.2.5](#changelog-since-v125) - - [Other notable changes](#other-notable-changes-76) + - [Other notable changes](#other-notable-changes-78) - [v1.4.0-alpha.1](#v140-alpha1) - [Downloads](#downloads-28) - [Changelog since v1.3.0](#changelog-since-v130-1) - [Experimental Features](#experimental-features-1) - - [Action Required](#action-required-19) - - [Other notable changes](#other-notable-changes-77) + - [Action Required](#action-required-20) + - [Other notable changes](#other-notable-changes-79) - [v1.3.0](#v130) - [Downloads](#downloads-29) - [Highlights](#highlights) @@ -692,70 +707,70 @@ - [v1.3.0-beta.3](#v130-beta3) - [Downloads](#downloads-30) - [Changelog since v1.3.0-beta.2](#changelog-since-v130-beta2) - - [Action Required](#action-required-20) - - [Other notable changes](#other-notable-changes-78) + - [Action Required](#action-required-21) + - [Other notable changes](#other-notable-changes-80) - [v1.2.5](#v125) - [Downloads](#downloads-31) - [Changes since v1.2.4](#changes-since-v124) - - [Other notable changes](#other-notable-changes-79) + - [Other notable changes](#other-notable-changes-81) - [v1.3.0-beta.2](#v130-beta2) - [Downloads](#downloads-32) - [Changes since v1.3.0-beta.1](#changes-since-v130-beta1) - [Experimental Features](#experimental-features-2) - - [Other notable changes](#other-notable-changes-80) + - [Other notable changes](#other-notable-changes-82) - [v1.3.0-beta.1](#v130-beta1) - [Downloads](#downloads-33) - [Changes since v1.3.0-alpha.5](#changes-since-v130-alpha5) - - [Action Required](#action-required-21) - - [Other notable changes](#other-notable-changes-81) + - [Action Required](#action-required-22) + - [Other notable changes](#other-notable-changes-83) - [v1.3.0-alpha.5](#v130-alpha5) - [Downloads](#downloads-34) - [Changes since v1.3.0-alpha.4](#changes-since-v130-alpha4) - - [Action Required](#action-required-22) - - [Other notable changes](#other-notable-changes-82) + - [Action Required](#action-required-23) + - [Other notable changes](#other-notable-changes-84) - [v1.3.0-alpha.4](#v130-alpha4) - [Downloads](#downloads-35) - [Changes since v1.3.0-alpha.3](#changes-since-v130-alpha3) - - [Action Required](#action-required-23) - - [Other notable changes](#other-notable-changes-83) + - [Action Required](#action-required-24) + - [Other notable changes](#other-notable-changes-85) - [v1.2.4](#v124) - [Downloads](#downloads-36) - [Changes since v1.2.3](#changes-since-v123) - - [Other notable changes](#other-notable-changes-84) + - [Other notable changes](#other-notable-changes-86) - [v1.3.0-alpha.3](#v130-alpha3) - [Downloads](#downloads-37) - [Changes since v1.3.0-alpha.2](#changes-since-v130-alpha2) - - [Action Required](#action-required-24) - - [Other notable changes](#other-notable-changes-85) + - [Action Required](#action-required-25) + - [Other notable changes](#other-notable-changes-87) - [v1.2.3](#v123) - [Downloads](#downloads-38) - [Changes since v1.2.2](#changes-since-v122) - - [Action Required](#action-required-25) - - [Other notable changes](#other-notable-changes-86) + - [Action Required](#action-required-26) + - [Other notable changes](#other-notable-changes-88) - [v1.3.0-alpha.2](#v130-alpha2) - [Downloads](#downloads-39) - [Changes since v1.3.0-alpha.1](#changes-since-v130-alpha1) - - [Other notable changes](#other-notable-changes-87) + - [Other notable changes](#other-notable-changes-89) - [v1.2.2](#v122) - [Downloads](#downloads-40) - [Changes since v1.2.1](#changes-since-v121) - - [Other notable changes](#other-notable-changes-88) + - [Other notable changes](#other-notable-changes-90) - [v1.2.1](#v121) - [Downloads](#downloads-41) - [Changes since v1.2.0](#changes-since-v120) - - [Other notable changes](#other-notable-changes-89) + - [Other notable changes](#other-notable-changes-91) - [v1.3.0-alpha.1](#v130-alpha1) - [Downloads](#downloads-42) - [Changes since v1.2.0](#changes-since-v120-1) - - [Action Required](#action-required-26) - - [Other notable changes](#other-notable-changes-90) + - [Action Required](#action-required-27) + - [Other notable changes](#other-notable-changes-92) - [v1.2.0](#v120) - [Downloads](#downloads-43) - [Changes since v1.1.1](#changes-since-v111) - [Major Themes](#major-themes-3) - [Other notable improvements](#other-notable-improvements) - [Experimental Features](#experimental-features-3) - - [Action required](#action-required-27) + - [Action required](#action-required-28) - [Known Issues](#known-issues-4) - [Docker Known Issues](#docker-known-issues) - [1.9.1](#191) @@ -768,6 +783,337 @@ +# v1.8.0-alpha.3 + +[Documentation](https://docs.k8s.io) & [Examples](https://releases.k8s.io/master/examples) + +## Downloads for v1.8.0-alpha.3 + + +filename | sha256 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes.tar.gz) | `c99042c4826352b724dc02c8d92c01c49e1ad1663d2c55e0bce931fe4d76c1e3` +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-src.tar.gz) | `3ee0cd3594bd5b326f042044d87e120fe335bd8e722635220dd5741485ab3493` + +### Client Binaries + +filename | sha256 hash +-------- | ----------- +[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-darwin-386.tar.gz) | `c716e167383d118373d7b10425bb8db6033675e4520591017c688575f28a596d` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-darwin-amd64.tar.gz) | `dfe87cad00600049c841c8fd96c49088d4f7cdd34e5a903ef8048f75718f2d21` +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-linux-386.tar.gz) | `97242dffee822cbf4e3e373acf05e9dc2f40176b18f4532a60264ecf92738356` +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-linux-amd64.tar.gz) | `42e25e810333b00434217bae0aece145f82d0c7043faea83ff62bed079bae651` +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-linux-arm64.tar.gz) | `7f9683c90dc894ee8cd7ad30ec58d0d49068d35478a71b315d2b7805ec28e14a` +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-linux-arm.tar.gz) | `76347a154128e97cdd81674045b28035d89d509b35dda051f2cbc58c9b67fed4` +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-linux-ppc64le.tar.gz) | `c991cbbf0afa6eccd005b6e5ea28b0b20ecbc79ab7d64e32c24e03fcf05b48ff` +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-linux-s390x.tar.gz) | `94c2c29e8fd20d2a5c4f96098bd5c7d879a78e872f59c3c58ca1c775a57ddefb` +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-windows-386.tar.gz) | `bc98fd5dc01c6e6117c2c78d65884190bf99fd1fec0904e2af05e6dbf503ccc8` +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-client-windows-amd64.tar.gz) | `e32b56dbc69045b5b2821a2e3eb3c3b4a18cf4c11afd44e0c7c9c0e67bb38d02` + +### Server Binaries + +filename | sha256 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-server-linux-amd64.tar.gz) | `5446addff583b0dc977b91375f3c399242f7996e1f66f52b9e14c015add3bf13` +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-server-linux-arm64.tar.gz) | `91e3cffed119b5105f6a6f74f583113384a26c746b459029c12babf45f680119` +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-server-linux-arm.tar.gz) | `d4cb93787651193ef4fdd1d10a4822101586b2994d6b0e04d064687df8729910` +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-server-linux-ppc64le.tar.gz) | `916e7f63a4e0c67d9f106fdda6eb24efcc94356b05cd9eb288e45fac9ff79fe8` +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-server-linux-s390x.tar.gz) | `15b999b08f5fe0d8252f8a1c7e936b9e06f2b01132010b3cece547ab00b45282` + +### Node Binaries + +filename | sha256 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-node-linux-amd64.tar.gz) | `9120f6a06053ed91566d378a26ae455f521ab46911f257d64f629d93d143b369` +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-node-linux-arm64.tar.gz) | `30af817f5de0ecb8a95ec898fba5b97e6b4f224927e1cf7efaf2d5479b23c116` +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-node-linux-arm.tar.gz) | `8b0913e461d8ac821c2104a1f0b4efe3151f0d8e8598e0945e60b4ba7ac2d1a0` +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-node-linux-ppc64le.tar.gz) | `a78a3a837c0fbf6e092b312472c89ef0f3872c268b0a5e1e344e725a88c0717d` +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-node-linux-s390x.tar.gz) | `a0a38c5830fc1b7996c5befc24502991fc8a095f82cf81ddd0a301163143a2c5` +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.8.0-alpha.3/kubernetes-node-windows-amd64.tar.gz) | `8af4253fe2c582843de329d12d84dbdc5f9f823f68ee08a42809864efc7c368d` + +## Changelog since v1.8.0-alpha.2 + +### Action Required + +* Remove deprecated kubectl command aliases `apiversions, clusterinfo, resize, rollingupdate, run-container, update` ([#49935](https://github.com/kubernetes/kubernetes/pull/49935), [@xiangpengzhao](https://github.com/xiangpengzhao)) +* The following deprecated flags have been removed from `kube-controller-manager`: `replication-controller-lookup-cache-size`, `replicaset-lookup-cache-size`, and `daemonset-lookup-cache-size`. Make sure you no longer attempt to set them. ([#50678](https://github.com/kubernetes/kubernetes/pull/50678), [@xiangpengzhao](https://github.com/xiangpengzhao)) +* Beta annotations `service.beta.kubernetes.io/external-traffic` and `service.beta.kubernetes.io/healthcheck-nodeport` have been removed. Please use fields `service.spec.externalTrafficPolicy` and `service.spec.healthCheckNodePort` instead. ([#50224](https://github.com/kubernetes/kubernetes/pull/50224), [@xiangpengzhao](https://github.com/xiangpengzhao)) +* A cluster using the AWS cloud provider will need to label existing nodes and resources with a ClusterID or the kube-controller-manager will not start. To run without a ClusterID pass --allow-untagged-cloud=true to the kube-controller-manager on startup. ([#49215](https://github.com/kubernetes/kubernetes/pull/49215), [@rrati](https://github.com/rrati)) +* RBAC: the `system:node` role is no longer automatically granted to the `system:nodes` group in new clusters. It is recommended that nodes be authorized using the `Node` authorization mode instead. Installations that wish to continue giving all members of the `system:nodes` group the `system:node` role (which grants broad read access, including all secrets and configmaps) must create an installation-specific `ClusterRoleBinding`. ([#49638](https://github.com/kubernetes/kubernetes/pull/49638), [@liggitt](https://github.com/liggitt)) +* StatefulSet: The deprecated `pod.alpha.kubernetes.io/initialized` annotation for interrupting StatefulSet Pod management is now ignored. If you were setting it to `true` or leaving it unset, no action is required. However, if you were setting it to `false`, be aware that previously-dormant StatefulSets may become active after upgrading. ([#49251](https://github.com/kubernetes/kubernetes/pull/49251), [@enisoc](https://github.com/enisoc)) +* add some more deprecation warnings to cluster ([#49148](https://github.com/kubernetes/kubernetes/pull/49148), [@mikedanese](https://github.com/mikedanese)) +* The --insecure-allow-any-token flag has been removed from kube-apiserver. Users of the flag should use impersonation headers instead for debugging. ([#49045](https://github.com/kubernetes/kubernetes/pull/49045), [@ericchiang](https://github.com/ericchiang)) +* Restored cAdvisor prometheus metrics to the main port -- a regression that existed in v1.7.0-v1.7.2 ([#49079](https://github.com/kubernetes/kubernetes/pull/49079), [@smarterclayton](https://github.com/smarterclayton)) + * cAdvisor metrics can now be scraped from `/metrics/cadvisor` on the kubelet ports. + * Note that you have to update your scraping jobs to get kubelet-only metrics from `/metrics` and `container_*` metrics from `/metrics/cadvisor` +* Change the default kubeadm bootstrap token TTL from infinite to 24 hours. This is a breaking change. If you require the old behavior, use `kubeadm init --token-ttl 0` / `kubeadm token create --ttl 0`. ([#48783](https://github.com/kubernetes/kubernetes/pull/48783), [@mattmoyer](https://github.com/mattmoyer)) + +### Other notable changes + +* /release-note Remove duplicate command example from `kubectl port-forward --help` ([#50229](https://github.com/kubernetes/kubernetes/pull/50229), [@tcharding](https://github.com/tcharding)) +* Adds a new `kubeadm config` command that lets users tell `kubeadm upgrade` what kubeadm configuration to use and lets users view the current state. ([#50980](https://github.com/kubernetes/kubernetes/pull/50980), [@luxas](https://github.com/luxas)) +* Kubectl uses openapi for validation. If OpenAPI is not available on the server, it defaults back to the old Swagger. ([#50546](https://github.com/kubernetes/kubernetes/pull/50546), [@apelisse](https://github.com/apelisse)) +* kubectl show node role if defined ([#50438](https://github.com/kubernetes/kubernetes/pull/50438), [@dixudx](https://github.com/dixudx)) +* iSCSI volume plugin: iSCSI initiatorname support ([#48789](https://github.com/kubernetes/kubernetes/pull/48789), [@mtanino](https://github.com/mtanino)) +* On AttachDetachController node status update, do not retry when node doesn't exist but keep the node entry in cache. ([#50806](https://github.com/kubernetes/kubernetes/pull/50806), [@verult](https://github.com/verult)) +* Prevent unneeded endpoint updates ([#50934](https://github.com/kubernetes/kubernetes/pull/50934), [@joelsmith](https://github.com/joelsmith)) +* Affinity in annotations alpha feature is no longer supported in 1.8. Anyone upgrading from 1.7 with AffinityInAnnotation feature enabled must ensure pods (specifically with pod anti-affinity PreferredDuringSchedulingIgnoredDuringExecution) with empty TopologyKey fields must be removed before upgrading to 1.8. ([#49976](https://github.com/kubernetes/kubernetes/pull/49976), [@aveshagarwal](https://github.com/aveshagarwal)) +* NONE ([#50871](https://github.com/kubernetes/kubernetes/pull/50871), [@liyinan926](https://github.com/liyinan926)) +* - kubeadm now supports "ci/latest-1.8" or "ci-cross/latest-1.8" and similar labels. ([#49119](https://github.com/kubernetes/kubernetes/pull/49119), [@kad](https://github.com/kad)) +* kubeadm: Adds dry-run support for kubeadm using the `--dry-run` option ([#50631](https://github.com/kubernetes/kubernetes/pull/50631), [@luxas](https://github.com/luxas)) +* Change GCE installs (kube-up.sh) to use GCI/COS for node OS, by default. ([#46512](https://github.com/kubernetes/kubernetes/pull/46512), [@thockin](https://github.com/thockin)) +* Use CollisionCount for collision avoidance when creating ControllerRevisions in StatefulSet controller ([#50490](https://github.com/kubernetes/kubernetes/pull/50490), [@liyinan926](https://github.com/liyinan926)) +* AWS: Arbitrarily choose first (lexicographically) subnet in AZ ([#50255](https://github.com/kubernetes/kubernetes/pull/50255), [@mattlandis](https://github.com/mattlandis)) +* Change CollisionCount from int64 to int32 across controllers ([#50575](https://github.com/kubernetes/kubernetes/pull/50575), [@dixudx](https://github.com/dixudx)) +* fix GPU resource validation that incorrectly allows zero limits ([#50218](https://github.com/kubernetes/kubernetes/pull/50218), [@dixudx](https://github.com/dixudx)) +* The `kubernetes.io/created-by` annotation is now deprecated and will be removed in v1.9. Use [ControllerRef](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/controller-ref.md) instead to determine which controller, if any, owns an object. ([#50536](https://github.com/kubernetes/kubernetes/pull/50536), [@crimsonfaith91](https://github.com/crimsonfaith91)) +* Disable Docker's health check until we officially support it ([#50796](https://github.com/kubernetes/kubernetes/pull/50796), [@yguo0905](https://github.com/yguo0905)) +* Add ControllerRevision to apps/v1beta2 ([#50698](https://github.com/kubernetes/kubernetes/pull/50698), [@liyinan926](https://github.com/liyinan926)) +* StorageClass has a new field to configure reclaim policy of dynamically provisioned PVs. ([#47987](https://github.com/kubernetes/kubernetes/pull/47987), [@wongma7](https://github.com/wongma7)) +* Rerun init containers when the pod needs to be restarted ([#47599](https://github.com/kubernetes/kubernetes/pull/47599), [@yujuhong](https://github.com/yujuhong)) +* Resources outside the `*kubernetes.io` namespace are integers and cannot be over-committed. ([#48922](https://github.com/kubernetes/kubernetes/pull/48922), [@ConnorDoyle](https://github.com/ConnorDoyle)) +* apps/v1beta2 is enabled by default. DaemonSet, Deployment, ReplicaSet, and StatefulSet have been moved to this group version. ([#50643](https://github.com/kubernetes/kubernetes/pull/50643), [@kow3ns](https://github.com/kow3ns)) +* TLS cert storage for self-hosted clusters is now configurable. You can store them as secrets (alpha) or as usual host mounts. ([#50762](https://github.com/kubernetes/kubernetes/pull/50762), [@jamiehannaford](https://github.com/jamiehannaford)) +* Remove deprecated command 'kubectl stop' ([#46927](https://github.com/kubernetes/kubernetes/pull/46927), [@shiywang](https://github.com/shiywang)) +* Add new Prometheus metric that monitors the remaining lifetime of certificates used to authenticate requests to the API server. ([#50387](https://github.com/kubernetes/kubernetes/pull/50387), [@jcbsmpsn](https://github.com/jcbsmpsn)) +* Upgrade advanced audit to version v1beta1 ([#49115](https://github.com/kubernetes/kubernetes/pull/49115), [@CaoShuFeng](https://github.com/CaoShuFeng)) +* Cluster Autoscaler - fixes issues with taints and updates kube-proxy cpu request. ([#50514](https://github.com/kubernetes/kubernetes/pull/50514), [@mwielgus](https://github.com/mwielgus)) +* fluentd-elasticsearch addon: change the fluentd base image to fix crashes on systems with non-standard systemd installation ([#50679](https://github.com/kubernetes/kubernetes/pull/50679), [@aknuds1](https://github.com/aknuds1)) +* advanced audit: shutdown batching audit webhook gracefully ([#50577](https://github.com/kubernetes/kubernetes/pull/50577), [@crassirostris](https://github.com/crassirostris)) +* Add Priority admission controller for monitoring and resolving PriorityClasses. ([#49322](https://github.com/kubernetes/kubernetes/pull/49322), [@bsalamat](https://github.com/bsalamat)) +* apiservers: add synchronous shutdown mechanism on SIGTERM+INT ([#50439](https://github.com/kubernetes/kubernetes/pull/50439), [@sttts](https://github.com/sttts)) +* Fix kubernetes-worker charm hook failure when applying labels ([#50633](https://github.com/kubernetes/kubernetes/pull/50633), [@Cynerva](https://github.com/Cynerva)) +* kubeadm: Implementing the controlplane phase ([#50302](https://github.com/kubernetes/kubernetes/pull/50302), [@fabriziopandini](https://github.com/fabriziopandini)) +* Refactor addons into multiple packages ([#50214](https://github.com/kubernetes/kubernetes/pull/50214), [@andrewrynhard](https://github.com/andrewrynhard)) +* Kubelet now manages `/etc/hosts` file for both hostNetwork Pods and non-hostNetwork Pods. ([#49140](https://github.com/kubernetes/kubernetes/pull/49140), [@rickypai](https://github.com/rickypai)) +* After 1.8, admission controller will add 'MemoryPressure' toleration to Guaranteed and Burstable pods. ([#50180](https://github.com/kubernetes/kubernetes/pull/50180), [@k82cn](https://github.com/k82cn)) +* A new predicates, named 'CheckNodeCondition', was added to replace node condition filter. 'NetworkUnavailable', 'OutOfDisk' and 'NotReady' maybe reported as a reason when failed to schedule pods. ([#50362](https://github.com/kubernetes/kubernetes/pull/50362), [@k82cn](https://github.com/k82cn)) +* fix apps DeploymentSpec conversion issue ([#49719](https://github.com/kubernetes/kubernetes/pull/49719), [@dixudx](https://github.com/dixudx)) +* fluentd-gcp addon: Fix a bug in the event-exporter, when repeated events were not sent to Stackdriver. ([#50511](https://github.com/kubernetes/kubernetes/pull/50511), [@crassirostris](https://github.com/crassirostris)) +* not allowing "kubectl edit " when you got an empty list ([#50205](https://github.com/kubernetes/kubernetes/pull/50205), [@dixudx](https://github.com/dixudx)) +* NONE ([#49025](https://github.com/kubernetes/kubernetes/pull/49025), [@danwinship](https://github.com/danwinship)) +* fixes kubefed's ability to create RBAC roles in version-skewed clusters ([#50537](https://github.com/kubernetes/kubernetes/pull/50537), [@liggitt](https://github.com/liggitt)) +* API server authentication now caches successful bearer token authentication results for a few seconds. ([#50258](https://github.com/kubernetes/kubernetes/pull/50258), [@liggitt](https://github.com/liggitt)) +* Added field CollisionCount to StatefulSetStatus in both apps/v1beta1 and apps/v1beta2 ([#49983](https://github.com/kubernetes/kubernetes/pull/49983), [@liyinan926](https://github.com/liyinan926)) +* FC volume plugin: Support WWID for volume identifier ([#48741](https://github.com/kubernetes/kubernetes/pull/48741), [@mtanino](https://github.com/mtanino)) +* kubeadm: added enhanced TLS validation for token-based discovery in `kubeadm join` using a new `--discovery-token-ca-cert-hash` flag. ([#49520](https://github.com/kubernetes/kubernetes/pull/49520), [@mattmoyer](https://github.com/mattmoyer)) +* federation: Support for leader-election among federation controller-manager instances introduced. ([#46090](https://github.com/kubernetes/kubernetes/pull/46090), [@shashidharatd](https://github.com/shashidharatd)) +* New get-kube.sh option: KUBERNETES_SKIP_RELEASE_VALIDATION ([#50391](https://github.com/kubernetes/kubernetes/pull/50391), [@pipejakob](https://github.com/pipejakob)) +* Azure: Allow VNet to be in a separate Resource Group. ([#49725](https://github.com/kubernetes/kubernetes/pull/49725), [@sylr](https://github.com/sylr)) +* fix bug when azure cloud provider configuration file is not specified ([#49283](https://github.com/kubernetes/kubernetes/pull/49283), [@dixudx](https://github.com/dixudx)) +* The `rbac.authorization.k8s.io/v1beta1` API has been promoted to `rbac.authorization.k8s.io/v1` with no changes. ([#49642](https://github.com/kubernetes/kubernetes/pull/49642), [@liggitt](https://github.com/liggitt)) + * The `rbac.authorization.k8s.io/v1alpha1` version is deprecated and will be removed in a future release. +* Fix an issue where if a CSR is not approved initially by the SAR approver is not retried. ([#49788](https://github.com/kubernetes/kubernetes/pull/49788), [@mikedanese](https://github.com/mikedanese)) +* The v1.Service.PublishNotReadyAddresses field is added to notify DNS addons to publish the notReadyAddresses of Enpdoints. The "service.alpha.kubernetes.io/tolerate-unready-endpoints" annotation has been deprecated and will be removed when clients have sufficient time to consume the field. ([#49061](https://github.com/kubernetes/kubernetes/pull/49061), [@kow3ns](https://github.com/kow3ns)) +* vSphere cloud provider: vSphere cloud provider code refactoring ([#49164](https://github.com/kubernetes/kubernetes/pull/49164), [@BaluDontu](https://github.com/BaluDontu)) +* `cluster/gke` has been removed. GKE end-to-end testing should be done using `kubetest --deployment=gke` ([#50338](https://github.com/kubernetes/kubernetes/pull/50338), [@zmerlynn](https://github.com/zmerlynn)) +* kubeadm: Upload configuration used at 'kubeadm init' time to ConfigMap for easier upgrades ([#50320](https://github.com/kubernetes/kubernetes/pull/50320), [@luxas](https://github.com/luxas)) +* Adds (alpha feature) the ability to dynamically configure Kubelets by enabling the DynamicKubeletConfig feature gate, posting a ConfigMap to the API server, and setting the spec.configSource field on Node objects. See the proposal at https://github.com/kubernetes/community/blob/master/contributors/design-proposals/dynamic-kubelet-configuration.md for details. ([#46254](https://github.com/kubernetes/kubernetes/pull/46254), [@mtaufen](https://github.com/mtaufen)) +* Remove deprecated ScheduledJobs endpoints, use CronJobs instead. ([#49930](https://github.com/kubernetes/kubernetes/pull/49930), [@soltysh](https://github.com/soltysh)) +* [Federation] Make the hpa scale time window configurable ([#49583](https://github.com/kubernetes/kubernetes/pull/49583), [@irfanurrehman](https://github.com/irfanurrehman)) +* fuse daemons for GlusterFS and CephFS are now run in their own systemd scope when Kubernetes runs on a system with systemd. ([#49640](https://github.com/kubernetes/kubernetes/pull/49640), [@jsafrane](https://github.com/jsafrane)) +* `kubectl proxy` will now correctly handle the `exec`, `attach`, and `portforward` commands. You must pass `--disable-filter` to the command in order to allow these endpoints. ([#49534](https://github.com/kubernetes/kubernetes/pull/49534), [@smarterclayton](https://github.com/smarterclayton)) +* Copy annotations from a StatefulSet's metadata to the ControllerRevisions it owns ([#50263](https://github.com/kubernetes/kubernetes/pull/50263), [@liyinan926](https://github.com/liyinan926)) +* Make rolling update the default update strategy for v1beta2.DaemonSet and v1beta2.StatefulSet ([#50175](https://github.com/kubernetes/kubernetes/pull/50175), [@foxish](https://github.com/foxish)) +* Deprecate Deployment .spec.rollbackTo field ([#49340](https://github.com/kubernetes/kubernetes/pull/49340), [@janetkuo](https://github.com/janetkuo)) +* Collect metrics from Heapster in Stackdriver mode. ([#50290](https://github.com/kubernetes/kubernetes/pull/50290), [@piosz](https://github.com/piosz)) +* N/A ([#50179](https://github.com/kubernetes/kubernetes/pull/50179), [@k82cn](https://github.com/k82cn)) +* [Federation] HPA controller ([#45993](https://github.com/kubernetes/kubernetes/pull/45993), [@irfanurrehman](https://github.com/irfanurrehman)) +* Relax restrictions on environment variable names. ([#48986](https://github.com/kubernetes/kubernetes/pull/48986), [@timoreimann](https://github.com/timoreimann)) +* The node condition 'NodeInodePressure' was removed, as kubelet did not report it. ([#50124](https://github.com/kubernetes/kubernetes/pull/50124), [@k82cn](https://github.com/k82cn)) +* Fix premature return ([#49834](https://github.com/kubernetes/kubernetes/pull/49834), [@guoshimin](https://github.com/guoshimin)) +* StatefulSet uses scale subresource when scaling in accord with ReplicationController, ReplicaSet, and Deployment implementations. ([#49168](https://github.com/kubernetes/kubernetes/pull/49168), [@crimsonfaith91](https://github.com/crimsonfaith91)) +* Feature gates now determine whether a cluster is self-hosted. For more information, see the FeatureGates configuration flag. ([#50241](https://github.com/kubernetes/kubernetes/pull/50241), [@jamiehannaford](https://github.com/jamiehannaford)) +* Updates Cinder AttachDisk operation to be more reliable by delegating Detaches to volume manager. ([#50042](https://github.com/kubernetes/kubernetes/pull/50042), [@jingxu97](https://github.com/jingxu97)) +* add fieldSelector podIP ([#50091](https://github.com/kubernetes/kubernetes/pull/50091), [@dixudx](https://github.com/dixudx)) +* Return Audit-Id http response header for trouble shooting ([#49377](https://github.com/kubernetes/kubernetes/pull/49377), [@CaoShuFeng](https://github.com/CaoShuFeng)) +* Status objects for 404 API errors will have the correct APIVersion ([#49868](https://github.com/kubernetes/kubernetes/pull/49868), [@shiywang](https://github.com/shiywang)) +* Fix incorrect retry logic in scheduler ([#50106](https://github.com/kubernetes/kubernetes/pull/50106), [@julia-stripe](https://github.com/julia-stripe)) +* Enforce explicit references to API group client interfaces in clientsets to avoid ambiguity. ([#49370](https://github.com/kubernetes/kubernetes/pull/49370), [@sttts](https://github.com/sttts)) +* update dashboard image version ([#49855](https://github.com/kubernetes/kubernetes/pull/49855), [@zouyee](https://github.com/zouyee)) +* kubeadm: Implementing the kubeconfig phase fully ([#49419](https://github.com/kubernetes/kubernetes/pull/49419), [@fabriziopandini](https://github.com/fabriziopandini)) +* fixes a bug around using the Global config ElbSecurityGroup where Kuberentes would modify the passed in Security Group. ([#49805](https://github.com/kubernetes/kubernetes/pull/49805), [@nbutton23](https://github.com/nbutton23)) +* Fluentd DaemonSet in the fluentd-elasticsearch addon is configured via ConfigMap and includes journald plugin ([#50082](https://github.com/kubernetes/kubernetes/pull/50082), [@crassirostris](https://github.com/crassirostris)) + * Elasticsearch StatefulSet in the fluentd-elasticsearch addon uses local storage instead of PVC by default +* Add possibility to use multiple floatingip pools in openstack loadbalancer ([#49697](https://github.com/kubernetes/kubernetes/pull/49697), [@zetaab](https://github.com/zetaab)) +* The 504 timeout error was returning a JSON error body that indicated it was a 500. The body contents now correctly report a 500 error. ([#49678](https://github.com/kubernetes/kubernetes/pull/49678), [@smarterclayton](https://github.com/smarterclayton)) +* add examples for kubectl run --labels ([#49862](https://github.com/kubernetes/kubernetes/pull/49862), [@dixudx](https://github.com/dixudx)) +* Kubelet will by default fail with swap enabled from now on. The experimental flag "--experimental-fail-swap-on" has been deprecated, please set the new "--fail-swap-on" flag to false if you wish to run with /proc/swaps on. ([#47181](https://github.com/kubernetes/kubernetes/pull/47181), [@dims](https://github.com/dims)) +* Fix bug in scheduler that caused initially unschedulable pods to stuck in Pending state forever. ([#50028](https://github.com/kubernetes/kubernetes/pull/50028), [@julia-stripe](https://github.com/julia-stripe)) +* GCE: Bump GLBC version to 0.9.6 ([#50096](https://github.com/kubernetes/kubernetes/pull/50096), [@nicksardo](https://github.com/nicksardo)) +* Remove 0,1,3 from rand.String, to avoid 'bad words' ([#50070](https://github.com/kubernetes/kubernetes/pull/50070), [@dixudx](https://github.com/dixudx)) +* Fix data race during addition of new CRD ([#50098](https://github.com/kubernetes/kubernetes/pull/50098), [@nikhita](https://github.com/nikhita)) +* Do not try to run preStopHook when the gracePeriod is 0 ([#49449](https://github.com/kubernetes/kubernetes/pull/49449), [@dhilipkumars](https://github.com/dhilipkumars)) +* The SubjectAccessReview API in the authorization.k8s.io API group now allows providing the user uid. ([#49677](https://github.com/kubernetes/kubernetes/pull/49677), [@dims](https://github.com/dims)) +* Increase default value of apps/v1beta2 DeploymentSpec.RevisionHistoryLimit to 10 ([#49924](https://github.com/kubernetes/kubernetes/pull/49924), [@dixudx](https://github.com/dixudx)) +* Upgrade Elasticsearch/Kibana to 5.5.1 in fluentd-elasticsearch addon ([#48722](https://github.com/kubernetes/kubernetes/pull/48722), [@aknuds1](https://github.com/aknuds1)) + * Switch to basing our image of Elasticsearch in fluentd-elasticsearch addon off the official one + * Switch to the official image of Kibana in fluentd-elasticsearch addon + * Use StatefulSet for Elasticsearch instead of ReplicationController, with persistent volume claims + * Require authenticating towards Elasticsearch, as Elasticsearch 5.5 by default requires basic authentication +* Rebase hyperkube image on debian-hyperkube-base, based on debian-base. ([#48365](https://github.com/kubernetes/kubernetes/pull/48365), [@ixdy](https://github.com/ixdy)) +* change apps/v1beta2 StatefulSet observedGeneration (optional field) from a pointer to an int for consistency ([#49607](https://github.com/kubernetes/kubernetes/pull/49607), [@dixudx](https://github.com/dixudx)) +* After a kubelet rotates its client cert, it now closes its connections to the API server to force a handshake using the new cert. Previously, the kubelet could keep its existing connection open, even if the cert used for that connection was expired and rejected by the API server. ([#49899](https://github.com/kubernetes/kubernetes/pull/49899), [@ericchiang](https://github.com/ericchiang)) +* Improve our Instance Metadata coverage in Azure. ([#49237](https://github.com/kubernetes/kubernetes/pull/49237), [@brendandburns](https://github.com/brendandburns)) +* Add etcd connectivity endpoint to healthz ([#49412](https://github.com/kubernetes/kubernetes/pull/49412), [@bjhaid](https://github.com/bjhaid)) +* kube-proxy will emit "FailedToStartNodeHealthcheck" event when fails to start healthz server. ([#49267](https://github.com/kubernetes/kubernetes/pull/49267), [@MrHohn](https://github.com/MrHohn)) +* Fixed a bug in the API server watch cache, which could cause a missing watch event immediately after cache initialization. ([#49992](https://github.com/kubernetes/kubernetes/pull/49992), [@liggitt](https://github.com/liggitt)) +* Enforcement of fsGroup; enable ScaleIO multiple-instance volume mapping; default PVC capacity; alignment of PVC, PV, and volume names for dynamic provisioning ([#48999](https://github.com/kubernetes/kubernetes/pull/48999), [@vladimirvivien](https://github.com/vladimirvivien)) +* In GCE, add measures to prevent corruption of known_tokens.csv. ([#49897](https://github.com/kubernetes/kubernetes/pull/49897), [@mikedanese](https://github.com/mikedanese)) +* kubeadm: Fix join preflight check false negative ([#49825](https://github.com/kubernetes/kubernetes/pull/49825), [@erhudy](https://github.com/erhudy)) +* route_controller will emit "FailedToCreateRoute" event when fails to create route. ([#49821](https://github.com/kubernetes/kubernetes/pull/49821), [@MrHohn](https://github.com/MrHohn)) +* Fix incorrect parsing of io_priority in Portworx volume StorageClass and add support for new paramters. ([#49526](https://github.com/kubernetes/kubernetes/pull/49526), [@harsh-px](https://github.com/harsh-px)) +* The API Server now automatically creates RBAC ClusterRoles for CSR approving. ([#49284](https://github.com/kubernetes/kubernetes/pull/49284), [@luxas](https://github.com/luxas)) + * Each deployment method should bind users/groups to the ClusterRoles if they are using this feature. +* Adds AllowPrivilegeEscalation to control whether a process can gain more privileges than it's parent process ([#47019](https://github.com/kubernetes/kubernetes/pull/47019), [@jessfraz](https://github.com/jessfraz)) +* `hack/local-up-cluster.sh` now enables the Node authorizer by default. Authorization modes can be overridden with the `AUTHORIZATION_MODE` environment variable, and the `ENABLE_RBAC` environment variable is no longer used. ([#49812](https://github.com/kubernetes/kubernetes/pull/49812), [@liggitt](https://github.com/liggitt)) +* rename stop.go file to delete.go to avoid confusion ([#49533](https://github.com/kubernetes/kubernetes/pull/49533), [@dixudx](https://github.com/dixudx)) +* Adding option to set the federation api server port if nodeport is set ([#46283](https://github.com/kubernetes/kubernetes/pull/46283), [@ktsakalozos](https://github.com/ktsakalozos)) +* The garbage collector now supports custom APIs added via CustomResourceDefinition or aggregated apiservers. Note that the garbage collector controller refreshes periodically, so there is a latency between when the API is added and when the garbage collector starts to manage it. ([#47665](https://github.com/kubernetes/kubernetes/pull/47665), [@ironcladlou](https://github.com/ironcladlou)) +* set the juju master charm state to blocked if the services appear to be failing ([#49717](https://github.com/kubernetes/kubernetes/pull/49717), [@wwwtyro](https://github.com/wwwtyro)) +* keep-terminated-pod-volumes flag on kubelet is deprecated. ([#47539](https://github.com/kubernetes/kubernetes/pull/47539), [@gnufied](https://github.com/gnufied)) +* kubectl describe podsecuritypolicy describes all fields. ([#45813](https://github.com/kubernetes/kubernetes/pull/45813), [@xilabao](https://github.com/xilabao)) +* Added flag support to kubectl plugins ([#47267](https://github.com/kubernetes/kubernetes/pull/47267), [@fabianofranz](https://github.com/fabianofranz)) +* Adding metrics support to local volume ([#49598](https://github.com/kubernetes/kubernetes/pull/49598), [@sbezverk](https://github.com/sbezverk)) +* Bug fix: Parsing of `--requestheader-group-headers` in requests should be case-insensitive. ([#49219](https://github.com/kubernetes/kubernetes/pull/49219), [@jmillikin-stripe](https://github.com/jmillikin-stripe)) +* Fix instance metadata service URL. ([#49081](https://github.com/kubernetes/kubernetes/pull/49081), [@brendandburns](https://github.com/brendandburns)) +* Add a new API object apps/v1beta2.ReplicaSet ([#49238](https://github.com/kubernetes/kubernetes/pull/49238), [@janetkuo](https://github.com/janetkuo)) +* fix pdb validation bug on PodDisruptionBudgetSpec ([#48706](https://github.com/kubernetes/kubernetes/pull/48706), [@dixudx](https://github.com/dixudx)) +* Revert deprecation of vCenter port in vSphere Cloud Provider. ([#49689](https://github.com/kubernetes/kubernetes/pull/49689), [@divyenpatel](https://github.com/divyenpatel)) +* Rev version of Calico's Typha daemon used in add-on to v0.2.3 to pull in bug-fixes. ([#48469](https://github.com/kubernetes/kubernetes/pull/48469), [@fasaxc](https://github.com/fasaxc)) +* set default adminid for rbd deleter if unset ([#49271](https://github.com/kubernetes/kubernetes/pull/49271), [@dixudx](https://github.com/dixudx)) +* Adding type apps/v1beta2.DaemonSet ([#49071](https://github.com/kubernetes/kubernetes/pull/49071), [@foxish](https://github.com/foxish)) +* Fix nil value issue when creating json patch for merge ([#49259](https://github.com/kubernetes/kubernetes/pull/49259), [@dixudx](https://github.com/dixudx)) +* Adds metrics for checking reflector health. ([#48224](https://github.com/kubernetes/kubernetes/pull/48224), [@deads2k](https://github.com/deads2k)) +* remove deads2k from volume reviewer ([#49566](https://github.com/kubernetes/kubernetes/pull/49566), [@deads2k](https://github.com/deads2k)) +* Unify genclient tags and add more fine control on verbs generated ([#49192](https://github.com/kubernetes/kubernetes/pull/49192), [@mfojtik](https://github.com/mfojtik)) +* kubeadm: Fixes a small bug where `--config` and `--skip-*` flags couldn't be passed at the same time in validation. ([#49498](https://github.com/kubernetes/kubernetes/pull/49498), [@luxas](https://github.com/luxas)) +* Remove depreciated flags: --low-diskspace-threshold-mb and --outofdisk-transition-frequency, which are replaced by --eviction-hard ([#48846](https://github.com/kubernetes/kubernetes/pull/48846), [@dashpole](https://github.com/dashpole)) +* Fixed OpenAPI Description and Nickname of API objects with subresources ([#49357](https://github.com/kubernetes/kubernetes/pull/49357), [@mbohlool](https://github.com/mbohlool)) +* set RBD default values as constant vars ([#49274](https://github.com/kubernetes/kubernetes/pull/49274), [@dixudx](https://github.com/dixudx)) +* Fix a bug with binding mount directories and files using flexVolumes ([#49118](https://github.com/kubernetes/kubernetes/pull/49118), [@adelton](https://github.com/adelton)) +* PodPreset is not injected if conflict occurs while applying PodPresets to a Pod. ([#47864](https://github.com/kubernetes/kubernetes/pull/47864), [@droot](https://github.com/droot)) +* `kubectl drain` no longer spins trying to delete pods that do not exist ([#49444](https://github.com/kubernetes/kubernetes/pull/49444), [@eparis](https://github.com/eparis)) +* Support specifying of FSType in StorageClass ([#45345](https://github.com/kubernetes/kubernetes/pull/45345), [@codablock](https://github.com/codablock)) +* The NodeRestriction admission plugin now allows a node to evict pods bound to itself ([#48707](https://github.com/kubernetes/kubernetes/pull/48707), [@danielfm](https://github.com/danielfm)) +* more robust stat handling from ceph df output in the kubernetes-master charm create-rbd-pv action ([#49394](https://github.com/kubernetes/kubernetes/pull/49394), [@wwwtyro](https://github.com/wwwtyro)) +* added cronjobs.batch to all, so kubectl get all returns them. ([#49326](https://github.com/kubernetes/kubernetes/pull/49326), [@deads2k](https://github.com/deads2k)) +* Update status to show failing services. ([#49296](https://github.com/kubernetes/kubernetes/pull/49296), [@ktsakalozos](https://github.com/ktsakalozos)) +* Fixes [#49418](https://github.com/kubernetes/kubernetes/pull/49418) where kube-controller-manager can panic on volume.CanSupport methods and enter a crash loop. ([#49420](https://github.com/kubernetes/kubernetes/pull/49420), [@gnufied](https://github.com/gnufied)) +* Add a new API version apps/v1beta2 ([#48746](https://github.com/kubernetes/kubernetes/pull/48746), [@janetkuo](https://github.com/janetkuo)) +* Websocket requests to aggregated APIs now perform TLS verification using the service DNS name instead of the backend server's IP address, consistent with non-websocket requests. ([#49353](https://github.com/kubernetes/kubernetes/pull/49353), [@liggitt](https://github.com/liggitt)) +* kubeadm: Don't set a specific `spc_t` SELinux label on the etcd Static Pod as that is more privs than etcd needs and due to that `spc_t` isn't compatible with some OSes. ([#49328](https://github.com/kubernetes/kubernetes/pull/49328), [@euank](https://github.com/euank)) +* GCE Cloud Provider: New created LoadBalancer type Service will have health checks for nodes by default if all nodes have version >= v1.7.2. ([#49330](https://github.com/kubernetes/kubernetes/pull/49330), [@MrHohn](https://github.com/MrHohn)) +* hack/local-up-cluster.sh now enables RBAC authorization by default ([#49323](https://github.com/kubernetes/kubernetes/pull/49323), [@mtanino](https://github.com/mtanino)) +* Use port 20256 for node-problem-detector in standalone mode. ([#49316](https://github.com/kubernetes/kubernetes/pull/49316), [@ajitak](https://github.com/ajitak)) +* Fixed unmounting of vSphere volumes when kubelet runs in a container. ([#49111](https://github.com/kubernetes/kubernetes/pull/49111), [@jsafrane](https://github.com/jsafrane)) +* use informers for quota evaluation of core resources where possible ([#49230](https://github.com/kubernetes/kubernetes/pull/49230), [@deads2k](https://github.com/deads2k)) +* additional backoff in azure cloudprovider ([#48967](https://github.com/kubernetes/kubernetes/pull/48967), [@jackfrancis](https://github.com/jackfrancis)) +* allow impersonate serviceaccount in cli ([#48253](https://github.com/kubernetes/kubernetes/pull/48253), [@CaoShuFeng](https://github.com/CaoShuFeng)) +* Add PriorityClass API object under new "scheduling" API group ([#48377](https://github.com/kubernetes/kubernetes/pull/48377), [@bsalamat](https://github.com/bsalamat)) +* None ([#45319](https://github.com/kubernetes/kubernetes/pull/45319), [@ericchiang](https://github.com/ericchiang)) +* Added golint check for pkg/kubelet. ([#47316](https://github.com/kubernetes/kubernetes/pull/47316), [@k82cn](https://github.com/k82cn)) +* azure: acr: support MSI with preview ACR with AAD auth ([#48981](https://github.com/kubernetes/kubernetes/pull/48981), [@colemickens](https://github.com/colemickens)) +* Set default CIDR to /16 for Juju deployments ([#49182](https://github.com/kubernetes/kubernetes/pull/49182), [@ktsakalozos](https://github.com/ktsakalozos)) +* Fix pod preset to ignore input pod namespace in favor of request namespace ([#49120](https://github.com/kubernetes/kubernetes/pull/49120), [@jpeeler](https://github.com/jpeeler)) +* None ([#48983](https://github.com/kubernetes/kubernetes/pull/48983), [@k82cn](https://github.com/k82cn)) +* Previously a deleted bootstrapping token secret would be considered valid until it was reaped. Now it is invalid as soon as the deletionTimestamp is set. ([#49057](https://github.com/kubernetes/kubernetes/pull/49057), [@ericchiang](https://github.com/ericchiang)) +* Set default snap channel on charms to 1.7 stable ([#48874](https://github.com/kubernetes/kubernetes/pull/48874), [@ktsakalozos](https://github.com/ktsakalozos)) +* prevent unsetting of nonexistent previous port in kubeapi-load-balancer charm ([#49033](https://github.com/kubernetes/kubernetes/pull/49033), [@wwwtyro](https://github.com/wwwtyro)) +* kubeadm: Make kube-proxy tolerate the external cloud provider taint so that an external cloud provider can be easily used on top of kubeadm ([#49017](https://github.com/kubernetes/kubernetes/pull/49017), [@luxas](https://github.com/luxas)) +* Fix Pods using Portworx volumes getting stuck in ContainerCreating phase. ([#48898](https://github.com/kubernetes/kubernetes/pull/48898), [@harsh-px](https://github.com/harsh-px)) +* hpa: Prevent scaling below MinReplicas if desiredReplicas is zero ([#48997](https://github.com/kubernetes/kubernetes/pull/48997), [@johanneswuerbach](https://github.com/johanneswuerbach)) +* Kubelet CRI: move seccomp from annotations to security context. ([#46332](https://github.com/kubernetes/kubernetes/pull/46332), [@feiskyer](https://github.com/feiskyer)) +* Never prevent deletion of resources as part of namespace lifecycle ([#48733](https://github.com/kubernetes/kubernetes/pull/48733), [@liggitt](https://github.com/liggitt)) +* The generic RESTClient type (`k8s.io/client-go/rest`) no longer exposes `LabelSelectorParam` or `FieldSelectorParam` methods - use `VersionedParams` with `metav1.ListOptions` instead. The `UintParam` method has been removed. The `timeout` parameter will no longer cause an error when using `Param()`. ([#48991](https://github.com/kubernetes/kubernetes/pull/48991), [@smarterclayton](https://github.com/smarterclayton)) +* Support completion for kubectl config delete-cluster ([#48381](https://github.com/kubernetes/kubernetes/pull/48381), [@superbrothers](https://github.com/superbrothers)) +* Could get the patch from kubectl edit command ([#46091](https://github.com/kubernetes/kubernetes/pull/46091), [@xilabao](https://github.com/xilabao)) +* Added scheduler integration test owners. ([#46930](https://github.com/kubernetes/kubernetes/pull/46930), [@k82cn](https://github.com/k82cn)) +* `kubectl run` learned how to set a service account name in the generated pod spec with the `--serviceaccount` flag. ([#46318](https://github.com/kubernetes/kubernetes/pull/46318), [@liggitt](https://github.com/liggitt)) +* Fix share name generation in azure file provisioner. ([#48326](https://github.com/kubernetes/kubernetes/pull/48326), [@karataliu](https://github.com/karataliu)) +* Fixed a bug where a jsonpath filter would return an error if one of the items being evaluated did not contain all of the nested elements in the filter query. ([#47846](https://github.com/kubernetes/kubernetes/pull/47846), [@ncdc](https://github.com/ncdc)) +* Uses the port config option in the kubeapi-load-balancer charm. ([#48958](https://github.com/kubernetes/kubernetes/pull/48958), [@wwwtyro](https://github.com/wwwtyro)) +* azure: support retrieving access tokens via managed identity extension ([#48854](https://github.com/kubernetes/kubernetes/pull/48854), [@colemickens](https://github.com/colemickens)) +* Add a runtime warning about the kubeadm default token TTL changes. ([#48838](https://github.com/kubernetes/kubernetes/pull/48838), [@mattmoyer](https://github.com/mattmoyer)) +* Azure PD (Managed/Blob) ([#46360](https://github.com/kubernetes/kubernetes/pull/46360), [@khenidak](https://github.com/khenidak)) +* Redirect all examples README to the the kubernetes/examples repo ([#46362](https://github.com/kubernetes/kubernetes/pull/46362), [@sebgoa](https://github.com/sebgoa)) +* Fix a regression that broke the `--config` flag for `kubeadm init`. ([#48915](https://github.com/kubernetes/kubernetes/pull/48915), [@mattmoyer](https://github.com/mattmoyer)) +* Fluentd-gcp DaemonSet exposes different set of metrics. ([#48812](https://github.com/kubernetes/kubernetes/pull/48812), [@crassirostris](https://github.com/crassirostris)) +* MountPath should be absolute ([#48815](https://github.com/kubernetes/kubernetes/pull/48815), [@dixudx](https://github.com/dixudx)) +* Updated comments of func in testapi. ([#48407](https://github.com/kubernetes/kubernetes/pull/48407), [@k82cn](https://github.com/k82cn)) +* Fix service controller crash loop when Service with GCP LoadBalancer uses static IP ([#48848](https://github.com/kubernetes/kubernetes/pull/48848), [@nicksardo](https://github.com/nicksardo)) ([#48849](https://github.com/kubernetes/kubernetes/pull/48849), [@nicksardo](https://github.com/nicksardo)) +* Fix pods failing to start when subPath is a dangling symlink from kubelet point of view, which can happen if it is running inside a container ([#48555](https://github.com/kubernetes/kubernetes/pull/48555), [@redbaron](https://github.com/redbaron)) +* Add initial support for the Azure instance metadata service. ([#48243](https://github.com/kubernetes/kubernetes/pull/48243), [@brendandburns](https://github.com/brendandburns)) +* Added new flag to `kubeadm init`: --node-name, that lets you specify the name of the Node object that will be created ([#48594](https://github.com/kubernetes/kubernetes/pull/48594), [@GheRivero](https://github.com/GheRivero)) +* Added pod evictors for new zone. ([#47952](https://github.com/kubernetes/kubernetes/pull/47952), [@k82cn](https://github.com/k82cn)) +* kube-up and kubemark will default to using cos (GCI) images for nodes. ([#48279](https://github.com/kubernetes/kubernetes/pull/48279), [@abgworrall](https://github.com/abgworrall)) + * The previous default was container-vm (CVM, "debian"), which is deprecated. + * If you need to explicitly use container-vm for some reason, you should set + * KUBE_NODE_OS_DISTRIBUTION=debian +* kubectl: Fix bug that showed terminated/evicted pods even without `--show-all`. ([#48786](https://github.com/kubernetes/kubernetes/pull/48786), [@janetkuo](https://github.com/janetkuo)) +* Fixed GlusterFS volumes taking too long to time out ([#48709](https://github.com/kubernetes/kubernetes/pull/48709), [@jsafrane](https://github.com/jsafrane)) + + + +# v1.6.9 + +[Documentation](https://docs.k8s.io) & [Examples](https://releases.k8s.io/release-1.6/examples) + +## Downloads for v1.6.9 + + +filename | sha256 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes.tar.gz) | `08be94c252e7fbdd7c14811ec021818e687c1259e557b70db10aac64c0e8e4b2` +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-src.tar.gz) | `519501e26afc341b236c5b46602f010a33fc190e3d1bfb7802969b2e979faaeb` + +### Client Binaries + +filename | sha256 hash +-------- | ----------- +[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-darwin-386.tar.gz) | `864f2307dd22c055063d1a55354596754a94d03f023e7278c24d5978bba00b3e` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-darwin-amd64.tar.gz) | `0a107e0a1d7e6865ddd9241f1e8357405f476889a6f1a16989ba01f6cffd3be7` +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-linux-386.tar.gz) | `b20599e266248e7e176383e0318acd855c1aad8014396cc4018adde11a33d0c8` +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-linux-amd64.tar.gz) | `0690a8c9858f91cc000b3acd602799bf2320756b7471e463df2e3a36fbdde886` +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-linux-arm64.tar.gz) | `354897ffc6382b8eb27f434d8e7aa3cbfae4b819da3160a43db8ccb8cae1275b` +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-linux-arm.tar.gz) | `6897408bf8d65d1281555c21ae978a4ccd69482a7ad2549bcec381416e312d7a` +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-linux-ppc64le.tar.gz) | `2afae0c211eb415829446f90a0bf9d48b9f8311ac4566fa74a08415ed9a31e75` +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-linux-s390x.tar.gz) | `abde354528cc9c8ced49bb767ffcd8bfae47a0b4b5501502f560cf663a0c4a05` +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-windows-386.tar.gz) | `83083c0d78e9468c7be395282a4697d2c703d3310593e7b70cd09fa9e7791d80` +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-client-windows-amd64.tar.gz) | `3471db3463d60d22d82edb34fbe3ca301cc583ebddffc2664569255302e7d304` + +### Server Binaries + +filename | sha256 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-server-linux-amd64.tar.gz) | `598e49c8a22e4e8db1e1c0ed9d8955c991425cd4e06c072ac36fd5ed693b1c61` +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-server-linux-arm64.tar.gz) | `5ce75f57636d537b4bf3ca00c4a1322e9c1aaf273bd945304333b558af3c081b` +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-server-linux-arm.tar.gz) | `afea9780049c5e6548f64973bd8679aae60672ab05027f8c36784ccf2a83a1b2` +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-server-linux-ppc64le.tar.gz) | `cd131b3e39e4160cd9920fe2635b4f6da4679cce12cb2483cfe28197e366bceb` +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-server-linux-s390x.tar.gz) | `93ee43f33cbe061ac088acf62099be1abd0d9c0b4a8a79be4069904c3780c76d` + +### Node Binaries + +filename | sha256 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-node-linux-amd64.tar.gz) | `f8f13233b168c4833af685817f9591c73658d1377ceb9d550cbea929c6e27c2e` +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-node-linux-arm64.tar.gz) | `1ed434f9e6469c8cc7a3bb15404e918cf242ef92ef075e7cf479b7e951269b5c` +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-node-linux-arm.tar.gz) | `3fd8e089184f83bd9ed2cf5f193253e1f7b9b853876a08a2babf91647d6d0ac8` +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-node-linux-ppc64le.tar.gz) | `9673547a32f83498bb28f02212d419b28cc50a0a4d7b866396994b5ea9313e79` +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-node-linux-s390x.tar.gz) | `80044cdeb4260e807660c166ed15bb2a9db03d59d8c186b1d4f9a53841cea327` +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.6.9/kubernetes-node-windows-amd64.tar.gz) | `b1dd678ee2974dc83ea7cfe8516557c9360ed55e40cad1b68803b71786f8d16f` + +## Changelog since v1.6.8 + +### Other notable changes + +* StatefulSet: Set hostname/subdomain fields on new Pods in addition to the deprecated annotations, to allow mitigation of Pod DNS issues upon upgrading to Kubernetes v1.7.x. ([#50942](https://github.com/kubernetes/kubernetes/pull/50942), [@enisoc](https://github.com/enisoc)) +* Azure: Allow VNet to be in a separate Resource Group. ([#49725](https://github.com/kubernetes/kubernetes/pull/49725), [@sylr](https://github.com/sylr)) +* In GCE, add measures to prevent corruption of known_tokens.csv. ([#49897](https://github.com/kubernetes/kubernetes/pull/49897), [@mikedanese](https://github.com/mikedanese)) +* Fixed a bug in the API server watch cache, which could cause a missing watch event immediately after cache initialization. ([#49992](https://github.com/kubernetes/kubernetes/pull/49992), [@liggitt](https://github.com/liggitt)) + + + # v1.7.4 [Documentation](https://docs.k8s.io) & [Examples](https://releases.k8s.io/release-1.7/examples) @@ -2653,7 +2999,7 @@ filename | sha256 hash * AWS: Avoid spurious ELB listener recreation - ignore case when matching protocol ([#47391](https://github.com/kubernetes/kubernetes/pull/47391), [@justinsb](https://github.com/justinsb)) * gce kube-up: The `Node` authorization mode and `NodeRestriction` admission controller are now enabled ([#46796](https://github.com/kubernetes/kubernetes/pull/46796), [@mikedanese](https://github.com/mikedanese)) * update gophercloud/gophercloud dependency for reauthentication fixes ([#45545](https://github.com/kubernetes/kubernetes/pull/45545), [@stuart-warren](https://github.com/stuart-warren)) -* fix sync loop health check with seperating runtime errors ([#47124](https://github.com/kubernetes/kubernetes/pull/47124), [@andyxning](https://github.com/andyxning)) +* fix sync loop health check with separating runtime errors ([#47124](https://github.com/kubernetes/kubernetes/pull/47124), [@andyxning](https://github.com/andyxning)) * servicecontroller: Fix node selection logic on initial LB creation ([#45773](https://github.com/kubernetes/kubernetes/pull/45773), [@justinsb](https://github.com/justinsb)) * Fix iSCSI iSER mounting. ([#47281](https://github.com/kubernetes/kubernetes/pull/47281), [@mtanino](https://github.com/mtanino)) * StorageOS Volume Driver ([#42156](https://github.com/kubernetes/kubernetes/pull/42156), [@croomes](https://github.com/croomes)) @@ -2915,7 +3261,7 @@ filename | sha256 hash * AWS: Avoid spurious ELB listener recreation - ignore case when matching protocol ([#47391](https://github.com/kubernetes/kubernetes/pull/47391), [@justinsb](https://github.com/justinsb)) * gce kube-up: The `Node` authorization mode and `NodeRestriction` admission controller are now enabled ([#46796](https://github.com/kubernetes/kubernetes/pull/46796), [@mikedanese](https://github.com/mikedanese)) * update gophercloud/gophercloud dependency for reauthentication fixes ([#45545](https://github.com/kubernetes/kubernetes/pull/45545), [@stuart-warren](https://github.com/stuart-warren)) -* fix sync loop health check with seperating runtime errors ([#47124](https://github.com/kubernetes/kubernetes/pull/47124), [@andyxning](https://github.com/andyxning)) +* fix sync loop health check with separating runtime errors ([#47124](https://github.com/kubernetes/kubernetes/pull/47124), [@andyxning](https://github.com/andyxning)) * servicecontroller: Fix node selection logic on initial LB creation ([#45773](https://github.com/kubernetes/kubernetes/pull/45773), [@justinsb](https://github.com/justinsb)) * Fix iSCSI iSER mounting. ([#47281](https://github.com/kubernetes/kubernetes/pull/47281), [@mtanino](https://github.com/mtanino)) * StorageOS Volume Driver ([#42156](https://github.com/kubernetes/kubernetes/pull/42156), [@croomes](https://github.com/croomes)) @@ -3854,7 +4200,7 @@ filename | sha256 hash * Adds annotations to all Federation objects created by kubefed. ([#42683](https://github.com/kubernetes/kubernetes/pull/42683), [@perotinus](https://github.com/perotinus)) * [Federation][Kubefed] Bug fix to enable disabling federation controllers through override args ([#44209](https://github.com/kubernetes/kubernetes/pull/44209), [@irfanurrehman](https://github.com/irfanurrehman)) * [Federation] Remove deprecated federation-apiserver-kubeconfig secret ([#44287](https://github.com/kubernetes/kubernetes/pull/44287), [@shashidharatd](https://github.com/shashidharatd)) -* Scheduler can recieve its policy configuration from a ConfigMap ([#43892](https://github.com/kubernetes/kubernetes/pull/43892), [@bsalamat](https://github.com/bsalamat)) +* Scheduler can receive its policy configuration from a ConfigMap ([#43892](https://github.com/kubernetes/kubernetes/pull/43892), [@bsalamat](https://github.com/bsalamat)) * AWS cloud provider: fix support running the master with a different AWS account or even on a different cloud provider than the nodes. ([#44235](https://github.com/kubernetes/kubernetes/pull/44235), [@mrIncompetent](https://github.com/mrIncompetent)) * add rancher credential provider ([#40160](https://github.com/kubernetes/kubernetes/pull/40160), [@wlan0](https://github.com/wlan0)) * Support generating Open API extensions for strategic merge patch tags in go struct tags ([#44121](https://github.com/kubernetes/kubernetes/pull/44121), [@mbohlool](https://github.com/mbohlool)) @@ -4313,7 +4659,7 @@ Features for this release were tracked via the use of the [kubernetes/features]( * **[beta]** `kubefed` has graduated to beta: supports hosting federation on on-prem clusters, automatically configures `kube-dns` in joining clusters and allows passing arguments to federation components. ### Internal Storage Layer -* **[stable]** The internal storage layer for kubernetes cluster state has been updated to use etcd v3 by default. Existing clusters will have to plan for a data migration window. ([docs](https://github.com/kubernetes/kubernetes.github.io/pull/2763))([kubernetes/features#44](https://github.com/kubernetes/features/issues/44)) +* **[stable]** The internal storage layer for kubernetes cluster state has been updated to use etcd v3 by default. Existing clusters will have to plan for a data migration window. ([docs](https://kubernetes.io/docs/tasks/administer-cluster/upgrade-1-6/))([kubernetes/features#44](https://github.com/kubernetes/features/issues/44)) ### kubeadm * **[beta]** Introduces an API for clients to request TLS certificates from the API server. See the [tutorial](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster). @@ -4403,7 +4749,7 @@ Features for this release were tracked via the use of the [kubernetes/features]( * Remove the deprecated vsphere kube-up. ([#39140](https://github.com/kubernetes/kubernetes/pull/39140), [@kerneltime](https://github.com/kerneltime)) ### kubeadm -* Quite a few flags been renamed or removed. Those options that are removed as flags can still be accessed via the config file. Most noteably this includes external etcd settings and the option for setting the cloud provider on the API server. The [kubeadm reference documentation](https://kubernetes.io/docs/admin/kubeadm/) is up to date with the new flags. +* Quite a few flags been renamed or removed. Those options that are removed as flags can still be accessed via the config file. Most notably this includes external etcd settings and the option for setting the cloud provider on the API server. The [kubeadm reference documentation](https://kubernetes.io/docs/admin/kubeadm/) is up to date with the new flags. ### Other Deprecations * Remove cmd/kube-discovery from the tree since it's not necessary anymore ([#42070](https://github.com/kubernetes/kubernetes/pull/42070), [@luxas](https://github.com/luxas)) @@ -6459,7 +6805,7 @@ release [38537](https://github.com/kubernetes/kubernetes/issues/38537) * extensions/v1beta1.Jobs is deprecated, use batch/v1.Job instead ([#36355](https://github.com/kubernetes/kubernetes/pull/36355), [@soltysh](https://github.com/soltysh)) * The kubelet --reconcile-cdir flag is deprecated because it has no function anymore. ([#35523](https://github.com/kubernetes/kubernetes/pull/35523), [@luxas](https://github.com/luxas)) * Notice of deprecation for recycler [#36760](https://github.com/kubernetes/kubernetes/pull/36760) -* The init-container (pod.beta.kubernetes.io/init-containers) annotations used to accept capitalized field names that could be accidently generated by the k8s.io/kubernetes/pkg/api package. Using an upper case field name will now return an error and all users should use the versioned API types from `pkg/api/v1` when serializing from Golang. +* The init-container (pod.beta.kubernetes.io/init-containers) annotations used to accept capitalized field names that could be accidentally generated by the k8s.io/kubernetes/pkg/api package. Using an upper case field name will now return an error and all users should use the versioned API types from `pkg/api/v1` when serializing from Golang. ## Action Required Before Upgrading @@ -7678,7 +8024,7 @@ This is the first release tracked via the use of the [kubernetes/features](https - **Scheduling** - [alpha] Allows pods to require or prohibit (or prefer or prefer not) co-scheduling on the same node (or zone or other topology domain) as another set of pods. ([docs](http://kubernetes.io/docs/user-guide/node-selection/) ([kubernetes/features#51](https://github.com/kubernetes/features/issues/51)) - **Storage** - - [beta] Persistant Volume provisioning now supports multiple provisioners using StorageClass configuration. ([docs](http://kubernetes.io/docs/user-guide/persistent-volumes/)) ([kubernetes/features#36](https://github.com/kubernetes/features/issues/36)) + - [beta] Persistent Volume provisioning now supports multiple provisioners using StorageClass configuration. ([docs](http://kubernetes.io/docs/user-guide/persistent-volumes/)) ([kubernetes/features#36](https://github.com/kubernetes/features/issues/36)) - [stable] New volume plugin for the Quobyte Distributed File System ([docs](http://kubernetes.io/docs/user-guide/volumes/#quobyte)) ([kubernetes/features#80](https://github.com/kubernetes/features/issues/80)) - [stable] New volume plugin for Azure Data Disk ([docs](http://kubernetes.io/docs/user-guide/volumes/#azurediskvolume)) ([kubernetes/features#79](https://github.com/kubernetes/features/issues/79)) - **UI** diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 5ab012331896e..77ab2ffbbbf8a 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1100,6 +1100,11 @@ "ImportPath": "github.com/fatih/camelcase", "Rev": "f6a740d52f961c60348ebb109adde9f4635d7540" }, + { + "ImportPath": "github.com/fatih/structs", + "Comment": "v1.0-4-g7e5a8ee", + "Rev": "7e5a8eef611ee84dd359503f3969f80df4c50723" + }, { "ImportPath": "github.com/fsnotify/fsnotify", "Comment": "v1.3.1-1-gf12c623", @@ -1330,214 +1335,218 @@ "ImportPath": "github.com/golang/protobuf/ptypes/timestamp", "Rev": "4bd1920723d7b7c925de087aa32e2187708897f7" }, + { + "ImportPath": "github.com/golang/snappy", + "Rev": "553a641470496b2327abcac10b36396bd98e45c9" + }, { "ImportPath": "github.com/google/btree", "Rev": "7d79101e329e5a3adf994758c578dab82b90c017" }, { "ImportPath": "github.com/google/cadvisor/api", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/cache/memory", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/client/v2", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/collector", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/container", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/container/common", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/container/docker", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/container/libcontainer", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/container/raw", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/container/rkt", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/container/systemd", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/devicemapper", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/events", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/fs", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/healthz", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/http", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/http/mux", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/info/v1", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/info/v2", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/machine", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/manager", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/manager/watcher", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/manager/watcher/raw", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/manager/watcher/rkt", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/metrics", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/pages", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/pages/static", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/storage", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/summary", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/utils", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/utils/cloudinfo", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/utils/cpuload", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/utils/cpuload/netlink", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/utils/docker", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/utils/oomparser", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/utils/sysfs", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/utils/sysinfo", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/utils/tail", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/validate", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/version", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/cadvisor/zfs", - "Comment": "v0.26.1", - "Rev": "d19cc94b760cd8f150a0a5d95b404dec39a121a1" + "Comment": "v0.26.0-37-g27e1acb", + "Rev": "27e1acbb4ef0fe1889208b21f8f4a6d0863e02f6" }, { "ImportPath": "github.com/google/certificate-transparency/go", @@ -1731,6 +1740,22 @@ "Comment": "v1.1.0-25-g84398b9", "Rev": "84398b94e188ee336f307779b57b3aa91af7063c" }, + { + "ImportPath": "github.com/hashicorp/errwrap", + "Rev": "7554cd9344cec97297fa6649b055a8c98c2a1e55" + }, + { + "ImportPath": "github.com/hashicorp/go-cleanhttp", + "Rev": "3573b8b52aa7b37b9358d966a898feb387f62437" + }, + { + "ImportPath": "github.com/hashicorp/go-multierror", + "Rev": "83588e72410abfbe4df460eeb6f30841ae47d4c4" + }, + { + "ImportPath": "github.com/hashicorp/go-rootcerts", + "Rev": "6bb64b370b90e7ef1fa532be9e591a81c3493e00" + }, { "ImportPath": "github.com/hashicorp/golang-lru", "Rev": "a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4" @@ -1775,6 +1800,26 @@ "ImportPath": "github.com/hashicorp/hcl/json/token", "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" }, + { + "ImportPath": "github.com/hashicorp/vault/api", + "Comment": "v0.8.1-22-g52401fd", + "Rev": "52401fd0e5d983e44112c81d933553d2c03f612a" + }, + { + "ImportPath": "github.com/hashicorp/vault/helper/compressutil", + "Comment": "v0.8.1-22-g52401fd", + "Rev": "52401fd0e5d983e44112c81d933553d2c03f612a" + }, + { + "ImportPath": "github.com/hashicorp/vault/helper/jsonutil", + "Comment": "v0.8.1-22-g52401fd", + "Rev": "52401fd0e5d983e44112c81d933553d2c03f612a" + }, + { + "ImportPath": "github.com/hashicorp/vault/helper/parseutil", + "Comment": "v0.8.1-22-g52401fd", + "Rev": "52401fd0e5d983e44112c81d933553d2c03f612a" + }, { "ImportPath": "github.com/hawkular/hawkular-client-go/metrics", "Comment": "v0.5.1-1-g1d46ce7", @@ -1934,6 +1979,10 @@ "Comment": "v2.1.1-5-g1b4ae6f", "Rev": "1b4ae6fb4e77b095934d4430860ff202060169f8" }, + { + "ImportPath": "github.com/mitchellh/go-homedir", + "Rev": "b8bc1bf767474819792c23f32d8286a45736f1c6" + }, { "ImportPath": "github.com/mitchellh/go-wordwrap", "Rev": "ad45545899c7b13c020ea92b2072220eefad42b8" @@ -2434,6 +2483,10 @@ "ImportPath": "github.com/seccomp/libseccomp-golang", "Rev": "1b506fc7c24eec5a3693cdcbed40d9c226cfc6a1" }, + { + "ImportPath": "github.com/sethgrid/pester", + "Rev": "a86a2d88f4dc3c7dbf3a6a6bbbfb095690b834b6" + }, { "ImportPath": "github.com/shurcooL/sanitized_anchor_name", "Rev": "10ef21a441db47d8b13ebcc5fd2310f636973c77" @@ -2715,51 +2768,51 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/context/ctxhttp", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/html", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/html/atom", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/internal/timeseries", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/proxy", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/trace", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/oauth2", @@ -2791,63 +2844,67 @@ }, { "ImportPath": "golang.org/x/text/cases", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/encoding", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/encoding/internal", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/encoding/internal/identifier", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/encoding/unicode", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" + }, + { + "ImportPath": "golang.org/x/text/internal", + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/internal/tag", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/internal/utf8internal", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/language", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/runes", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/bidirule", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/precis", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/transform", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/bidi", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/norm", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/width", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/time/rate", @@ -2859,100 +2916,129 @@ }, { "ImportPath": "google.golang.org/api/cloudkms/v1", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/cloudmonitoring/v2beta2", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/compute/v0.alpha", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/compute/v0.beta", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/compute/v1", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/container/v1", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/dns/v1", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/gensupport", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/googleapi", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/googleapi/internal/uritemplates", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/logging/v2beta1", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/monitoring/v3", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" }, { "ImportPath": "google.golang.org/api/pubsub/v1", - "Rev": "e3824ed33c72bf7e81da0286772c34b987520914" + "Rev": "98825bb0065da4054e5da6db34f5fc598e50bc24" + }, + { + "ImportPath": "google.golang.org/genproto/googleapis/rpc/status", + "Rev": "09f6ed296fc66555a25fe4ce95173148778dfa85" }, { "ImportPath": "google.golang.org/grpc", - "Comment": "v1.0.4", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/codes", - "Comment": "v1.0.4", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/credentials", - "Comment": "v1.0.4", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/grpclb/grpc_lb_v1", + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/grpclog", - "Comment": "v1.0.4", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/internal", - "Comment": "v1.0.4", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/keepalive", + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/metadata", - "Comment": "v1.0.4", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/naming", - "Comment": "v1.0.4", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/peer", - "Comment": "v1.0.4", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/stats", + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/status", + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/tap", + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/transport", - "Comment": "v1.0.4", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Comment": "v1.3.0", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "gopkg.in/gcfg.v1", diff --git a/Godeps/LICENSES b/Godeps/LICENSES index b5c59d6a43487..0da9359cb2918 100644 --- a/Godeps/LICENSES +++ b/Godeps/LICENSES @@ -38416,6 +38416,34 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================================ +================================================================================ += vendor/github.com/fatih/structs licensed under: = + +The MIT License (MIT) + +Copyright (c) 2014 Fatih Arslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. += vendor/github.com/fatih/structs/LICENSE 9ee102d4ddee082e91c1fce103a8cfdd - +================================================================================ + + ================================================================================ = vendor/github.com/fsnotify/fsnotify licensed under: = @@ -43151,231 +43179,43 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/btree licensed under: = - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and += vendor/github.com/golang/snappy licensed under: = - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - http://www.apache.org/licenses/LICENSE-2.0 + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/github.com/google/btree/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 - += vendor/github.com/golang/snappy/LICENSE b8b79c7d4cda128290b98c6a21f9aac6 - ================================================================================ ================================================================================ -= vendor/github.com/google/cadvisor/api licensed under: = - - Copyright 2014 The cAdvisor Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 += vendor/github.com/google/btree licensed under: = - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. Apache License Version 2.0, January 2004 @@ -43554,14 +43394,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. END OF TERMS AND CONDITIONS -= vendor/github.com/google/cadvisor/LICENSE e7790b946bfacb700e8a8f2baedb3205 - -================================================================================ - + APPENDIX: How to apply the Apache License to your work. -================================================================================ -= vendor/github.com/google/cadvisor/cache/memory licensed under: = + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. - Copyright 2014 The cAdvisor Authors + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -43575,189 +43419,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. See the License for the specific language governing permissions and limitations under the License. - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - -= vendor/github.com/google/cadvisor/LICENSE e7790b946bfacb700e8a8f2baedb3205 - += vendor/github.com/google/btree/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 - ================================================================================ ================================================================================ -= vendor/github.com/google/cadvisor/client/v2 licensed under: = += vendor/github.com/google/cadvisor/api licensed under: = Copyright 2014 The cAdvisor Authors @@ -43955,7 +43622,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/collector licensed under: = += vendor/github.com/google/cadvisor/cache/memory licensed under: = Copyright 2014 The cAdvisor Authors @@ -44153,7 +43820,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/container licensed under: = += vendor/github.com/google/cadvisor/client/v2 licensed under: = Copyright 2014 The cAdvisor Authors @@ -44351,7 +44018,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/container/common licensed under: = += vendor/github.com/google/cadvisor/collector licensed under: = Copyright 2014 The cAdvisor Authors @@ -44549,7 +44216,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/container/docker licensed under: = += vendor/github.com/google/cadvisor/container licensed under: = Copyright 2014 The cAdvisor Authors @@ -44747,7 +44414,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/container/libcontainer licensed under: = += vendor/github.com/google/cadvisor/container/common licensed under: = Copyright 2014 The cAdvisor Authors @@ -44945,7 +44612,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/container/raw licensed under: = += vendor/github.com/google/cadvisor/container/docker licensed under: = Copyright 2014 The cAdvisor Authors @@ -45143,7 +44810,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/container/rkt licensed under: = += vendor/github.com/google/cadvisor/container/libcontainer licensed under: = Copyright 2014 The cAdvisor Authors @@ -45341,7 +45008,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/container/systemd licensed under: = += vendor/github.com/google/cadvisor/container/raw licensed under: = Copyright 2014 The cAdvisor Authors @@ -45539,7 +45206,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/devicemapper licensed under: = += vendor/github.com/google/cadvisor/container/rkt licensed under: = Copyright 2014 The cAdvisor Authors @@ -45737,7 +45404,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/events licensed under: = += vendor/github.com/google/cadvisor/container/systemd licensed under: = Copyright 2014 The cAdvisor Authors @@ -45935,7 +45602,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/fs licensed under: = += vendor/github.com/google/cadvisor/devicemapper licensed under: = Copyright 2014 The cAdvisor Authors @@ -46133,7 +45800,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/healthz licensed under: = += vendor/github.com/google/cadvisor/events licensed under: = Copyright 2014 The cAdvisor Authors @@ -46331,7 +45998,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/http licensed under: = += vendor/github.com/google/cadvisor/fs licensed under: = Copyright 2014 The cAdvisor Authors @@ -46529,7 +46196,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/http/mux licensed under: = += vendor/github.com/google/cadvisor/healthz licensed under: = Copyright 2014 The cAdvisor Authors @@ -46727,7 +46394,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/info/v1 licensed under: = += vendor/github.com/google/cadvisor/http licensed under: = Copyright 2014 The cAdvisor Authors @@ -46925,7 +46592,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/info/v2 licensed under: = += vendor/github.com/google/cadvisor/http/mux licensed under: = Copyright 2014 The cAdvisor Authors @@ -47123,7 +46790,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/machine licensed under: = += vendor/github.com/google/cadvisor/info/v1 licensed under: = Copyright 2014 The cAdvisor Authors @@ -47321,7 +46988,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/manager licensed under: = += vendor/github.com/google/cadvisor/info/v2 licensed under: = Copyright 2014 The cAdvisor Authors @@ -47519,7 +47186,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/manager/watcher licensed under: = += vendor/github.com/google/cadvisor/machine licensed under: = Copyright 2014 The cAdvisor Authors @@ -47717,7 +47384,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/manager/watcher/raw licensed under: = += vendor/github.com/google/cadvisor/manager licensed under: = Copyright 2014 The cAdvisor Authors @@ -47915,7 +47582,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/manager/watcher/rkt licensed under: = += vendor/github.com/google/cadvisor/manager/watcher licensed under: = Copyright 2014 The cAdvisor Authors @@ -48113,7 +47780,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/metrics licensed under: = += vendor/github.com/google/cadvisor/manager/watcher/raw licensed under: = Copyright 2014 The cAdvisor Authors @@ -48311,7 +47978,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/pages licensed under: = += vendor/github.com/google/cadvisor/manager/watcher/rkt licensed under: = Copyright 2014 The cAdvisor Authors @@ -48509,7 +48176,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/pages/static licensed under: = += vendor/github.com/google/cadvisor/metrics licensed under: = Copyright 2014 The cAdvisor Authors @@ -48707,7 +48374,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/storage licensed under: = += vendor/github.com/google/cadvisor/pages licensed under: = Copyright 2014 The cAdvisor Authors @@ -48905,7 +48572,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/summary licensed under: = += vendor/github.com/google/cadvisor/pages/static licensed under: = Copyright 2014 The cAdvisor Authors @@ -49103,7 +48770,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/utils licensed under: = += vendor/github.com/google/cadvisor/storage licensed under: = Copyright 2014 The cAdvisor Authors @@ -49301,7 +48968,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/utils/cloudinfo licensed under: = += vendor/github.com/google/cadvisor/summary licensed under: = Copyright 2014 The cAdvisor Authors @@ -49499,7 +49166,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/utils/cpuload licensed under: = += vendor/github.com/google/cadvisor/utils licensed under: = Copyright 2014 The cAdvisor Authors @@ -49697,7 +49364,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/utils/cpuload/netlink licensed under: = += vendor/github.com/google/cadvisor/utils/cloudinfo licensed under: = Copyright 2014 The cAdvisor Authors @@ -49895,7 +49562,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/utils/docker licensed under: = += vendor/github.com/google/cadvisor/utils/cpuload licensed under: = Copyright 2014 The cAdvisor Authors @@ -50093,7 +49760,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/utils/oomparser licensed under: = += vendor/github.com/google/cadvisor/utils/cpuload/netlink licensed under: = Copyright 2014 The cAdvisor Authors @@ -50291,7 +49958,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/utils/sysfs licensed under: = += vendor/github.com/google/cadvisor/utils/docker licensed under: = Copyright 2014 The cAdvisor Authors @@ -50489,7 +50156,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/utils/sysinfo licensed under: = += vendor/github.com/google/cadvisor/utils/oomparser licensed under: = Copyright 2014 The cAdvisor Authors @@ -50687,7 +50354,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/utils/tail licensed under: = += vendor/github.com/google/cadvisor/utils/sysfs licensed under: = Copyright 2014 The cAdvisor Authors @@ -50885,7 +50552,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/validate licensed under: = += vendor/github.com/google/cadvisor/utils/sysinfo licensed under: = Copyright 2014 The cAdvisor Authors @@ -51083,7 +50750,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/version licensed under: = += vendor/github.com/google/cadvisor/utils/tail licensed under: = Copyright 2014 The cAdvisor Authors @@ -51281,7 +50948,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/cadvisor/zfs licensed under: = += vendor/github.com/google/cadvisor/validate licensed under: = Copyright 2014 The cAdvisor Authors @@ -51479,8 +51146,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/certificate-transparency/go licensed under: = += vendor/github.com/google/cadvisor/version licensed under: = + Copyright 2014 The cAdvisor Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. Apache License Version 2.0, January 2004 @@ -51659,18 +51339,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. += vendor/github.com/google/cadvisor/LICENSE e7790b946bfacb700e8a8f2baedb3205 - +================================================================================ - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - Copyright [yyyy] [name of copyright owner] +================================================================================ += vendor/github.com/google/cadvisor/zfs licensed under: = + + Copyright 2014 The cAdvisor Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -51684,12 +51360,189 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. See the License for the specific language governing permissions and limitations under the License. -= vendor/github.com/google/certificate-transparency/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 - + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + += vendor/github.com/google/cadvisor/LICENSE e7790b946bfacb700e8a8f2baedb3205 - ================================================================================ ================================================================================ -= vendor/github.com/google/certificate-transparency/go/asn1 licensed under: = += vendor/github.com/google/certificate-transparency/go licensed under: = Apache License @@ -51899,7 +51752,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/certificate-transparency/go/client licensed under: = += vendor/github.com/google/certificate-transparency/go/asn1 licensed under: = Apache License @@ -52109,7 +51962,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/certificate-transparency/go/x509 licensed under: = += vendor/github.com/google/certificate-transparency/go/client licensed under: = Apache License @@ -52319,7 +52172,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/certificate-transparency/go/x509/pkix licensed under: = += vendor/github.com/google/certificate-transparency/go/x509 licensed under: = Apache License @@ -52529,7 +52382,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/google/gofuzz licensed under: = += vendor/github.com/google/certificate-transparency/go/x509/pkix licensed under: = Apache License @@ -52734,12 +52587,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. See the License for the specific language governing permissions and limitations under the License. -= vendor/github.com/google/gofuzz/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 - += vendor/github.com/google/certificate-transparency/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 - ================================================================================ ================================================================================ -= vendor/github.com/googleapis/gnostic/compiler licensed under: = += vendor/github.com/google/gofuzz licensed under: = Apache License @@ -52944,13 +52797,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. See the License for the specific language governing permissions and limitations under the License. - -= vendor/github.com/googleapis/gnostic/LICENSE b1e01b26bacfc2232046c90a330332b3 - += vendor/github.com/google/gofuzz/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 - ================================================================================ ================================================================================ -= vendor/github.com/googleapis/gnostic/extensions licensed under: = += vendor/github.com/googleapis/gnostic/compiler licensed under: = Apache License @@ -53161,7 +53013,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/googleapis/gnostic/OpenAPIv2 licensed under: = += vendor/github.com/googleapis/gnostic/extensions licensed under: = Apache License @@ -53372,24 +53224,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/gophercloud/gophercloud licensed under: = - -Copyright 2012-2013 Rackspace, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of the -License at - - http://www.apache.org/licenses/LICENSE-2.0 += vendor/github.com/googleapis/gnostic/OpenAPIv2 licensed under: = -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. ------- - - Apache License + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -53566,211 +53404,38 @@ specific language governing permissions and limitations under the License. END OF TERMS AND CONDITIONS -= vendor/github.com/gophercloud/gophercloud/LICENSE dd19699707373c2ca31531a659130416 - -================================================================================ - - -================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack licensed under: = - -Copyright 2012-2013 Rackspace, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of the -License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. - ------- - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. + APPENDIX: How to apply the Apache License to your work. - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. + Copyright [yyyy] [name of copyright owner] - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. + http://www.apache.org/licenses/LICENSE-2.0 - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. - END OF TERMS AND CONDITIONS -= vendor/github.com/gophercloud/gophercloud/LICENSE dd19699707373c2ca31531a659130416 - += vendor/github.com/googleapis/gnostic/LICENSE b1e01b26bacfc2232046c90a330332b3 - ================================================================================ ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/apiversions licensed under: = += vendor/github.com/gophercloud/gophercloud licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -53969,7 +53634,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -54168,7 +53833,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/apiversions licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -54367,7 +54032,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/common/extensions licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -54566,7 +54231,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -54765,7 +54430,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/common/extensions licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -54964,7 +54629,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/flavors licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -55163,7 +54828,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/images licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -55362,7 +55027,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/flavors licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -55561,7 +55226,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/identity/v2/tenants licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/images licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -55760,7 +55425,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/identity/v2/tokens licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -55959,7 +55624,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/identity/v2/tenants licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -56158,7 +55823,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/identity/v3/tokens licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/identity/v2/tokens licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -56357,7 +56022,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -56556,7 +56221,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/identity/v3/tokens licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -56755,7 +56420,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -56954,7 +56619,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/members licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -57153,7 +56818,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/monitors licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -57352,7 +57017,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/pools licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/members licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -57551,7 +57216,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/vips licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/monitors licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -57750,7 +57415,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/listeners licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/pools licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -57949,7 +57614,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/loadbalancers licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/vips licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -58148,7 +57813,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/monitors licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/listeners licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -58347,7 +58012,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/pools licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/loadbalancers licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -58546,7 +58211,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/monitors licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -58745,7 +58410,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/pools licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -58944,7 +58609,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -59143,7 +58808,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/openstack/utils licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -59342,7 +59007,7 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gophercloud/gophercloud/pagination licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports licensed under: = Copyright 2012-2013 Rackspace, Inc. @@ -59541,109 +59206,507 @@ specific language governing permissions and limitations under the License. ================================================================================ -= vendor/github.com/gorilla/context licensed under: = += vendor/github.com/gophercloud/gophercloud/openstack/utils licensed under: = -Copyright (c) 2012 Rodrigo Moraes. All rights reserved. +Copyright 2012-2013 Rackspace, Inc. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. + http://www.apache.org/licenses/LICENSE-2.0 -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. -= vendor/github.com/gorilla/context/LICENSE c50f6bd9c1e15ed0bad3bea18e3c1b7f - -================================================================================ +------ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -================================================================================ -= vendor/github.com/gorilla/mux licensed under: = + 1. Definitions. -Copyright (c) 2012 Rodrigo Moraes. All rights reserved. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. -= vendor/github.com/gorilla/mux/LICENSE c50f6bd9c1e15ed0bad3bea18e3c1b7f - -================================================================================ + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. -================================================================================ -= vendor/github.com/gorilla/websocket licensed under: = + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). -Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." - Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. - Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. -= vendor/github.com/gorilla/websocket/LICENSE c007b54a1743d596f46b2748d9f8c044 - + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + += vendor/github.com/gophercloud/gophercloud/LICENSE dd19699707373c2ca31531a659130416 - ================================================================================ ================================================================================ -= vendor/github.com/grpc-ecosystem/go-grpc-prometheus licensed under: = += vendor/github.com/gophercloud/gophercloud/pagination licensed under: = - Apache License +Copyright 2012-2013 Rackspace, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +------ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + += vendor/github.com/gophercloud/gophercloud/LICENSE dd19699707373c2ca31531a659130416 - +================================================================================ + + +================================================================================ += vendor/github.com/gorilla/context licensed under: = + +Copyright (c) 2012 Rodrigo Moraes. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/github.com/gorilla/context/LICENSE c50f6bd9c1e15ed0bad3bea18e3c1b7f - +================================================================================ + + +================================================================================ += vendor/github.com/gorilla/mux licensed under: = + +Copyright (c) 2012 Rodrigo Moraes. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/github.com/gorilla/mux/LICENSE c50f6bd9c1e15ed0bad3bea18e3c1b7f - +================================================================================ + + +================================================================================ += vendor/github.com/gorilla/websocket licensed under: = + +Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/github.com/gorilla/websocket/LICENSE c007b54a1743d596f46b2748d9f8c044 - +================================================================================ + + +================================================================================ += vendor/github.com/grpc-ecosystem/go-grpc-prometheus licensed under: = + + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -59954,95 +60017,95 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/github.com/hashicorp/golang-lru licensed under: = += vendor/github.com/hashicorp/errwrap licensed under: = Mozilla Public License, version 2.0 1. Definitions -1.1. "Contributor" +1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. -1.2. "Contributor Version" +1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor's Contribution. + Contributor and that particular Contributor’s Contribution. -1.3. "Contribution" +1.3. “Contribution” means Covered Software of a particular Contributor. -1.4. "Covered Software" +1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. -1.5. "Incompatible With Secondary Licenses" +1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or - b. that the Covered Software was made available under the terms of - version 1.1 or earlier of the License, but not also under the terms of - a Secondary License. + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. -1.6. "Executable Form" +1.6. “Executable Form” means any form of the work other than Source Code Form. -1.7. "Larger Work" +1.7. “Larger Work” - means a work that combines Covered Software with other material, in a - separate file or files, that is not Covered Software. + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. -1.8. "License" +1.8. “License” means this document. -1.9. "Licensable" +1.9. “Licensable” - means having the right to grant, to the maximum extent possible, whether - at the time of the initial grant or subsequently, any and all of the - rights conveyed by this License. + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. -1.10. "Modifications" +1.10. “Modifications” means any of the following: - a. any file in Source Code Form that results from an addition to, - deletion from, or modification of the contents of Covered Software; or + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. -1.11. "Patent Claims" of a Contributor +1.11. “Patent Claims” of a Contributor - means any patent claim(s), including without limitation, method, - process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the License, - by the making, using, selling, offering for sale, having made, import, - or transfer of either its Contributions or its Contributor Version. + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. -1.12. "Secondary License" +1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. -1.13. "Source Code Form" +1.13. “Source Code Form” means the form of the work preferred for making modifications. -1.14. "You" (or "Your") +1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this - License. For legal entities, "You" includes any entity that controls, is + License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this - definition, "control" means (a) the power, direct or indirect, to cause + definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. @@ -60058,59 +60121,57 @@ Mozilla Public License, version 2.0 a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its - Contributions or its Contributor Version. + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. 2.2. Effective Date - The licenses granted in Section 2.1 with respect to any Contribution - become effective for each Contribution on the date the Contributor first - distributes such Contribution. + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. 2.3. Limitations on Grant Scope - The licenses granted in this Section 2 are the only rights granted under - this License. No additional rights or licenses will be implied from the - distribution or licensing of Covered Software under this License. - Notwithstanding Section 2.1(b) above, no patent license is granted by a - Contributor: + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or - b. for infringements caused by: (i) Your and any other third party's + b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or - c. under Patent Claims infringed by Covered Software in the absence of - its Contributions. + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. - This License does not grant any rights in the trademarks, service marks, - or logos of any Contributor (except as may be necessary to comply with - the notice requirements in Section 3.4). + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this - License (see Section 10.2) or under the terms of a Secondary License (if - permitted under the terms of Section 3.3). + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). 2.5. Representation - Each Contributor represents that the Contributor believes its - Contributions are its original creation(s) or it has sufficient rights to - grant the rights to its Contributions conveyed by this License. + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. 2.6. Fair Use - This License is not intended to limit any rights You have under - applicable copyright doctrines of fair use, fair dealing, or other - equivalents. + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions @@ -60123,12 +60184,11 @@ Mozilla Public License, version 2.0 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under - the terms of this License. You must inform recipients that the Source - Code Form of the Covered Software is governed by the terms of this - License, and how they can obtain a copy of this License. You may not - attempt to alter or restrict the recipients' rights in the Source Code - Form. + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form @@ -60140,40 +60200,39 @@ Mozilla Public License, version 2.0 reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and - b. You may distribute such Executable Form under the terms of this - License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter the - recipients' rights in the Source Code Form under this License. + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for - the Covered Software. If the Larger Work is a combination of Covered - Software with a work governed by one or more Secondary Licenses, and the - Covered Software is not Incompatible With Secondary Licenses, this - License permits You to additionally distribute such Covered Software - under the terms of such Secondary License(s), so that the recipient of - the Larger Work may, at their option, further distribute the Covered - Software under the terms of either this License or such Secondary - License(s). + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). 3.4. Notices - You may not remove or alter the substance of any license notices - (including copyright notices, patent notices, disclaimers of warranty, or - limitations of liability) contained within the Source Code Form of the - Covered Software, except that You may alter any license notices to the - extent required to remedy known factual inaccuracies. + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on - behalf of any Contributor. You must make it absolutely clear that any - such warranty, support, indemnity, or liability obligation is offered by - You alone, and You hereby agree to indemnify every Contributor for any + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any @@ -60182,14 +60241,14 @@ Mozilla Public License, version 2.0 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, - judicial order, or regulation then You must: (a) comply with the terms of - this License to the maximum extent possible; and (b) describe the - limitations and the code they affect. Such description must be placed in a - text file included with all distributions of the Covered Software under - this License. Except to the extent prohibited by statute or regulation, - such description must be sufficiently detailed for a recipient of ordinary - skill to be able to understand it. + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. 5. Termination @@ -60197,22 +60256,21 @@ Mozilla Public License, version 2.0 fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing - basis, if such Contributor fails to notify You of the non-compliance by - some reasonable means prior to 60 days after You have come back into - compliance. Moreover, Your grants from a particular Contributor are - reinstated on an ongoing basis if such Contributor notifies You of the - non-compliance by some reasonable means, this is the first time You have - received notice of non-compliance with this License from such - Contributor, and You become compliant prior to 30 days after Your receipt - of the notice. + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, - counter-claims, and cross-claims) alleging that a Contributor Version - directly or indirectly infringes any patent, then the rights granted to - You by any and all Contributors for the Covered Software under Section - 2.1 of this License shall terminate. + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been @@ -60221,20 +60279,20 @@ Mozilla Public License, version 2.0 6. Disclaimer of Warranty - Covered Software is provided under this License on an "as is" basis, - without warranty of any kind, either expressed, implied, or statutory, - including, without limitation, warranties that the Covered Software is free - of defects, merchantable, fit for a particular purpose or non-infringing. - The entire risk as to the quality and performance of the Covered Software - is with You. Should any Covered Software prove defective in any respect, - You (not any Contributor) assume the cost of any necessary servicing, - repair, or correction. This disclaimer of warranty constitutes an essential - part of this License. No use of any Covered Software is authorized under - this License except under this disclaimer. - -7. Limitation of Liability - - Under no circumstances and under no legal theory, whether tort (including + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any @@ -60242,29 +60300,27 @@ Mozilla Public License, version 2.0 goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from - such party's negligence to the extent applicable law prohibits such - limitation. Some jurisdictions do not allow the exclusion or limitation of - incidental or consequential damages, so this exclusion and limitation may - not apply to You. + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation - Any litigation relating to this License may be brought only in the courts - of a jurisdiction where the defendant maintains its principal place of - business and such litigation shall be governed by laws of that - jurisdiction, without reference to its conflict-of-law provisions. Nothing - in this Section shall prevent a party's ability to bring cross-claims or - counter-claims. + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous - This License represents the complete agreement concerning the subject - matter hereof. If any provision of this License is held to be - unenforceable, such provision shall be reformed only to the extent - necessary to make it enforceable. Any law or regulation which provides that - the language of a contract shall be construed against the drafter shall not - be used to construe this License against a Contributor. + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. 10. Versions of the License @@ -60278,24 +60334,23 @@ Mozilla Public License, version 2.0 10.2. Effect of New Versions - You may distribute the Covered Software under the terms of the version - of the License under which You originally received the Covered Software, - or under the terms of any subsequent version published by the license + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a - modified version of this License if you rename the license and remove - any references to the name of the license steward (except to note that - such modified license differs from this License). + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). -10.4. Distributing Source Code Form that is Incompatible With Secondary - Licenses If You choose to distribute Source Code Form that is - Incompatible With Secondary Licenses under the terms of this version of - the License, the notice described in Exhibit B of this License must be - attached. +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice @@ -60306,25 +60361,25 @@ Exhibit A - Source Code Form License Notice obtain one at http://mozilla.org/MPL/2.0/. -If it is not possible or desirable to put the notice in a particular file, -then You may include the notice in a location (such as a LICENSE file in a -relevant directory) where a recipient would be likely to look for such a -notice. +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. -Exhibit B - "Incompatible With Secondary Licenses" Notice +Exhibit B - “Incompatible With Secondary Licenses” Notice - This Source Code Form is "Incompatible - With Secondary Licenses", as defined by + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. -= vendor/github.com/hashicorp/golang-lru/LICENSE f27a50d2e878867827842f2c60e30bfc - + += vendor/github.com/hashicorp/errwrap/LICENSE b278a92d2c1509760384428817710378 - ================================================================================ ================================================================================ -= vendor/github.com/hashicorp/golang-lru/simplelru licensed under: = += vendor/github.com/hashicorp/go-cleanhttp licensed under: = Mozilla Public License, version 2.0 @@ -60689,12 +60744,13 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. -= vendor/github.com/hashicorp/golang-lru/LICENSE f27a50d2e878867827842f2c60e30bfc - + += vendor/github.com/hashicorp/go-cleanhttp/LICENSE 65d26fcc2f35ea6a181ac777e42db1ea - ================================================================================ ================================================================================ -= vendor/github.com/hashicorp/hcl licensed under: = += vendor/github.com/hashicorp/go-multierror licensed under: = Mozilla Public License, version 2.0 @@ -61050,101 +61106,100 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. - -= vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - += vendor/github.com/hashicorp/go-multierror/LICENSE d44fdeb607e2d2614db9464dbedd4094 - ================================================================================ ================================================================================ -= vendor/github.com/hashicorp/hcl/hcl/ast licensed under: = += vendor/github.com/hashicorp/go-rootcerts licensed under: = Mozilla Public License, version 2.0 1. Definitions -1.1. “Contributor” +1.1. "Contributor" means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. -1.2. “Contributor Version” +1.2. "Contributor Version" means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. + Contributor and that particular Contributor's Contribution. -1.3. “Contribution” +1.3. "Contribution" means Covered Software of a particular Contributor. -1.4. “Covered Software” +1.4. "Covered Software" means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. -1.5. “Incompatible With Secondary Licenses” +1.5. "Incompatible With Secondary Licenses" means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. -1.6. “Executable Form” +1.6. "Executable Form" means any form of the work other than Source Code Form. -1.7. “Larger Work” +1.7. "Larger Work" - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. -1.8. “License” +1.8. "License" means this document. -1.9. “Licensable” +1.9. "Licensable" - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. -1.10. “Modifications” +1.10. "Modifications" means any of the following: - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. -1.11. “Patent Claims” of a Contributor +1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. -1.12. “Secondary License” +1.12. "Secondary License" means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. -1.13. “Source Code Form” +1.13. "Source Code Form" means the form of the work preferred for making modifications. -1.14. “You” (or “Your”) +1.14. "You" (or "Your") means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is + License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause + definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. @@ -61160,57 +61215,59 @@ Mozilla Public License, version 2.0 a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. 2.2. Effective Date - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. 2.3. Limitations on Grant Scope - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: a. for any code that a Contributor has removed from Covered Software; or - b. for infringements caused by: (i) Your and any other third party’s + b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). 2.5. Representation - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. 2.6. Fair Use - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. 2.7. Conditions @@ -61223,11 +61280,12 @@ Mozilla Public License, version 2.0 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. 3.2. Distribution of Executable Form @@ -61239,39 +61297,40 @@ Mozilla Public License, version 2.0 reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). 3.4. Notices - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any @@ -61280,14 +61339,14 @@ Mozilla Public License, version 2.0 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. 5. Termination @@ -61295,21 +61354,22 @@ Mozilla Public License, version 2.0 fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. 5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been @@ -61318,16 +61378,16 @@ Mozilla Public License, version 2.0 6. Disclaimer of Warranty - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. 7. Limitation of Liability @@ -61339,27 +61399,29 @@ Mozilla Public License, version 2.0 goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. 8. Litigation - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. 9. Miscellaneous - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. 10. Versions of the License @@ -61373,23 +61435,24 @@ Mozilla Public License, version 2.0 10.2. Effect of New Versions - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. Exhibit A - Source Code Form License Notice @@ -61400,113 +61463,114 @@ Exhibit A - Source Code Form License Notice obtain one at http://mozilla.org/MPL/2.0/. -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. You may add additional accurate notices of copyright ownership. -Exhibit B - “Incompatible With Secondary Licenses” Notice +Exhibit B - "Incompatible With Secondary Licenses" Notice - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. -= vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - += vendor/github.com/hashicorp/go-rootcerts/LICENSE 65d26fcc2f35ea6a181ac777e42db1ea - ================================================================================ ================================================================================ -= vendor/github.com/hashicorp/hcl/hcl/parser licensed under: = += vendor/github.com/hashicorp/golang-lru licensed under: = Mozilla Public License, version 2.0 1. Definitions -1.1. “Contributor” +1.1. "Contributor" means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. -1.2. “Contributor Version” +1.2. "Contributor Version" means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. + Contributor and that particular Contributor's Contribution. -1.3. “Contribution” +1.3. "Contribution" means Covered Software of a particular Contributor. -1.4. “Covered Software” +1.4. "Covered Software" means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. -1.5. “Incompatible With Secondary Licenses” +1.5. "Incompatible With Secondary Licenses" means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. -1.6. “Executable Form” +1.6. "Executable Form" means any form of the work other than Source Code Form. -1.7. “Larger Work” +1.7. "Larger Work" - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. -1.8. “License” +1.8. "License" means this document. -1.9. “Licensable” +1.9. "Licensable" - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. -1.10. “Modifications” +1.10. "Modifications" means any of the following: - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. -1.11. “Patent Claims” of a Contributor +1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. -1.12. “Secondary License” +1.12. "Secondary License" means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. -1.13. “Source Code Form” +1.13. "Source Code Form" means the form of the work preferred for making modifications. -1.14. “You” (or “Your”) +1.14. "You" (or "Your") means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is + License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause + definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. @@ -61522,57 +61586,59 @@ Mozilla Public License, version 2.0 a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. 2.2. Effective Date - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. 2.3. Limitations on Grant Scope - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: a. for any code that a Contributor has removed from Covered Software; or - b. for infringements caused by: (i) Your and any other third party’s + b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). 2.5. Representation - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. 2.6. Fair Use - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. 2.7. Conditions @@ -61585,11 +61651,12 @@ Mozilla Public License, version 2.0 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. 3.2. Distribution of Executable Form @@ -61601,39 +61668,40 @@ Mozilla Public License, version 2.0 reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). 3.4. Notices - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any @@ -61642,14 +61710,14 @@ Mozilla Public License, version 2.0 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. 5. Termination @@ -61657,21 +61725,22 @@ Mozilla Public License, version 2.0 fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. 5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been @@ -61680,16 +61749,16 @@ Mozilla Public License, version 2.0 6. Disclaimer of Warranty - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. 7. Limitation of Liability @@ -61701,27 +61770,29 @@ Mozilla Public License, version 2.0 goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. 8. Litigation - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. 9. Miscellaneous - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. 10. Versions of the License @@ -61735,23 +61806,24 @@ Mozilla Public License, version 2.0 10.2. Effect of New Versions - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. Exhibit A - Source Code Form License Notice @@ -61762,113 +61834,113 @@ Exhibit A - Source Code Form License Notice obtain one at http://mozilla.org/MPL/2.0/. -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. You may add additional accurate notices of copyright ownership. -Exhibit B - “Incompatible With Secondary Licenses” Notice +Exhibit B - "Incompatible With Secondary Licenses" Notice - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. - -= vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - += vendor/github.com/hashicorp/golang-lru/LICENSE f27a50d2e878867827842f2c60e30bfc - ================================================================================ ================================================================================ -= vendor/github.com/hashicorp/hcl/hcl/scanner licensed under: = += vendor/github.com/hashicorp/golang-lru/simplelru licensed under: = Mozilla Public License, version 2.0 1. Definitions -1.1. “Contributor” +1.1. "Contributor" means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. -1.2. “Contributor Version” +1.2. "Contributor Version" means the combination of the Contributions of others (if any) used by a - Contributor and that particular Contributor’s Contribution. + Contributor and that particular Contributor's Contribution. -1.3. “Contribution” +1.3. "Contribution" means Covered Software of a particular Contributor. -1.4. “Covered Software” +1.4. "Covered Software" means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. -1.5. “Incompatible With Secondary Licenses” +1.5. "Incompatible With Secondary Licenses" means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or - b. that the Covered Software was made available under the terms of version - 1.1 or earlier of the License, but not also under the terms of a - Secondary License. + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. -1.6. “Executable Form” +1.6. "Executable Form" means any form of the work other than Source Code Form. -1.7. “Larger Work” +1.7. "Larger Work" - means a work that combines Covered Software with other material, in a separate - file or files, that is not Covered Software. + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. -1.8. “License” +1.8. "License" means this document. -1.9. “Licensable” +1.9. "Licensable" - means having the right to grant, to the maximum extent possible, whether at the - time of the initial grant or subsequently, any and all of the rights conveyed by - this License. + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. -1.10. “Modifications” +1.10. "Modifications" means any of the following: - a. any file in Source Code Form that results from an addition to, deletion - from, or modification of the contents of Covered Software; or + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. -1.11. “Patent Claims” of a Contributor +1.11. "Patent Claims" of a Contributor - means any patent claim(s), including without limitation, method, process, - and apparatus claims, in any patent Licensable by such Contributor that - would be infringed, but for the grant of the License, by the making, - using, selling, offering for sale, having made, import, or transfer of - either its Contributions or its Contributor Version. + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. -1.12. “Secondary License” +1.12. "Secondary License" means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. -1.13. “Source Code Form” +1.13. "Source Code Form" means the form of the work preferred for making modifications. -1.14. “You” (or “Your”) +1.14. "You" (or "Your") means an individual or a legal entity exercising rights under this - License. For legal entities, “You” includes any entity that controls, is + License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with You. For purposes of this - definition, “control” means (a) the power, direct or indirect, to cause + definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. @@ -61884,57 +61956,59 @@ Mozilla Public License, version 2.0 a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its - Contributions, either on an unmodified basis, with Modifications, or as - part of a Larger Work; and + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for - sale, have made, import, and otherwise transfer either its Contributions - or its Contributor Version. + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. 2.2. Effective Date - The licenses granted in Section 2.1 with respect to any Contribution become - effective for each Contribution on the date the Contributor first distributes - such Contribution. + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. 2.3. Limitations on Grant Scope - The licenses granted in this Section 2 are the only rights granted under this - License. No additional rights or licenses will be implied from the distribution - or licensing of Covered Software under this License. Notwithstanding Section - 2.1(b) above, no patent license is granted by a Contributor: + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: a. for any code that a Contributor has removed from Covered Software; or - b. for infringements caused by: (i) Your and any other third party’s + b. for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or - c. under Patent Claims infringed by Covered Software in the absence of its - Contributions. + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. - This License does not grant any rights in the trademarks, service marks, or - logos of any Contributor (except as may be necessary to comply with the - notice requirements in Section 3.4). + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to - distribute the Covered Software under a subsequent version of this License - (see Section 10.2) or under the terms of a Secondary License (if permitted - under the terms of Section 3.3). + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). 2.5. Representation - Each Contributor represents that the Contributor believes its Contributions - are its original creation(s) or it has sufficient rights to grant the - rights to its Contributions conveyed by this License. + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. 2.6. Fair Use - This License is not intended to limit any rights You have under applicable - copyright doctrines of fair use, fair dealing, or other equivalents. + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. 2.7. Conditions @@ -61947,11 +62021,12 @@ Mozilla Public License, version 2.0 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any - Modifications that You create or to which You contribute, must be under the - terms of this License. You must inform recipients that the Source Code Form - of the Covered Software is governed by the terms of this License, and how - they can obtain a copy of this License. You may not attempt to alter or - restrict the recipients’ rights in the Source Code Form. + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. 3.2. Distribution of Executable Form @@ -61963,39 +62038,40 @@ Mozilla Public License, version 2.0 reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and - b. You may distribute such Executable Form under the terms of this License, - or sublicense it under different terms, provided that the license for - the Executable Form does not attempt to limit or alter the recipients’ - rights in the Source Code Form under this License. + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, - provided that You also comply with the requirements of this License for the - Covered Software. If the Larger Work is a combination of Covered Software - with a work governed by one or more Secondary Licenses, and the Covered - Software is not Incompatible With Secondary Licenses, this License permits - You to additionally distribute such Covered Software under the terms of - such Secondary License(s), so that the recipient of the Larger Work may, at - their option, further distribute the Covered Software under the terms of - either this License or such Secondary License(s). + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). 3.4. Notices - You may not remove or alter the substance of any license notices (including - copyright notices, patent notices, disclaimers of warranty, or limitations - of liability) contained within the Source Code Form of the Covered - Software, except that You may alter any license notices to the extent - required to remedy known factual inaccuracies. + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered - Software. However, You may do so only on Your own behalf, and not on behalf - of any Contributor. You must make it absolutely clear that any such - warranty, support, indemnity, or liability obligation is offered by You - alone, and You hereby agree to indemnify every Contributor for any + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any @@ -62004,14 +62080,14 @@ Mozilla Public License, version 2.0 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License - with respect to some or all of the Covered Software due to statute, judicial - order, or regulation then You must: (a) comply with the terms of this License - to the maximum extent possible; and (b) describe the limitations and the code - they affect. Such description must be placed in a text file included with all - distributions of the Covered Software under this License. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. 5. Termination @@ -62019,21 +62095,22 @@ Mozilla Public License, version 2.0 fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor - explicitly and finally terminates Your grants, and (b) on an ongoing basis, - if such Contributor fails to notify You of the non-compliance by some - reasonable means prior to 60 days after You have come back into compliance. - Moreover, Your grants from a particular Contributor are reinstated on an - ongoing basis if such Contributor notifies You of the non-compliance by - some reasonable means, this is the first time You have received notice of - non-compliance with this License from such Contributor, and You become - compliant prior to 30 days after Your receipt of the notice. + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. 5.2. If You initiate litigation against any entity by asserting a patent - infringement claim (excluding declaratory judgment actions, counter-claims, - and cross-claims) alleging that a Contributor Version directly or - indirectly infringes any patent, then the rights granted to You by any and - all Contributors for the Covered Software under Section 2.1 of this License - shall terminate. + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been @@ -62042,16 +62119,16 @@ Mozilla Public License, version 2.0 6. Disclaimer of Warranty - Covered Software is provided under this License on an “as is” basis, without - warranty of any kind, either expressed, implied, or statutory, including, - without limitation, warranties that the Covered Software is free of defects, - merchantable, fit for a particular purpose or non-infringing. The entire - risk as to the quality and performance of the Covered Software is with You. - Should any Covered Software prove defective in any respect, You (not any - Contributor) assume the cost of any necessary servicing, repair, or - correction. This disclaimer of warranty constitutes an essential part of this - License. No use of any Covered Software is authorized under this License - except under this disclaimer. + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. 7. Limitation of Liability @@ -62063,27 +62140,29 @@ Mozilla Public License, version 2.0 goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability - shall not apply to liability for death or personal injury resulting from such - party’s negligence to the extent applicable law prohibits such limitation. - Some jurisdictions do not allow the exclusion or limitation of incidental or - consequential damages, so this exclusion and limitation may not apply to You. + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. 8. Litigation - Any litigation relating to this License may be brought only in the courts of - a jurisdiction where the defendant maintains its principal place of business - and such litigation shall be governed by laws of that jurisdiction, without - reference to its conflict-of-law provisions. Nothing in this Section shall - prevent a party’s ability to bring cross-claims or counter-claims. + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. 9. Miscellaneous - This License represents the complete agreement concerning the subject matter - hereof. If any provision of this License is held to be unenforceable, such - provision shall be reformed only to the extent necessary to make it - enforceable. Any law or regulation which provides that the language of a - contract shall be construed against the drafter shall not be used to construe - this License against a Contributor. + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. 10. Versions of the License @@ -62097,23 +62176,24 @@ Mozilla Public License, version 2.0 10.2. Effect of New Versions - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. Exhibit A - Source Code Form License Notice @@ -62124,25 +62204,25 @@ Exhibit A - Source Code Form License Notice obtain one at http://mozilla.org/MPL/2.0/. -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. You may add additional accurate notices of copyright ownership. -Exhibit B - “Incompatible With Secondary Licenses” Notice +Exhibit B - "Incompatible With Secondary Licenses" Notice - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. - -= vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - += vendor/github.com/hashicorp/golang-lru/LICENSE f27a50d2e878867827842f2c60e30bfc - ================================================================================ ================================================================================ -= vendor/github.com/hashicorp/hcl/hcl/strconv licensed under: = += vendor/github.com/hashicorp/hcl licensed under: = Mozilla Public License, version 2.0 @@ -62504,7 +62584,7 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice ================================================================================ -= vendor/github.com/hashicorp/hcl/hcl/token licensed under: = += vendor/github.com/hashicorp/hcl/hcl/ast licensed under: = Mozilla Public License, version 2.0 @@ -62866,7 +62946,7 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice ================================================================================ -= vendor/github.com/hashicorp/hcl/json/parser licensed under: = += vendor/github.com/hashicorp/hcl/hcl/parser licensed under: = Mozilla Public License, version 2.0 @@ -63228,7 +63308,7 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice ================================================================================ -= vendor/github.com/hashicorp/hcl/json/scanner licensed under: = += vendor/github.com/hashicorp/hcl/hcl/scanner licensed under: = Mozilla Public License, version 2.0 @@ -63590,7 +63670,369 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice ================================================================================ -= vendor/github.com/hashicorp/hcl/json/token licensed under: = += vendor/github.com/hashicorp/hcl/hcl/strconv licensed under: = + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + += vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - +================================================================================ + + +================================================================================ += vendor/github.com/hashicorp/hcl/hcl/token licensed under: = Mozilla Public License, version 2.0 @@ -63896,65 +64338,3103 @@ Mozilla Public License, version 2.0 this License against a Contributor. -10. Versions of the License +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + += vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - +================================================================================ + + +================================================================================ += vendor/github.com/hashicorp/hcl/json/parser licensed under: = + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + += vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - +================================================================================ + + +================================================================================ += vendor/github.com/hashicorp/hcl/json/scanner licensed under: = + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + += vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - +================================================================================ + + +================================================================================ += vendor/github.com/hashicorp/hcl/json/token licensed under: = + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + += vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - +================================================================================ + + +================================================================================ += vendor/github.com/hashicorp/vault/api licensed under: = + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + + += vendor/github.com/hashicorp/vault/LICENSE 65d26fcc2f35ea6a181ac777e42db1ea - +================================================================================ + + +================================================================================ += vendor/github.com/hashicorp/vault/helper/compressutil licensed under: = + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + + += vendor/github.com/hashicorp/vault/LICENSE 65d26fcc2f35ea6a181ac777e42db1ea - +================================================================================ + + +================================================================================ += vendor/github.com/hashicorp/vault/helper/jsonutil licensed under: = + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + + += vendor/github.com/hashicorp/vault/LICENSE 65d26fcc2f35ea6a181ac777e42db1ea - +================================================================================ + + +================================================================================ += vendor/github.com/hashicorp/vault/helper/parseutil licensed under: = + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + + += vendor/github.com/hashicorp/vault/LICENSE 65d26fcc2f35ea6a181ac777e42db1ea - +================================================================================ + + +================================================================================ += vendor/github.com/hawkular/hawkular-client-go/metrics licensed under: = + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + += vendor/github.com/hawkular/hawkular-client-go/LICENSE fa818a259cbed7ce8bc2a22d35a464fc - +================================================================================ + + +================================================================================ += vendor/github.com/heketi/heketi/client/api/go-client licensed under: = + +Heketi code is released under various licenses: + +The REST API client code (in go and python) is released +under a dual license of Apache 2.0 or LGPLv3+. + +The other parts of heketi (server, cli, tests, ...) are released +under a dual license of LGPLv3+ or GPLv2. + += vendor/github.com/heketi/heketi/LICENSE a58e72c3bda574189508cb90d56fa19f - +================================================================================ + + +================================================================================ += vendor/github.com/heketi/heketi/pkg/glusterfs/api licensed under: = + +Heketi code is released under various licenses: + +The REST API client code (in go and python) is released +under a dual license of Apache 2.0 or LGPLv3+. + +The other parts of heketi (server, cli, tests, ...) are released +under a dual license of LGPLv3+ or GPLv2. + += vendor/github.com/heketi/heketi/LICENSE a58e72c3bda574189508cb90d56fa19f - +================================================================================ + + +================================================================================ += vendor/github.com/heketi/heketi/pkg/utils licensed under: = + +Heketi code is released under various licenses: + +The REST API client code (in go and python) is released +under a dual license of Apache 2.0 or LGPLv3+. + +The other parts of heketi (server, cli, tests, ...) are released +under a dual license of LGPLv3+ or GPLv2. + += vendor/github.com/heketi/heketi/LICENSE a58e72c3bda574189508cb90d56fa19f - +================================================================================ + + +================================================================================ += vendor/github.com/howeyc/gopass licensed under: = + +ISC License + +Copyright (c) 2012 Chris Howey + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + += vendor/github.com/howeyc/gopass/LICENSE.txt b24abd09a925eaf2b6de5a9b4c9bea07 - +================================================================================ + + +================================================================================ += vendor/github.com/imdario/mergo licensed under: = + +Copyright (c) 2013 Dario Castañé. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/github.com/imdario/mergo/LICENSE ff13e03bb57bf9c52645f2f942afa28b - +================================================================================ + + +================================================================================ += vendor/github.com/inconshreveable/mousetrap licensed under: = + +Copyright 2014 Alan Shreve + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + += vendor/github.com/inconshreveable/mousetrap/LICENSE b23cff9db13f093a4e6ff77105cbd8eb - +================================================================================ + + +================================================================================ += vendor/github.com/influxdata/influxdb/client licensed under: = + +The MIT License (MIT) + +Copyright (c) 2013-2016 Errplane Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + += vendor/github.com/influxdata/influxdb/LICENSE ba8146ad9cc2a128209983265136e06a - +================================================================================ + + +================================================================================ += vendor/github.com/influxdata/influxdb/client/v2 licensed under: = + +The MIT License (MIT) + +Copyright (c) 2013-2016 Errplane Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + += vendor/github.com/influxdata/influxdb/LICENSE ba8146ad9cc2a128209983265136e06a - +================================================================================ + + +================================================================================ += vendor/github.com/influxdata/influxdb/models licensed under: = + +The MIT License (MIT) + +Copyright (c) 2013-2016 Errplane Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + += vendor/github.com/influxdata/influxdb/LICENSE ba8146ad9cc2a128209983265136e06a - +================================================================================ -10.1. New Versions - Mozilla Foundation is the license steward. Except as provided in Section - 10.3, no one other than the license steward has the right to modify or - publish new versions of this License. Each version will be given a - distinguishing version number. +================================================================================ += vendor/github.com/influxdata/influxdb/pkg/escape licensed under: = -10.2. Effect of New Versions +The MIT License (MIT) - You may distribute the Covered Software under the terms of the version of - the License under which You originally received the Covered Software, or - under the terms of any subsequent version published by the license - steward. +Copyright (c) 2013-2016 Errplane Inc. -10.3. Modified Versions +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: - If you create software not governed by this License, and you want to - create a new license for such software, you may create and use a modified - version of this License if you rename the license and remove any - references to the name of the license steward (except to note that such - modified license differs from this License). +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - If You choose to distribute Source Code Form that is Incompatible With - Secondary Licenses under the terms of this version of the License, the - notice described in Exhibit B of this License must be attached. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -Exhibit A - Source Code Form License Notice += vendor/github.com/influxdata/influxdb/LICENSE ba8146ad9cc2a128209983265136e06a - +================================================================================ - This Source Code Form is subject to the - terms of the Mozilla Public License, v. - 2.0. If a copy of the MPL was not - distributed with this file, You can - obtain one at - http://mozilla.org/MPL/2.0/. -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. +================================================================================ += vendor/github.com/jmespath/go-jmespath licensed under: = -You may add additional accurate notices of copyright ownership. +Copyright 2015 James Saryerwinnie -Exhibit B - “Incompatible With Secondary Licenses” Notice +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - This Source Code Form is “Incompatible - With Secondary Licenses”, as defined by - the Mozilla Public License, v. 2.0. + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. -= vendor/github.com/hashicorp/hcl/LICENSE b278a92d2c1509760384428817710378 - += vendor/github.com/jmespath/go-jmespath/LICENSE 9abfa8353fce3f2cb28364e1e9016852 - ================================================================================ ================================================================================ -= vendor/github.com/hawkular/hawkular-client-go/metrics licensed under: = += vendor/github.com/jonboulle/clockwork licensed under: = - Apache License +Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -64156,83 +67636,234 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice See the License for the specific language governing permissions and limitations under the License. += vendor/github.com/jonboulle/clockwork/LICENSE 136e4f49dbf29942c572a3a8f6e88a77 - +================================================================================ + -= vendor/github.com/hawkular/hawkular-client-go/LICENSE fa818a259cbed7ce8bc2a22d35a464fc - ================================================================================ += vendor/github.com/jteeuwen/go-bindata licensed under: = +This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication +license. Its contents can be found at: +http://creativecommons.org/publicdomain/zero/1.0 += vendor/github.com/jteeuwen/go-bindata/LICENSE 8dcedca69f7a474372829521f37954b1 - ================================================================================ -= vendor/github.com/heketi/heketi/client/api/go-client licensed under: = -Heketi code is released under various licenses: -The REST API client code (in go and python) is released -under a dual license of Apache 2.0 or LGPLv3+. +================================================================================ += vendor/github.com/jteeuwen/go-bindata/go-bindata licensed under: = -The other parts of heketi (server, cli, tests, ...) are released -under a dual license of LGPLv3+ or GPLv2. +This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication +license. Its contents can be found at: +http://creativecommons.org/publicdomain/zero/1.0 -= vendor/github.com/heketi/heketi/LICENSE a58e72c3bda574189508cb90d56fa19f - += vendor/github.com/jteeuwen/go-bindata/LICENSE 8dcedca69f7a474372829521f37954b1 - ================================================================================ ================================================================================ -= vendor/github.com/heketi/heketi/pkg/glusterfs/api licensed under: = += vendor/github.com/juju/ratelimit licensed under: = -Heketi code is released under various licenses: +All files in this repository are licensed as follows. If you contribute +to this repository, it is assumed that you license your contribution +under the same license unless you state otherwise. -The REST API client code (in go and python) is released -under a dual license of Apache 2.0 or LGPLv3+. +All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file. -The other parts of heketi (server, cli, tests, ...) are released -under a dual license of LGPLv3+ or GPLv2. +This software is licensed under the LGPLv3, included below. -= vendor/github.com/heketi/heketi/LICENSE a58e72c3bda574189508cb90d56fa19f - -================================================================================ +As a special exception to the GNU Lesser General Public License version 3 +("LGPL3"), the copyright holders of this Library give you permission to +convey to a third party a Combined Work that links statically or dynamically +to this Library without providing any Minimal Corresponding Source or +Minimal Application Code as set out in 4d or providing the installation +information set out in section 4e, provided that you comply with the other +provisions of LGPL3 and provided that you meet, for the Application the +terms and conditions of the license(s) which apply to the Application. + +Except as stated in this special exception, the provisions of LGPL3 will +continue to comply in full to this Library. If you modify this Library, you +may apply this exception to your version of this Library, but you are not +obliged to do so. If you do not wish to do so, delete this exception +statement from your version. This exception does not (and cannot) modify any +license terms which apply to the Application, with which you must still +comply. -================================================================================ -= vendor/github.com/heketi/heketi/pkg/utils licensed under: = + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 -Heketi code is released under various licenses: + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. -The REST API client code (in go and python) is released -under a dual license of Apache 2.0 or LGPLv3+. -The other parts of heketi (server, cli, tests, ...) are released -under a dual license of LGPLv3+ or GPLv2. + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. -= vendor/github.com/heketi/heketi/LICENSE a58e72c3bda574189508cb90d56fa19f - -================================================================================ + 0. Additional Definitions. + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. -================================================================================ -= vendor/github.com/howeyc/gopass licensed under: = + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. -ISC License + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. -Copyright (c) 2012 Chris Howey + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. -= vendor/github.com/howeyc/gopass/LICENSE.txt b24abd09a925eaf2b6de5a9b4c9bea07 - + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. + += vendor/github.com/juju/ratelimit/LICENSE 2d1c30374313ae40df7772dc92ef9fd5 - ================================================================================ ================================================================================ -= vendor/github.com/imdario/mergo licensed under: = += vendor/github.com/kardianos/osext licensed under: = -Copyright (c) 2013 Dario Castañé. All rights reserved. Copyright (c) 2012 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -64261,168 +67892,108 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/github.com/imdario/mergo/LICENSE ff13e03bb57bf9c52645f2f942afa28b - -================================================================================ - - -================================================================================ -= vendor/github.com/inconshreveable/mousetrap licensed under: = - -Copyright 2014 Alan Shreve - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -= vendor/github.com/inconshreveable/mousetrap/LICENSE b23cff9db13f093a4e6ff77105cbd8eb - -================================================================================ - - -================================================================================ -= vendor/github.com/influxdata/influxdb/client licensed under: = - -The MIT License (MIT) - -Copyright (c) 2013-2016 Errplane Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -= vendor/github.com/influxdata/influxdb/LICENSE ba8146ad9cc2a128209983265136e06a - += vendor/github.com/kardianos/osext/LICENSE 591778525c869cdde0ab5a1bf283cd81 - ================================================================================ ================================================================================ -= vendor/github.com/influxdata/influxdb/client/v2 licensed under: = - -The MIT License (MIT) += vendor/github.com/karlseguin/ccache licensed under: = -Copyright (c) 2013-2016 Errplane Inc. +Copyright (c) 2013 Karl Seguin. -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -= vendor/github.com/influxdata/influxdb/LICENSE ba8146ad9cc2a128209983265136e06a - += vendor/github.com/karlseguin/ccache/license.txt fb40cd712dfcf5e0a8de4c13c3399db2 - ================================================================================ ================================================================================ -= vendor/github.com/influxdata/influxdb/models licensed under: = - -The MIT License (MIT) += vendor/github.com/kr/fs licensed under: = -Copyright (c) 2013-2016 Errplane Inc. +Copyright (c) 2012 The Go Authors. All rights reserved. -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/github.com/influxdata/influxdb/LICENSE ba8146ad9cc2a128209983265136e06a - += vendor/github.com/kr/fs/LICENSE 591778525c869cdde0ab5a1bf283cd81 - ================================================================================ ================================================================================ -= vendor/github.com/influxdata/influxdb/pkg/escape licensed under: = - -The MIT License (MIT) += vendor/github.com/kr/pty licensed under: = -Copyright (c) 2013-2016 Errplane Inc. +Copyright (c) 2011 Keith Rarick -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -= vendor/github.com/influxdata/influxdb/LICENSE ba8146ad9cc2a128209983265136e06a - -================================================================================ +The above copyright notice and this permission notice shall +be included in all copies or substantial portions of the +Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. += vendor/github.com/kr/pty/License 93958070863d769117fa33b129020050 - ================================================================================ -= vendor/github.com/jmespath/go-jmespath licensed under: = - -Copyright 2015 James Saryerwinnie - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -= vendor/github.com/jmespath/go-jmespath/LICENSE 9abfa8353fce3f2cb28364e1e9016852 - ================================================================================ += vendor/github.com/libopenstorage/openstorage/api licensed under: = -================================================================================ -= vendor/github.com/jonboulle/clockwork licensed under: = - -Apache License + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -64599,18 +68170,7 @@ Apache License END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} + Copyright 2015 Openstorage.org. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -64624,361 +68184,12 @@ Apache License See the License for the specific language governing permissions and limitations under the License. -= vendor/github.com/jonboulle/clockwork/LICENSE 136e4f49dbf29942c572a3a8f6e88a77 - -================================================================================ - - -================================================================================ -= vendor/github.com/jteeuwen/go-bindata licensed under: = - -This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication -license. Its contents can be found at: -http://creativecommons.org/publicdomain/zero/1.0 - -= vendor/github.com/jteeuwen/go-bindata/LICENSE 8dcedca69f7a474372829521f37954b1 - -================================================================================ - - -================================================================================ -= vendor/github.com/jteeuwen/go-bindata/go-bindata licensed under: = - -This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication -license. Its contents can be found at: -http://creativecommons.org/publicdomain/zero/1.0 - -= vendor/github.com/jteeuwen/go-bindata/LICENSE 8dcedca69f7a474372829521f37954b1 - -================================================================================ - - -================================================================================ -= vendor/github.com/juju/ratelimit licensed under: = - -All files in this repository are licensed as follows. If you contribute -to this repository, it is assumed that you license your contribution -under the same license unless you state otherwise. - -All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file. - -This software is licensed under the LGPLv3, included below. - -As a special exception to the GNU Lesser General Public License version 3 -("LGPL3"), the copyright holders of this Library give you permission to -convey to a third party a Combined Work that links statically or dynamically -to this Library without providing any Minimal Corresponding Source or -Minimal Application Code as set out in 4d or providing the installation -information set out in section 4e, provided that you comply with the other -provisions of LGPL3 and provided that you meet, for the Application the -terms and conditions of the license(s) which apply to the Application. - -Except as stated in this special exception, the provisions of LGPL3 will -continue to comply in full to this Library. If you modify this Library, you -may apply this exception to your version of this Library, but you are not -obliged to do so. If you do not wish to do so, delete this exception -statement from your version. This exception does not (and cannot) modify any -license terms which apply to the Application, with which you must still -comply. - - - GNU LESSER GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - - This version of the GNU Lesser General Public License incorporates -the terms and conditions of version 3 of the GNU General Public -License, supplemented by the additional permissions listed below. - - 0. Additional Definitions. - - As used herein, "this License" refers to version 3 of the GNU Lesser -General Public License, and the "GNU GPL" refers to version 3 of the GNU -General Public License. - - "The Library" refers to a covered work governed by this License, -other than an Application or a Combined Work as defined below. - - An "Application" is any work that makes use of an interface provided -by the Library, but which is not otherwise based on the Library. -Defining a subclass of a class defined by the Library is deemed a mode -of using an interface provided by the Library. - - A "Combined Work" is a work produced by combining or linking an -Application with the Library. The particular version of the Library -with which the Combined Work was made is also called the "Linked -Version". - - The "Minimal Corresponding Source" for a Combined Work means the -Corresponding Source for the Combined Work, excluding any source code -for portions of the Combined Work that, considered in isolation, are -based on the Application, and not on the Linked Version. - - The "Corresponding Application Code" for a Combined Work means the -object code and/or source code for the Application, including any data -and utility programs needed for reproducing the Combined Work from the -Application, but excluding the System Libraries of the Combined Work. - - 1. Exception to Section 3 of the GNU GPL. - - You may convey a covered work under sections 3 and 4 of this License -without being bound by section 3 of the GNU GPL. - - 2. Conveying Modified Versions. - - If you modify a copy of the Library, and, in your modifications, a -facility refers to a function or data to be supplied by an Application -that uses the facility (other than as an argument passed when the -facility is invoked), then you may convey a copy of the modified -version: - - a) under this License, provided that you make a good faith effort to - ensure that, in the event an Application does not supply the - function or data, the facility still operates, and performs - whatever part of its purpose remains meaningful, or - - b) under the GNU GPL, with none of the additional permissions of - this License applicable to that copy. - - 3. Object Code Incorporating Material from Library Header Files. - - The object code form of an Application may incorporate material from -a header file that is part of the Library. You may convey such object -code under terms of your choice, provided that, if the incorporated -material is not limited to numerical parameters, data structure -layouts and accessors, or small macros, inline functions and templates -(ten or fewer lines in length), you do both of the following: - - a) Give prominent notice with each copy of the object code that the - Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the object code with a copy of the GNU GPL and this license - document. - - 4. Combined Works. - - You may convey a Combined Work under terms of your choice that, -taken together, effectively do not restrict modification of the -portions of the Library contained in the Combined Work and reverse -engineering for debugging such modifications, if you also do each of -the following: - - a) Give prominent notice with each copy of the Combined Work that - the Library is used in it and that the Library and its use are - covered by this License. - - b) Accompany the Combined Work with a copy of the GNU GPL and this license - document. - - c) For a Combined Work that displays copyright notices during - execution, include the copyright notice for the Library among - these notices, as well as a reference directing the user to the - copies of the GNU GPL and this license document. - - d) Do one of the following: - - 0) Convey the Minimal Corresponding Source under the terms of this - License, and the Corresponding Application Code in a form - suitable for, and under terms that permit, the user to - recombine or relink the Application with a modified version of - the Linked Version to produce a modified Combined Work, in the - manner specified by section 6 of the GNU GPL for conveying - Corresponding Source. - - 1) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (a) uses at run time - a copy of the Library already present on the user's computer - system, and (b) will operate properly with a modified version - of the Library that is interface-compatible with the Linked - Version. - - e) Provide Installation Information, but only if you would otherwise - be required to provide such information under section 6 of the - GNU GPL, and only to the extent that such information is - necessary to install and execute a modified version of the - Combined Work produced by recombining or relinking the - Application with a modified version of the Linked Version. (If - you use option 4d0, the Installation Information must accompany - the Minimal Corresponding Source and Corresponding Application - Code. If you use option 4d1, you must provide the Installation - Information in the manner specified by section 6 of the GNU GPL - for conveying Corresponding Source.) - - 5. Combined Libraries. - - You may place library facilities that are a work based on the -Library side by side in a single library together with other library -facilities that are not Applications and are not covered by this -License, and convey such a combined library under terms of your -choice, if you do both of the following: - - a) Accompany the combined library with a copy of the same work based - on the Library, uncombined with any other library facilities, - conveyed under the terms of this License. - - b) Give prominent notice with the combined library that part of it - is a work based on the Library, and explaining where to find the - accompanying uncombined form of the same work. - - 6. Revised Versions of the GNU Lesser General Public License. - - The Free Software Foundation may publish revised and/or new versions -of the GNU Lesser General Public License from time to time. Such new -versions will be similar in spirit to the present version, but may -differ in detail to address new problems or concerns. - - Each version is given a distinguishing version number. If the -Library as you received it specifies that a certain numbered version -of the GNU Lesser General Public License "or any later version" -applies to it, you have the option of following the terms and -conditions either of that published version or of any later version -published by the Free Software Foundation. If the Library as you -received it does not specify a version number of the GNU Lesser -General Public License, you may choose any version of the GNU Lesser -General Public License ever published by the Free Software Foundation. - - If the Library as you received it specifies that a proxy can decide -whether future versions of the GNU Lesser General Public License shall -apply, that proxy's public statement of acceptance of any version is -permanent authorization for you to choose that version for the -Library. - -= vendor/github.com/juju/ratelimit/LICENSE 2d1c30374313ae40df7772dc92ef9fd5 - -================================================================================ - - -================================================================================ -= vendor/github.com/kardianos/osext licensed under: = - -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -= vendor/github.com/kardianos/osext/LICENSE 591778525c869cdde0ab5a1bf283cd81 - -================================================================================ - - -================================================================================ -= vendor/github.com/karlseguin/ccache licensed under: = - -Copyright (c) 2013 Karl Seguin. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -= vendor/github.com/karlseguin/ccache/license.txt fb40cd712dfcf5e0a8de4c13c3399db2 - -================================================================================ - - -================================================================================ -= vendor/github.com/kr/fs licensed under: = - -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -= vendor/github.com/kr/fs/LICENSE 591778525c869cdde0ab5a1bf283cd81 - -================================================================================ - - -================================================================================ -= vendor/github.com/kr/pty licensed under: = - -Copyright (c) 2011 Keith Rarick - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, -sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall -be included in all copies or substantial portions of the -Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -= vendor/github.com/kr/pty/License 93958070863d769117fa33b129020050 - += vendor/github.com/libopenstorage/openstorage/LICENSE 40c3e1c9eacda859a17048003909a2f8 - ================================================================================ ================================================================================ -= vendor/github.com/libopenstorage/openstorage/api licensed under: = += vendor/github.com/libopenstorage/openstorage/api/client licensed under: = Apache License @@ -65177,7 +68388,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================================ -= vendor/github.com/libopenstorage/openstorage/api/client licensed under: = += vendor/github.com/libopenstorage/openstorage/api/client/volume licensed under: = Apache License @@ -65376,7 +68587,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================================ -= vendor/github.com/libopenstorage/openstorage/api/client/volume licensed under: = += vendor/github.com/libopenstorage/openstorage/api/spec licensed under: = Apache License @@ -65575,7 +68786,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================================ -= vendor/github.com/libopenstorage/openstorage/api/spec licensed under: = += vendor/github.com/libopenstorage/openstorage/pkg/units licensed under: = Apache License @@ -65774,7 +68985,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================================ -= vendor/github.com/libopenstorage/openstorage/pkg/units licensed under: = += vendor/github.com/libopenstorage/openstorage/volume licensed under: = Apache License @@ -65973,10 +69184,9 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================================ -= vendor/github.com/libopenstorage/openstorage/volume licensed under: = - += vendor/github.com/lpabon/godbc licensed under: = - Apache License +Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -66153,7 +69363,18 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. END OF TERMS AND CONDITIONS - Copyright 2015 Openstorage.org. + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -66166,15 +69387,121 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. += vendor/github.com/lpabon/godbc/LICENSE 6c4db32a2fa8717faffa1d4f10136f47 - +================================================================================ + -= vendor/github.com/libopenstorage/openstorage/LICENSE 40c3e1c9eacda859a17048003909a2f8 - ================================================================================ += vendor/github.com/magiconair/properties licensed under: = +goproperties - properties file decoder for Go +Copyright (c) 2013-2014 - Frank Schroeder + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/github.com/magiconair/properties/LICENSE c383a608fb9a0d227953e928803b9631 - ================================================================================ -= vendor/github.com/lpabon/godbc licensed under: = -Apache License + +================================================================================ += vendor/github.com/mailru/easyjson/buffer licensed under: = + +Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + += vendor/github.com/mailru/easyjson/LICENSE 819e81c2ec13e1bbc47dc5e90bb4d88b - +================================================================================ + + +================================================================================ += vendor/github.com/mailru/easyjson/jlexer licensed under: = + +Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + += vendor/github.com/mailru/easyjson/LICENSE 819e81c2ec13e1bbc47dc5e90bb4d88b - +================================================================================ + + +================================================================================ += vendor/github.com/mailru/easyjson/jwriter licensed under: = + +Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + += vendor/github.com/mailru/easyjson/LICENSE 819e81c2ec13e1bbc47dc5e90bb4d88b - +================================================================================ + + +================================================================================ += vendor/github.com/MakeNowJust/heredoc licensed under: = + +The MIT License (MIT) + +Copyright (c) 2014 TSUYUSATO Kitsune + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + += vendor/github.com/MakeNowJust/heredoc/LICENSE 15e1c8f1d3c204c05f71630afacbc92b - +================================================================================ + + +================================================================================ += vendor/github.com/matttproud/golang_protobuf_extensions/pbutil licensed under: = + + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -66354,7 +69681,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -66362,7 +69689,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2013 Matt T. Proud Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -66375,119 +69702,43 @@ Apache License WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -= vendor/github.com/lpabon/godbc/LICENSE 6c4db32a2fa8717faffa1d4f10136f47 - -================================================================================ - - -================================================================================ -= vendor/github.com/magiconair/properties licensed under: = - -goproperties - properties file decoder for Go - -Copyright (c) 2013-2014 - Frank Schroeder - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -= vendor/github.com/magiconair/properties/LICENSE c383a608fb9a0d227953e928803b9631 - -================================================================================ - - -================================================================================ -= vendor/github.com/mailru/easyjson/buffer licensed under: = - -Copyright (c) 2016 Mail.Ru Group - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -= vendor/github.com/mailru/easyjson/LICENSE 819e81c2ec13e1bbc47dc5e90bb4d88b - -================================================================================ - - -================================================================================ -= vendor/github.com/mailru/easyjson/jlexer licensed under: = - -Copyright (c) 2016 Mail.Ru Group - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -= vendor/github.com/mailru/easyjson/LICENSE 819e81c2ec13e1bbc47dc5e90bb4d88b - += vendor/github.com/matttproud/golang_protobuf_extensions/LICENSE a45ffb9ad39d4b4c1053bb27f6bb4272 - ================================================================================ ================================================================================ -= vendor/github.com/mailru/easyjson/jwriter licensed under: = - -Copyright (c) 2016 Mail.Ru Group - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: += vendor/github.com/Microsoft/go-winio licensed under: = -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +The MIT License (MIT) -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Copyright (c) 2015 Microsoft -= vendor/github.com/mailru/easyjson/LICENSE 819e81c2ec13e1bbc47dc5e90bb4d88b - -================================================================================ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -================================================================================ -= vendor/github.com/MakeNowJust/heredoc licensed under: = +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -The MIT License (MIT) - -Copyright (c) 2014 TSUYUSATO Kitsune - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -= vendor/github.com/MakeNowJust/heredoc/LICENSE 15e1c8f1d3c204c05f71630afacbc92b - += vendor/github.com/Microsoft/go-winio/LICENSE 69205ff73858f2c22b2ca135b557e8ef - ================================================================================ ================================================================================ -= vendor/github.com/matttproud/golang_protobuf_extensions/pbutil licensed under: = += vendor/github.com/miekg/coredns/middleware/etcd/msg licensed under: = Apache License Version 2.0, January 2004 @@ -66669,7 +69920,7 @@ THE SOFTWARE. APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -66677,7 +69928,7 @@ THE SOFTWARE. same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2013 Matt T. Proud + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -66691,44 +69942,54 @@ THE SOFTWARE. See the License for the specific language governing permissions and limitations under the License. -= vendor/github.com/matttproud/golang_protobuf_extensions/LICENSE a45ffb9ad39d4b4c1053bb27f6bb4272 - += vendor/github.com/miekg/coredns/LICENSE e3fc50a88d0a364313df4b21ef20c29e - ================================================================================ ================================================================================ -= vendor/github.com/Microsoft/go-winio licensed under: = += vendor/github.com/miekg/dns licensed under: = -The MIT License (MIT) +Extensions of the original work are copyright (c) 2011 Miek Gieben -Copyright (c) 2015 Microsoft +As this is fork of the official Go code the same license applies: -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Copyright (c) 2009 The Go Authors. All rights reserved. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/github.com/Microsoft/go-winio/LICENSE 69205ff73858f2c22b2ca135b557e8ef - + += vendor/github.com/miekg/dns/LICENSE 147353de6868a20caa562d26eab7b3c5 - ================================================================================ ================================================================================ -= vendor/github.com/miekg/coredns/middleware/etcd/msg licensed under: = += vendor/github.com/mistifyio/go-zfs licensed under: = - Apache License +Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -66916,7 +70177,7 @@ SOFTWARE. same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright (c) 2014, OmniTI Computer Consulting, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -66929,256 +70190,36 @@ SOFTWARE. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -= vendor/github.com/miekg/coredns/LICENSE e3fc50a88d0a364313df4b21ef20c29e - -================================================================================ - - -================================================================================ -= vendor/github.com/miekg/dns licensed under: = - -Extensions of the original work are copyright (c) 2011 Miek Gieben - -As this is fork of the official Go code the same license applies: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -= vendor/github.com/miekg/dns/LICENSE 147353de6868a20caa562d26eab7b3c5 - += vendor/github.com/mistifyio/go-zfs/LICENSE cce9462224bfb44c1866ef7bd5eddf54 - ================================================================================ ================================================================================ -= vendor/github.com/mistifyio/go-zfs licensed under: = - -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS += vendor/github.com/mitchellh/go-homedir licensed under: = - APPENDIX: How to apply the Apache License to your work. +The MIT License (MIT) - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. +Copyright (c) 2013 Mitchell Hashimoto - Copyright (c) 2014, OmniTI Computer Consulting, Inc. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. - http://www.apache.org/licenses/LICENSE-2.0 +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -= vendor/github.com/mistifyio/go-zfs/LICENSE cce9462224bfb44c1866ef7bd5eddf54 - += vendor/github.com/mitchellh/go-homedir/LICENSE 3f7765c3d4f58e1f84c4313cecf0f5bd - ================================================================================ @@ -78166,6 +81207,35 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ +================================================================================ += vendor/github.com/sethgrid/pester licensed under: = + +MIT License + +Copyright (c) SendGrid 2016 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + += vendor/github.com/sethgrid/pester/LICENSE.md cf9c2cc5227e89ba3dab4b6847e9bfb8 - +================================================================================ + + ================================================================================ = vendor/github.com/shurcooL/sanitized_anchor_name licensed under: = @@ -85516,12 +88586,82 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/crypto/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/golang.org/x/crypto/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - +================================================================================ + + +================================================================================ += vendor/golang.org/x/crypto/ssh/terminal licensed under: = + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/golang.org/x/crypto/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - +================================================================================ + + +================================================================================ += vendor/golang.org/x/exp/inotify licensed under: = + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/golang.org/x/exp/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - ================================================================================ ================================================================================ -= vendor/golang.org/x/crypto/ssh/terminal licensed under: = += vendor/golang.org/x/net/context licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85551,12 +88691,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/crypto/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/golang.org/x/net/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - ================================================================================ ================================================================================ -= vendor/golang.org/x/exp/inotify licensed under: = += vendor/golang.org/x/net/context/ctxhttp licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85586,12 +88726,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/exp/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/golang.org/x/net/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - ================================================================================ ================================================================================ -= vendor/golang.org/x/net/context licensed under: = += vendor/golang.org/x/net/html licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85626,7 +88766,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/context/ctxhttp licensed under: = += vendor/golang.org/x/net/html/atom licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85661,7 +88801,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/html licensed under: = += vendor/golang.org/x/net/http2 licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85696,7 +88836,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/html/atom licensed under: = += vendor/golang.org/x/net/http2/hpack licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85731,7 +88871,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/http2 licensed under: = += vendor/golang.org/x/net/idna licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85766,7 +88906,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/http2/hpack licensed under: = += vendor/golang.org/x/net/internal/timeseries licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85801,7 +88941,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/idna licensed under: = += vendor/golang.org/x/net/lex/httplex licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85836,7 +88976,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/internal/timeseries licensed under: = += vendor/golang.org/x/net/proxy licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85871,7 +89011,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/lex/httplex licensed under: = += vendor/golang.org/x/net/trace licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85906,7 +89046,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/proxy licensed under: = += vendor/golang.org/x/net/websocket licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -85941,9 +89081,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/net/trace licensed under: = += vendor/golang.org/x/oauth2 licensed under: = -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright (c) 2009 The oauth2 Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -85971,14 +89111,14 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/net/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/golang.org/x/oauth2/LICENSE 704b1e0c436dbf193e7dcbd4cf06ec81 - ================================================================================ ================================================================================ -= vendor/golang.org/x/net/websocket licensed under: = += vendor/golang.org/x/oauth2/google licensed under: = -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright (c) 2009 The oauth2 Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -86006,12 +89146,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/net/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/golang.org/x/oauth2/LICENSE 704b1e0c436dbf193e7dcbd4cf06ec81 - ================================================================================ ================================================================================ -= vendor/golang.org/x/oauth2 licensed under: = += vendor/golang.org/x/oauth2/internal licensed under: = Copyright (c) 2009 The oauth2 Authors. All rights reserved. @@ -86046,7 +89186,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/oauth2/google licensed under: = += vendor/golang.org/x/oauth2/jws licensed under: = Copyright (c) 2009 The oauth2 Authors. All rights reserved. @@ -86081,7 +89221,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/oauth2/internal licensed under: = += vendor/golang.org/x/oauth2/jwt licensed under: = Copyright (c) 2009 The oauth2 Authors. All rights reserved. @@ -86116,9 +89256,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/oauth2/jws licensed under: = += vendor/golang.org/x/sys/unix licensed under: = -Copyright (c) 2009 The oauth2 Authors. All rights reserved. +Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -86146,14 +89286,14 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/oauth2/LICENSE 704b1e0c436dbf193e7dcbd4cf06ec81 - += vendor/golang.org/x/sys/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - ================================================================================ ================================================================================ -= vendor/golang.org/x/oauth2/jwt licensed under: = += vendor/golang.org/x/sys/windows licensed under: = -Copyright (c) 2009 The oauth2 Authors. All rights reserved. +Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -86181,12 +89321,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/oauth2/LICENSE 704b1e0c436dbf193e7dcbd4cf06ec81 - += vendor/golang.org/x/sys/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - ================================================================================ ================================================================================ -= vendor/golang.org/x/sys/unix licensed under: = += vendor/golang.org/x/text/cases licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86216,12 +89356,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/sys/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/golang.org/x/text/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - ================================================================================ ================================================================================ -= vendor/golang.org/x/sys/windows licensed under: = += vendor/golang.org/x/text/encoding licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86251,12 +89391,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/sys/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/golang.org/x/text/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - ================================================================================ ================================================================================ -= vendor/golang.org/x/text/cases licensed under: = += vendor/golang.org/x/text/encoding/internal licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86291,7 +89431,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/encoding licensed under: = += vendor/golang.org/x/text/encoding/internal/identifier licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86326,7 +89466,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/encoding/internal licensed under: = += vendor/golang.org/x/text/encoding/unicode licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86361,7 +89501,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/encoding/internal/identifier licensed under: = += vendor/golang.org/x/text/internal licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86396,7 +89536,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/encoding/unicode licensed under: = += vendor/golang.org/x/text/internal/tag licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86431,7 +89571,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/internal/tag licensed under: = += vendor/golang.org/x/text/internal/utf8internal licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86466,7 +89606,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/internal/utf8internal licensed under: = += vendor/golang.org/x/text/language licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86501,7 +89641,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/language licensed under: = += vendor/golang.org/x/text/runes licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86536,7 +89676,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/runes licensed under: = += vendor/golang.org/x/text/secure/bidirule licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86571,7 +89711,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/secure/bidirule licensed under: = += vendor/golang.org/x/text/secure/precis licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86606,7 +89746,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/secure/precis licensed under: = += vendor/golang.org/x/text/transform licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86641,7 +89781,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/transform licensed under: = += vendor/golang.org/x/text/unicode/bidi licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86676,7 +89816,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/unicode/bidi licensed under: = += vendor/golang.org/x/text/unicode/norm licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86711,7 +89851,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/unicode/norm licensed under: = += vendor/golang.org/x/text/width licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86746,7 +89886,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/golang.org/x/text/width licensed under: = += vendor/golang.org/x/time/rate licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86776,12 +89916,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/text/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/golang.org/x/time/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - ================================================================================ ================================================================================ -= vendor/golang.org/x/time/rate licensed under: = += vendor/golang.org/x/tools/container/intsets licensed under: = Copyright (c) 2009 The Go Authors. All rights reserved. @@ -86811,14 +89951,14 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/time/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/golang.org/x/tools/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - ================================================================================ ================================================================================ -= vendor/golang.org/x/tools/container/intsets licensed under: = += vendor/google.golang.org/api/cloudkms/v1 licensed under: = -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright (c) 2011 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -86846,12 +89986,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/tools/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 - += vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - ================================================================================ ================================================================================ -= vendor/google.golang.org/api/cloudkms/v1 licensed under: = += vendor/google.golang.org/api/cloudmonitoring/v2beta2 licensed under: = Copyright (c) 2011 Google Inc. All rights reserved. @@ -86886,7 +90026,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/api/cloudmonitoring/v2beta2 licensed under: = += vendor/google.golang.org/api/compute/v0.alpha licensed under: = Copyright (c) 2011 Google Inc. All rights reserved. @@ -86921,7 +90061,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/api/compute/v0.alpha licensed under: = += vendor/google.golang.org/api/compute/v0.beta licensed under: = Copyright (c) 2011 Google Inc. All rights reserved. @@ -86956,7 +90096,147 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/api/compute/v0.beta licensed under: = += vendor/google.golang.org/api/compute/v1 licensed under: = + +Copyright (c) 2011 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - +================================================================================ + + +================================================================================ += vendor/google.golang.org/api/container/v1 licensed under: = + +Copyright (c) 2011 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - +================================================================================ + + +================================================================================ += vendor/google.golang.org/api/dns/v1 licensed under: = + +Copyright (c) 2011 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - +================================================================================ + + +================================================================================ += vendor/google.golang.org/api/gensupport licensed under: = + +Copyright (c) 2011 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - +================================================================================ + + +================================================================================ += vendor/google.golang.org/api/googleapi licensed under: = Copyright (c) 2011 Google Inc. All rights reserved. @@ -86991,7 +90271,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/api/compute/v1 licensed under: = += vendor/google.golang.org/api/googleapi/internal/uritemplates licensed under: = Copyright (c) 2011 Google Inc. All rights reserved. @@ -87026,7 +90306,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/api/container/v1 licensed under: = += vendor/google.golang.org/api/logging/v2beta1 licensed under: = Copyright (c) 2011 Google Inc. All rights reserved. @@ -87061,7 +90341,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/api/dns/v1 licensed under: = += vendor/google.golang.org/api/monitoring/v3 licensed under: = Copyright (c) 2011 Google Inc. All rights reserved. @@ -87096,7 +90376,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/api/gensupport licensed under: = += vendor/google.golang.org/api/pubsub/v1 licensed under: = Copyright (c) 2011 Google Inc. All rights reserved. @@ -87131,21 +90411,232 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/api/googleapi licensed under: = += vendor/google.golang.org/genproto/googleapis/rpc/status licensed under: = -Copyright (c) 2011 Google Inc. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/google.golang.org/genproto/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 - +================================================================================ + + +================================================================================ += vendor/google.golang.org/grpc licensed under: = + +Copyright 2014, Google Inc. +All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -87161,26 +90652,27 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - += vendor/google.golang.org/grpc/LICENSE a4bad33881612090c6035d8393175996 - ================================================================================ ================================================================================ -= vendor/google.golang.org/api/googleapi/internal/uritemplates licensed under: = += vendor/google.golang.org/grpc/codes licensed under: = -Copyright (c) 2011 Google Inc. All rights reserved. +Copyright 2014, Google Inc. +All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -87196,26 +90688,27 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - += vendor/google.golang.org/grpc/LICENSE a4bad33881612090c6035d8393175996 - ================================================================================ ================================================================================ -= vendor/google.golang.org/api/logging/v2beta1 licensed under: = += vendor/google.golang.org/grpc/credentials licensed under: = -Copyright (c) 2011 Google Inc. All rights reserved. +Copyright 2014, Google Inc. +All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -87231,26 +90724,27 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - += vendor/google.golang.org/grpc/LICENSE a4bad33881612090c6035d8393175996 - ================================================================================ ================================================================================ -= vendor/google.golang.org/api/monitoring/v3 licensed under: = += vendor/google.golang.org/grpc/grpclb/grpc_lb_v1 licensed under: = -Copyright (c) 2011 Google Inc. All rights reserved. +Copyright 2014, Google Inc. +All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -87266,26 +90760,27 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - += vendor/google.golang.org/grpc/LICENSE a4bad33881612090c6035d8393175996 - ================================================================================ ================================================================================ -= vendor/google.golang.org/api/pubsub/v1 licensed under: = += vendor/google.golang.org/grpc/grpclog licensed under: = -Copyright (c) 2011 Google Inc. All rights reserved. +Copyright 2014, Google Inc. +All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -87301,12 +90796,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40 - += vendor/google.golang.org/grpc/LICENSE a4bad33881612090c6035d8393175996 - ================================================================================ ================================================================================ -= vendor/google.golang.org/grpc licensed under: = += vendor/google.golang.org/grpc/internal licensed under: = Copyright 2014, Google Inc. All rights reserved. @@ -87342,7 +90837,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/grpc/codes licensed under: = += vendor/google.golang.org/grpc/keepalive licensed under: = Copyright 2014, Google Inc. All rights reserved. @@ -87378,7 +90873,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/grpc/credentials licensed under: = += vendor/google.golang.org/grpc/metadata licensed under: = Copyright 2014, Google Inc. All rights reserved. @@ -87414,7 +90909,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/grpc/grpclog licensed under: = += vendor/google.golang.org/grpc/naming licensed under: = Copyright 2014, Google Inc. All rights reserved. @@ -87450,7 +90945,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/grpc/internal licensed under: = += vendor/google.golang.org/grpc/peer licensed under: = Copyright 2014, Google Inc. All rights reserved. @@ -87486,7 +90981,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/grpc/metadata licensed under: = += vendor/google.golang.org/grpc/stats licensed under: = Copyright 2014, Google Inc. All rights reserved. @@ -87522,7 +91017,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/grpc/naming licensed under: = += vendor/google.golang.org/grpc/status licensed under: = Copyright 2014, Google Inc. All rights reserved. @@ -87558,7 +91053,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -= vendor/google.golang.org/grpc/peer licensed under: = += vendor/google.golang.org/grpc/tap licensed under: = Copyright 2014, Google Inc. All rights reserved. diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index da659c46833c9..8f2e6304d9bb4 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -100,5 +100,77 @@ aliases: sig-apps-api-approvers: - erictune - smarterclayton - sig-leads: + milestone-maintainers: + - lavalamp + - deads2k + - michelleN + - mattfarina + - prydonius + - bgrant0607 + - jdumars + - ericchiang + - liggitt + - deads2k + - mwielgus + - directxman12 + - justinsb + - kris-nova + - chrislovecnm + - mfburnett + - slack + - colemickens + - foxish + - fabianofranz + - pwittrock + - AdoHe + - lukemarsden + - jbeda + - roberthbailey + - zehicle + - jdumars + - grodrigues3 + - Phillels + - devin-donnelly + - jaredbhatti + - csbell + - quinton-hoole + - piosz + - fabxc + - thockin + - dcbw + - caseydavenport + - dchen1107 + - derekwaynecarr + - zen + - marcoceppi + - dghubble + - idvoretskyi + - xsgordon + - apsinha + - idvoretskyi + - calebamiles + - pwittrock + - calebamiles + - wojtek-t + - countspongebob + - jbeda + - davidopp + - timothysc + - pmorie + - arschles + - vaikas-google + - duglin + - saad-ali + - childsb - spiffxp + - fejta + - timothysc + - danielromlein + - floreks + - michmike + - abgworrall + - krzyzacy + - steveperry-53 + - radhikpac + - jpbetz + diff --git a/api/OWNERS b/api/OWNERS index 7009d88bf109e..4d61bb354eab8 100644 --- a/api/OWNERS +++ b/api/OWNERS @@ -3,6 +3,8 @@ approvers: - lavalamp - smarterclayton - thockin +- liggitt +# - bgrant0607 # manual escalations only reviewers: - thockin - lavalamp diff --git a/api/openapi-spec/README.md b/api/openapi-spec/README.md new file mode 100644 index 0000000000000..31e0ed851400f --- /dev/null +++ b/api/openapi-spec/README.md @@ -0,0 +1,60 @@ +# Kubernetes's OpenAPI Specification + +This folder contains an [OpenAPI specification][openapi] for Kubernetes API. + +## Vendor Extensions + +Kuberntes extends OpenAPI using these extensions. Note the version that +extensions has been added. + +### `x-kubernetes-group-version-kind` + +Operations and Definitions may have `x-kubernetes-group-version-kind` if they +are associated with a [kubernetes resource](https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#resources). + + +For example: + +``` json +"paths": { + ... + "/api/v1/namespaces/{namespace}/pods/{name}": { + ... + "get": { + ... + "x-kubernetes-group-version-kind": { + "group": "", + "version": "v1", + "kind": "Pod" + } + } + } +} +``` + +### `x-kubernetes-action` + +Operations and Definitions may have `x-kubernetes-action` if they +are associated with a [kubernetes resource](https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#resources). +Action can be one of `get`, `list`, `put`, `patch`, `post`, `delete`, `deletecollection`, `watch`, `watchlist`, `proxy`, or `connect`. + + +For example: + +``` json +"paths": { + ... + "/api/v1/namespaces/{namespace}/pods/{name}": { + ... + "get": { + ... + "x-kubernetes-action": "list" + } + } +} +``` + +### `x-kubernetes-patch-strategy` and `x-kubernetes-patch-merge-key` + +Some of the definitions may have these extensions. For more information about PatchStrategy and PatchMergeKey see +[strategic-merge-patch] (https://github.com/kubernetes/community/blob/3a1e6d22f812751ee88eccf7c59101852de63d5b/contributors/devel/strategic-merge-patch.md). diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index ec76397775af5..b27589f30e246 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -54660,10 +54660,6 @@ "name" ], "properties": { - "failurePolicy": { - "description": "FailurePolicy defines what happens if the responsible initializer controller fails to takes action. Allowed values are Ignore, or Fail. If \"Ignore\" is set, initializer is removed from the initializers list of an object if the timeout is reached; If \"Fail\" is set, admissionregistration returns timeout error if the timeout is reached.", - "type": "string" - }, "name": { "description": "Name is the identifier of the initializer. It will be added to the object that needs to be initialized. Name should be fully qualified, e.g., alwayspullimages.kubernetes.io, where \"alwayspullimages\" is the name of the webhook, and kubernetes.io is the name of the organization. Required", "type": "string" @@ -55087,7 +55083,7 @@ "collisionCount": { "description": "Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet.", "type": "integer", - "format": "int64" + "format": "int32" }, "conditions": { "description": "Represents the latest available observations of a deployment's current state.", @@ -55355,7 +55351,7 @@ "collisionCount": { "description": "collisionCount is the count of hash collisions for the StatefulSet. The StatefulSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision.", "type": "integer", - "format": "int64" + "format": "int32" }, "currentReplicas": { "description": "currentReplicas is the number of Pods created by the StatefulSet controller from the StatefulSet version indicated by currentRevision.", @@ -55583,7 +55579,7 @@ "collisionCount": { "description": "Count of hash collisions for the DaemonSet. The DaemonSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision.", "type": "integer", - "format": "int64" + "format": "int32" }, "currentNumberScheduled": { "description": "The number of nodes that are running at least 1 daemon pod and are supposed to run the daemon pod. More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/", @@ -55794,7 +55790,7 @@ "collisionCount": { "description": "Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet.", "type": "integer", - "format": "int64" + "format": "int32" }, "conditions": { "description": "Represents the latest available observations of a deployment's current state.", @@ -56221,7 +56217,7 @@ "collisionCount": { "description": "collisionCount is the count of hash collisions for the StatefulSet. The StatefulSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision.", "type": "integer", - "format": "int64" + "format": "int32" }, "currentReplicas": { "description": "currentReplicas is the number of Pods created by the StatefulSet controller from the StatefulSet version indicated by currentRevision.", @@ -58134,6 +58130,31 @@ } } }, + "io.k8s.api.core.v1.AzureFilePersistentVolumeSource": { + "description": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", + "required": [ + "secretName", + "shareName" + ], + "properties": { + "readOnly": { + "description": "Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "secretName": { + "description": "the name of secret that contains Azure Storage Account Name and Key", + "type": "string" + }, + "secretNamespace": { + "description": "the namespace of the secret that contains Azure Storage Account Name and Key default is the same as the Pod", + "type": "string" + }, + "shareName": { + "description": "Share Name", + "type": "string" + } + } + }, "io.k8s.api.core.v1.AzureFileVolumeSource": { "description": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", "required": [ @@ -58205,6 +58226,41 @@ } } }, + "io.k8s.api.core.v1.CephFSPersistentVolumeSource": { + "description": "Represents a Ceph Filesystem mount that lasts the lifetime of a pod Cephfs volumes do not support ownership management or SELinux relabeling.", + "required": [ + "monitors" + ], + "properties": { + "monitors": { + "description": "Required: Monitors is a collection of Ceph monitors More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", + "type": "array", + "items": { + "type": "string" + } + }, + "path": { + "description": "Optional: Used as the mounted root, rather than the full Ceph tree, default is /", + "type": "string" + }, + "readOnly": { + "description": "Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", + "type": "boolean" + }, + "secretFile": { + "description": "Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", + "type": "string" + }, + "secretRef": { + "description": "Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretReference" + }, + "user": { + "description": "Optional: User is the rados user name, default is admin More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", + "type": "string" + } + } + }, "io.k8s.api.core.v1.CephFSVolumeSource": { "description": "Represents a Ceph Filesystem mount that lasts the lifetime of a pod Cephfs volumes do not support ownership management or SELinux relabeling.", "required": [ @@ -58260,6 +58316,16 @@ } } }, + "io.k8s.api.core.v1.ClientIPConfig": { + "description": "ClientIPConfig represents the configurations of Client IP based session affinity.", + "properties": { + "timeoutSeconds": { + "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", + "type": "integer", + "format": "int32" + } + } + }, "io.k8s.api.core.v1.ComponentCondition": { "description": "Information about the condition of a component.", "required": [ @@ -59048,7 +59114,7 @@ "$ref": "#/definitions/io.k8s.api.core.v1.ObjectFieldSelector" }, "resourceFieldRef": { - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.", + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.", "$ref": "#/definitions/io.k8s.api.core.v1.ResourceFieldSelector" }, "secretKeyRef": { @@ -59407,7 +59473,11 @@ ], "properties": { "path": { - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "type": "string" + }, + "type": { + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", "type": "string" } } @@ -59432,6 +59502,10 @@ "description": "Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi", "type": "string" }, + "initiatorName": { + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection.", + "type": "string" + }, "iqn": { "description": "Target iSCSI Qualified Name.", "type": "string" @@ -59964,9 +60038,7 @@ "properties": { "key": { "description": "The label key that the selector applies to.", - "type": "string", - "x-kubernetes-patch-merge-key": "key", - "x-kubernetes-patch-strategy": "merge" + "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", @@ -60423,7 +60495,7 @@ }, "azureFile": { "description": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", - "$ref": "#/definitions/io.k8s.api.core.v1.AzureFileVolumeSource" + "$ref": "#/definitions/io.k8s.api.core.v1.AzureFilePersistentVolumeSource" }, "capacity": { "description": "A description of the persistent volume's resources and capacity. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#capacity", @@ -60434,7 +60506,7 @@ }, "cephfs": { "description": "CephFS represents a Ceph FS mount on the host that shares a pod's lifetime", - "$ref": "#/definitions/io.k8s.api.core.v1.CephFSVolumeSource" + "$ref": "#/definitions/io.k8s.api.core.v1.CephFSPersistentVolumeSource" }, "cinder": { "description": "Cinder represents a cinder volume attached and mounted on kubelets host machine More info: https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md", @@ -60476,6 +60548,13 @@ "description": "Local represents directly-attached storage with node affinity", "$ref": "#/definitions/io.k8s.api.core.v1.LocalVolumeSource" }, + "mountOptions": { + "description": "A list of mount options, e.g. [\"ro\", \"soft\"]. Not validated - mount will simply fail if one is invalid. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options", + "type": "array", + "items": { + "type": "string" + } + }, "nfs": { "description": "NFS represents an NFS mount on the host. Provisioned by an admin. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", "$ref": "#/definitions/io.k8s.api.core.v1.NFSVolumeSource" @@ -60877,7 +60956,7 @@ "$ref": "#/definitions/io.k8s.api.core.v1.Volume" }, "x-kubernetes-patch-merge-key": "name", - "x-kubernetes-patch-strategy": "merge" + "x-kubernetes-patch-strategy": "merge,retainKeys" } } }, @@ -61692,6 +61771,19 @@ } } }, + "io.k8s.api.core.v1.SecretReference": { + "description": "SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace", + "properties": { + "name": { + "description": "Name is unique within a namespace to reference a secret resource.", + "type": "string" + }, + "namespace": { + "description": "Namespace defines the space within which the secret name must be unique.", + "type": "string" + } + } + }, "io.k8s.api.core.v1.SecretVolumeSource": { "description": "Adapts a Secret into a volume.\n\nThe contents of the target Secret's Data field will be presented in a volume as files using the keys in the Data field as the file names. Secret volumes support ownership management and SELinux relabeling.", "properties": { @@ -61987,6 +62079,10 @@ "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, + "sessionAffinityConfig": { + "description": "sessionAffinityConfig contains the configurations of session affinity.", + "$ref": "#/definitions/io.k8s.api.core.v1.SessionAffinityConfig" + }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services---service-types", "type": "string" @@ -62002,6 +62098,15 @@ } } }, + "io.k8s.api.core.v1.SessionAffinityConfig": { + "description": "SessionAffinityConfig represents the configurations of session affinity.", + "properties": { + "clientIP": { + "description": "clientIP contains the configurations of Client IP based session affinity.", + "$ref": "#/definitions/io.k8s.api.core.v1.ClientIPConfig" + } + } + }, "io.k8s.api.core.v1.StorageOSPersistentVolumeSource": { "description": "Represents a StorageOS persistent volume resource.", "properties": { @@ -62081,9 +62186,7 @@ }, "key": { "description": "Required. The taint key to be applied to a node.", - "type": "string", - "x-kubernetes-patch-merge-key": "key", - "x-kubernetes-patch-strategy": "merge" + "type": "string" }, "timeAdded": { "description": "TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints.", @@ -62104,9 +62207,7 @@ }, "key": { "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", - "type": "string", - "x-kubernetes-patch-merge-key": "key", - "x-kubernetes-patch-strategy": "merge" + "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", @@ -62440,7 +62541,7 @@ "collisionCount": { "description": "Count of hash collisions for the DaemonSet. The DaemonSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision.", "type": "integer", - "format": "int64" + "format": "int32" }, "currentNumberScheduled": { "description": "The number of nodes that are running at least 1 daemon pod and are supposed to run the daemon pod. More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/", @@ -62674,6 +62775,7 @@ }, "strategy": { "description": "The deployment strategy to use to replace existing pods with new ones.", + "x-kubernetes-patch-strategy": "retainKeys", "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DeploymentStrategy" }, "template": { @@ -62693,7 +62795,7 @@ "collisionCount": { "description": "Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet.", "type": "integer", - "format": "int64" + "format": "int32" }, "conditions": { "description": "Represents the latest available observations of a deployment's current state.", @@ -62829,6 +62931,25 @@ } } }, + "io.k8s.api.extensions.v1beta1.IPBlock": { + "description": "IPBlock describes a particular CIDR (Ex. \"192.168.1.1/24\") that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should not be included within this rule.", + "required": [ + "cidr" + ], + "properties": { + "cidr": { + "description": "CIDR is a string representing the IP Block Valid examples are \"192.168.1.1/24\"", + "type": "string" + }, + "except": { + "description": "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are \"192.168.1.1/24\" Except values will be rejected if they are outside the CIDR range", + "type": "array", + "items": { + "type": "string" + } + } + } + }, "io.k8s.api.extensions.v1beta1.Ingress": { "description": "Ingress is a collection of rules that allow inbound connections to reach the endpoints defined by a backend. An Ingress can be configured to give services externally-reachable urls, load balance traffic, terminate SSL, offer name based virtual hosting etc.", "properties": { @@ -63055,6 +63176,10 @@ }, "io.k8s.api.extensions.v1beta1.NetworkPolicyPeer": { "properties": { + "ipBlock": { + "description": "IPBlock defines policy on a particular IPBlock", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.IPBlock" + }, "namespaceSelector": { "description": "Selects Namespaces using cluster scoped-labels. This matches all pods in all namespaces selected by this label selector. This field follows standard label selector semantics. If present but empty, this selector selects all namespaces.", "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" @@ -63556,6 +63681,25 @@ } } }, + "io.k8s.api.networking.v1.IPBlock": { + "description": "IPBlock describes a particular CIDR (Ex. \"192.168.1.1/24\") that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should not be included within this rule.", + "required": [ + "cidr" + ], + "properties": { + "cidr": { + "description": "CIDR is a string representing the IP Block Valid examples are \"192.168.1.1/24\"", + "type": "string" + }, + "except": { + "description": "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are \"192.168.1.1/24\" Except values will be rejected if they are outside the CIDR range", + "type": "array", + "items": { + "type": "string" + } + } + } + }, "io.k8s.api.networking.v1.NetworkPolicy": { "description": "NetworkPolicy describes what network traffic is allowed for a set of Pods", "properties": { @@ -63640,6 +63784,10 @@ "io.k8s.api.networking.v1.NetworkPolicyPeer": { "description": "NetworkPolicyPeer describes a peer to allow traffic from. Exactly one of its fields must be specified.", "properties": { + "ipBlock": { + "description": "IPBlock defines policy on a particular IPBlock", + "$ref": "#/definitions/io.k8s.api.networking.v1.IPBlock" + }, "namespaceSelector": { "description": "Selects Namespaces using cluster scoped-labels. This matches all pods in all namespaces selected by this label selector. This field follows standard label selector semantics. If present but empty, this selector selects all namespaces.", "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" @@ -65682,7 +65830,9 @@ "type": "array", "items": { "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Initializer" - } + }, + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" }, "result": { "description": "If result is set with the Failure field, the object will be persisted to storage and then deleted, ensuring that other clients can observe the deletion.", @@ -66355,6 +66505,1314 @@ "type": "string" } } + }, + "io.k8s.kubernetes.pkg.api.v1.AWSElasticBlockStoreVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.AWSElasticBlockStoreVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.AWSElasticBlockStoreVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.Affinity": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Affinity instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "io.k8s.kubernetes.pkg.api.v1.AttachedVolume": { + "description": "Deprecated. Please use io.k8s.api.core.v1.AttachedVolume instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.AttachedVolume" + }, + "io.k8s.kubernetes.pkg.api.v1.AzureDiskVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.AzureDiskVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.AzureDiskVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.AzureFileVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.AzureFileVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.AzureFileVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.Binding": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Binding instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Binding" + }, + "io.k8s.kubernetes.pkg.api.v1.Capabilities": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Capabilities instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Capabilities" + }, + "io.k8s.kubernetes.pkg.api.v1.CephFSVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.CephFSVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.CephFSVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.CinderVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.CinderVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.CinderVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.ComponentCondition": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ComponentCondition instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ComponentCondition" + }, + "io.k8s.kubernetes.pkg.api.v1.ComponentStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ComponentStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ComponentStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.ComponentStatusList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ComponentStatusList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ComponentStatusList" + }, + "io.k8s.kubernetes.pkg.api.v1.ConfigMap": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ConfigMap instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMap" + }, + "io.k8s.kubernetes.pkg.api.v1.ConfigMapEnvSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ConfigMapEnvSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapEnvSource" + }, + "io.k8s.kubernetes.pkg.api.v1.ConfigMapKeySelector": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ConfigMapKeySelector instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector" + }, + "io.k8s.kubernetes.pkg.api.v1.ConfigMapList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ConfigMapList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapList" + }, + "io.k8s.kubernetes.pkg.api.v1.ConfigMapProjection": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ConfigMapProjection instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapProjection" + }, + "io.k8s.kubernetes.pkg.api.v1.ConfigMapVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ConfigMapVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.Container": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Container instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + }, + "io.k8s.kubernetes.pkg.api.v1.ContainerImage": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ContainerImage instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerImage" + }, + "io.k8s.kubernetes.pkg.api.v1.ContainerPort": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ContainerPort instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerPort" + }, + "io.k8s.kubernetes.pkg.api.v1.ContainerState": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ContainerState instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerState" + }, + "io.k8s.kubernetes.pkg.api.v1.ContainerStateRunning": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ContainerStateRunning instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerStateRunning" + }, + "io.k8s.kubernetes.pkg.api.v1.ContainerStateTerminated": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ContainerStateTerminated instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerStateTerminated" + }, + "io.k8s.kubernetes.pkg.api.v1.ContainerStateWaiting": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ContainerStateWaiting instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerStateWaiting" + }, + "io.k8s.kubernetes.pkg.api.v1.ContainerStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ContainerStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.DaemonEndpoint": { + "description": "Deprecated. Please use io.k8s.api.core.v1.DaemonEndpoint instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.DaemonEndpoint" + }, + "io.k8s.kubernetes.pkg.api.v1.DownwardAPIProjection": { + "description": "Deprecated. Please use io.k8s.api.core.v1.DownwardAPIProjection instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.DownwardAPIProjection" + }, + "io.k8s.kubernetes.pkg.api.v1.DownwardAPIVolumeFile": { + "description": "Deprecated. Please use io.k8s.api.core.v1.DownwardAPIVolumeFile instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.DownwardAPIVolumeFile" + }, + "io.k8s.kubernetes.pkg.api.v1.DownwardAPIVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.DownwardAPIVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.DownwardAPIVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.EmptyDirVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EmptyDirVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.EndpointAddress": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EndpointAddress instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EndpointAddress" + }, + "io.k8s.kubernetes.pkg.api.v1.EndpointPort": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EndpointPort instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EndpointPort" + }, + "io.k8s.kubernetes.pkg.api.v1.EndpointSubset": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EndpointSubset instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EndpointSubset" + }, + "io.k8s.kubernetes.pkg.api.v1.Endpoints": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Endpoints instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Endpoints" + }, + "io.k8s.kubernetes.pkg.api.v1.EndpointsList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EndpointsList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EndpointsList" + }, + "io.k8s.kubernetes.pkg.api.v1.EnvFromSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EnvFromSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EnvFromSource" + }, + "io.k8s.kubernetes.pkg.api.v1.EnvVar": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EnvVar instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "io.k8s.kubernetes.pkg.api.v1.EnvVarSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EnvVarSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVarSource" + }, + "io.k8s.kubernetes.pkg.api.v1.Event": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Event instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Event" + }, + "io.k8s.kubernetes.pkg.api.v1.EventList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EventList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EventList" + }, + "io.k8s.kubernetes.pkg.api.v1.EventSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.EventSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.EventSource" + }, + "io.k8s.kubernetes.pkg.api.v1.ExecAction": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ExecAction instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ExecAction" + }, + "io.k8s.kubernetes.pkg.api.v1.FCVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.FCVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.FCVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.FlexVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.FlexVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.FlexVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.FlockerVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.FlockerVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.FlockerVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.GCEPersistentDiskVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.GCEPersistentDiskVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.GCEPersistentDiskVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.GitRepoVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.GitRepoVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.GitRepoVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.GlusterfsVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.GlusterfsVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.GlusterfsVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.HTTPGetAction": { + "description": "Deprecated. Please use io.k8s.api.core.v1.HTTPGetAction instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.HTTPGetAction" + }, + "io.k8s.kubernetes.pkg.api.v1.HTTPHeader": { + "description": "Deprecated. Please use io.k8s.api.core.v1.HTTPHeader instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.HTTPHeader" + }, + "io.k8s.kubernetes.pkg.api.v1.Handler": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Handler instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Handler" + }, + "io.k8s.kubernetes.pkg.api.v1.HostAlias": { + "description": "Deprecated. Please use io.k8s.api.core.v1.HostAlias instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "io.k8s.kubernetes.pkg.api.v1.HostPathVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.HostPathVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.HostPathVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.ISCSIVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ISCSIVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ISCSIVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.KeyToPath": { + "description": "Deprecated. Please use io.k8s.api.core.v1.KeyToPath instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.KeyToPath" + }, + "io.k8s.kubernetes.pkg.api.v1.Lifecycle": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Lifecycle instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle" + }, + "io.k8s.kubernetes.pkg.api.v1.LimitRange": { + "description": "Deprecated. Please use io.k8s.api.core.v1.LimitRange instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.LimitRange" + }, + "io.k8s.kubernetes.pkg.api.v1.LimitRangeItem": { + "description": "Deprecated. Please use io.k8s.api.core.v1.LimitRangeItem instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.LimitRangeItem" + }, + "io.k8s.kubernetes.pkg.api.v1.LimitRangeList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.LimitRangeList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.LimitRangeList" + }, + "io.k8s.kubernetes.pkg.api.v1.LimitRangeSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.LimitRangeSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.LimitRangeSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.LoadBalancerIngress": { + "description": "Deprecated. Please use io.k8s.api.core.v1.LoadBalancerIngress instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.LoadBalancerIngress" + }, + "io.k8s.kubernetes.pkg.api.v1.LoadBalancerStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.LoadBalancerStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.LoadBalancerStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.LocalObjectReference": { + "description": "Deprecated. Please use io.k8s.api.core.v1.LocalObjectReference instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.LocalObjectReference" + }, + "io.k8s.kubernetes.pkg.api.v1.LocalVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.LocalVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.LocalVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.NFSVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NFSVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NFSVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.Namespace": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Namespace instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Namespace" + }, + "io.k8s.kubernetes.pkg.api.v1.NamespaceList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NamespaceList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NamespaceList" + }, + "io.k8s.kubernetes.pkg.api.v1.NamespaceSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NamespaceSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NamespaceSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.NamespaceStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NamespaceStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NamespaceStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.Node": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Node instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Node" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeAddress": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeAddress instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeAddress" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeAffinity": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeAffinity instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeAffinity" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeCondition": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeCondition instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeCondition" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeDaemonEndpoints": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeDaemonEndpoints instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeDaemonEndpoints" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeList" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeSelector": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeSelector instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSelector" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeSelectorRequirement": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeSelectorRequirement instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSelectorRequirement" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeSelectorTerm": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeSelectorTerm instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSelectorTerm" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.NodeSystemInfo": { + "description": "Deprecated. Please use io.k8s.api.core.v1.NodeSystemInfo instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.NodeSystemInfo" + }, + "io.k8s.kubernetes.pkg.api.v1.ObjectFieldSelector": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ObjectFieldSelector instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ObjectFieldSelector" + }, + "io.k8s.kubernetes.pkg.api.v1.ObjectReference": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ObjectReference instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ObjectReference" + }, + "io.k8s.kubernetes.pkg.api.v1.PersistentVolume": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PersistentVolume instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolume" + }, + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaim": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PersistentVolumeClaim instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaim" + }, + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaimList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PersistentVolumeClaimList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimList" + }, + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaimSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PersistentVolumeClaimSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaimStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PersistentVolumeClaimStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaimVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PersistentVolumeClaimVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeClaimVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PersistentVolumeList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeList" + }, + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PersistentVolumeSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PersistentVolumeStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PersistentVolumeStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.PhotonPersistentDiskVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PhotonPersistentDiskVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PhotonPersistentDiskVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.Pod": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Pod instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Pod" + }, + "io.k8s.kubernetes.pkg.api.v1.PodAffinity": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodAffinity instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodAffinity" + }, + "io.k8s.kubernetes.pkg.api.v1.PodAffinityTerm": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodAffinityTerm instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodAffinityTerm" + }, + "io.k8s.kubernetes.pkg.api.v1.PodAntiAffinity": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodAntiAffinity instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodAntiAffinity" + }, + "io.k8s.kubernetes.pkg.api.v1.PodCondition": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodCondition instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodCondition" + }, + "io.k8s.kubernetes.pkg.api.v1.PodList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodList" + }, + "io.k8s.kubernetes.pkg.api.v1.PodSecurityContext": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodSecurityContext instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSecurityContext" + }, + "io.k8s.kubernetes.pkg.api.v1.PodSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.PodStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.PodTemplate": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodTemplate instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodTemplate" + }, + "io.k8s.kubernetes.pkg.api.v1.PodTemplateList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodTemplateList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodTemplateList" + }, + "io.k8s.kubernetes.pkg.api.v1.PodTemplateSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PodTemplateSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodTemplateSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.PortworxVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PortworxVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PortworxVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.PreferredSchedulingTerm": { + "description": "Deprecated. Please use io.k8s.api.core.v1.PreferredSchedulingTerm instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.PreferredSchedulingTerm" + }, + "io.k8s.kubernetes.pkg.api.v1.Probe": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Probe instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Probe" + }, + "io.k8s.kubernetes.pkg.api.v1.ProjectedVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ProjectedVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ProjectedVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.QuobyteVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.QuobyteVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.QuobyteVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.RBDVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.RBDVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.RBDVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.ReplicationController": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ReplicationController instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ReplicationController" + }, + "io.k8s.kubernetes.pkg.api.v1.ReplicationControllerCondition": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ReplicationControllerCondition instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ReplicationControllerCondition" + }, + "io.k8s.kubernetes.pkg.api.v1.ReplicationControllerList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ReplicationControllerList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ReplicationControllerList" + }, + "io.k8s.kubernetes.pkg.api.v1.ReplicationControllerSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ReplicationControllerSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ReplicationControllerSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.ReplicationControllerStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ReplicationControllerStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ReplicationControllerStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.ResourceFieldSelector": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ResourceFieldSelector instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceFieldSelector" + }, + "io.k8s.kubernetes.pkg.api.v1.ResourceQuota": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ResourceQuota instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceQuota" + }, + "io.k8s.kubernetes.pkg.api.v1.ResourceQuotaList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ResourceQuotaList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceQuotaList" + }, + "io.k8s.kubernetes.pkg.api.v1.ResourceQuotaSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ResourceQuotaSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceQuotaSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.ResourceQuotaStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ResourceQuotaStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceQuotaStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.ResourceRequirements": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ResourceRequirements instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "io.k8s.kubernetes.pkg.api.v1.SELinuxOptions": { + "description": "Deprecated. Please use io.k8s.api.core.v1.SELinuxOptions instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.SELinuxOptions" + }, + "io.k8s.kubernetes.pkg.api.v1.ScaleIOVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ScaleIOVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ScaleIOVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.Secret": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Secret instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Secret" + }, + "io.k8s.kubernetes.pkg.api.v1.SecretEnvSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.SecretEnvSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretEnvSource" + }, + "io.k8s.kubernetes.pkg.api.v1.SecretKeySelector": { + "description": "Deprecated. Please use io.k8s.api.core.v1.SecretKeySelector instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" + }, + "io.k8s.kubernetes.pkg.api.v1.SecretList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.SecretList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretList" + }, + "io.k8s.kubernetes.pkg.api.v1.SecretProjection": { + "description": "Deprecated. Please use io.k8s.api.core.v1.SecretProjection instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretProjection" + }, + "io.k8s.kubernetes.pkg.api.v1.SecretVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.SecretVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.SecurityContext": { + "description": "Deprecated. Please use io.k8s.api.core.v1.SecurityContext instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext" + }, + "io.k8s.kubernetes.pkg.api.v1.Service": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Service instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Service" + }, + "io.k8s.kubernetes.pkg.api.v1.ServiceAccount": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ServiceAccount instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ServiceAccount" + }, + "io.k8s.kubernetes.pkg.api.v1.ServiceAccountList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ServiceAccountList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ServiceAccountList" + }, + "io.k8s.kubernetes.pkg.api.v1.ServiceList": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ServiceList instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ServiceList" + }, + "io.k8s.kubernetes.pkg.api.v1.ServicePort": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ServicePort instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ServicePort" + }, + "io.k8s.kubernetes.pkg.api.v1.ServiceSpec": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ServiceSpec instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ServiceSpec" + }, + "io.k8s.kubernetes.pkg.api.v1.ServiceStatus": { + "description": "Deprecated. Please use io.k8s.api.core.v1.ServiceStatus instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.ServiceStatus" + }, + "io.k8s.kubernetes.pkg.api.v1.StorageOSPersistentVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.StorageOSPersistentVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.StorageOSPersistentVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.StorageOSVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.StorageOSVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.StorageOSVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.TCPSocketAction": { + "description": "Deprecated. Please use io.k8s.api.core.v1.TCPSocketAction instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.TCPSocketAction" + }, + "io.k8s.kubernetes.pkg.api.v1.Taint": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Taint instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Taint" + }, + "io.k8s.kubernetes.pkg.api.v1.Toleration": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Toleration instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + }, + "io.k8s.kubernetes.pkg.api.v1.Volume": { + "description": "Deprecated. Please use io.k8s.api.core.v1.Volume instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + }, + "io.k8s.kubernetes.pkg.api.v1.VolumeMount": { + "description": "Deprecated. Please use io.k8s.api.core.v1.VolumeMount instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + }, + "io.k8s.kubernetes.pkg.api.v1.VolumeProjection": { + "description": "Deprecated. Please use io.k8s.api.core.v1.VolumeProjection instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeProjection" + }, + "io.k8s.kubernetes.pkg.api.v1.VsphereVirtualDiskVolumeSource": { + "description": "Deprecated. Please use io.k8s.api.core.v1.VsphereVirtualDiskVolumeSource instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.VsphereVirtualDiskVolumeSource" + }, + "io.k8s.kubernetes.pkg.api.v1.WeightedPodAffinityTerm": { + "description": "Deprecated. Please use io.k8s.api.core.v1.WeightedPodAffinityTerm instead.", + "$ref": "#/definitions/io.k8s.api.core.v1.WeightedPodAffinityTerm" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.AdmissionHookClientConfig": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.AdmissionHookClientConfig instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.AdmissionHookClientConfig" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.ExternalAdmissionHook": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.ExternalAdmissionHook instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.ExternalAdmissionHook" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.ExternalAdmissionHookConfiguration": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.ExternalAdmissionHookConfiguration instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.ExternalAdmissionHookConfiguration" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.ExternalAdmissionHookConfigurationList": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.ExternalAdmissionHookConfigurationList instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.ExternalAdmissionHookConfigurationList" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.Initializer": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.Initializer instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.Initializer" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.InitializerConfiguration": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.InitializerConfiguration instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.InitializerConfiguration" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.InitializerConfigurationList": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.InitializerConfigurationList instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.InitializerConfigurationList" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.Rule": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.Rule instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.Rule" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.RuleWithOperations": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.RuleWithOperations instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.RuleWithOperations" + }, + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.ServiceReference": { + "description": "Deprecated. Please use io.k8s.api.admissionregistration.v1alpha1.ServiceReference instead.", + "$ref": "#/definitions/io.k8s.api.admissionregistration.v1alpha1.ServiceReference" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.ControllerRevision": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.ControllerRevision instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.ControllerRevision" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.ControllerRevisionList": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.ControllerRevisionList instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.ControllerRevisionList" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.Deployment": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.Deployment instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.Deployment" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentCondition": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.DeploymentCondition instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.DeploymentCondition" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentList": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.DeploymentList instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.DeploymentList" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentRollback": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.DeploymentRollback instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.DeploymentRollback" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentSpec": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.DeploymentSpec instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.DeploymentSpec" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentStatus": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.DeploymentStatus instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.DeploymentStatus" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentStrategy": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.DeploymentStrategy instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.DeploymentStrategy" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.RollbackConfig": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.RollbackConfig instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.RollbackConfig" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.RollingUpdateDeployment": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.RollingUpdateDeployment instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.RollingUpdateDeployment" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.RollingUpdateStatefulSetStrategy": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.RollingUpdateStatefulSetStrategy instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.RollingUpdateStatefulSetStrategy" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.Scale": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.Scale instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.Scale" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.ScaleSpec": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.ScaleSpec instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.ScaleSpec" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.ScaleStatus": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.ScaleStatus instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.ScaleStatus" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSet": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.StatefulSet instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.StatefulSet" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSetList": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.StatefulSetList instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.StatefulSetList" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSetSpec": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.StatefulSetSpec instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.StatefulSetSpec" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSetStatus": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.StatefulSetStatus instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.StatefulSetStatus" + }, + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSetUpdateStrategy": { + "description": "Deprecated. Please use io.k8s.api.apps.v1beta1.StatefulSetUpdateStrategy instead.", + "$ref": "#/definitions/io.k8s.api.apps.v1beta1.StatefulSetUpdateStrategy" + }, + "io.k8s.kubernetes.pkg.apis.authentication.v1.TokenReview": { + "description": "Deprecated. Please use io.k8s.api.authentication.v1.TokenReview instead.", + "$ref": "#/definitions/io.k8s.api.authentication.v1.TokenReview" + }, + "io.k8s.kubernetes.pkg.apis.authentication.v1.TokenReviewSpec": { + "description": "Deprecated. Please use io.k8s.api.authentication.v1.TokenReviewSpec instead.", + "$ref": "#/definitions/io.k8s.api.authentication.v1.TokenReviewSpec" + }, + "io.k8s.kubernetes.pkg.apis.authentication.v1.TokenReviewStatus": { + "description": "Deprecated. Please use io.k8s.api.authentication.v1.TokenReviewStatus instead.", + "$ref": "#/definitions/io.k8s.api.authentication.v1.TokenReviewStatus" + }, + "io.k8s.kubernetes.pkg.apis.authentication.v1.UserInfo": { + "description": "Deprecated. Please use io.k8s.api.authentication.v1.UserInfo instead.", + "$ref": "#/definitions/io.k8s.api.authentication.v1.UserInfo" + }, + "io.k8s.kubernetes.pkg.apis.authentication.v1beta1.TokenReview": { + "description": "Deprecated. Please use io.k8s.api.authentication.v1beta1.TokenReview instead.", + "$ref": "#/definitions/io.k8s.api.authentication.v1beta1.TokenReview" + }, + "io.k8s.kubernetes.pkg.apis.authentication.v1beta1.TokenReviewSpec": { + "description": "Deprecated. Please use io.k8s.api.authentication.v1beta1.TokenReviewSpec instead.", + "$ref": "#/definitions/io.k8s.api.authentication.v1beta1.TokenReviewSpec" + }, + "io.k8s.kubernetes.pkg.apis.authentication.v1beta1.TokenReviewStatus": { + "description": "Deprecated. Please use io.k8s.api.authentication.v1beta1.TokenReviewStatus instead.", + "$ref": "#/definitions/io.k8s.api.authentication.v1beta1.TokenReviewStatus" + }, + "io.k8s.kubernetes.pkg.apis.authentication.v1beta1.UserInfo": { + "description": "Deprecated. Please use io.k8s.api.authentication.v1beta1.UserInfo instead.", + "$ref": "#/definitions/io.k8s.api.authentication.v1beta1.UserInfo" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1.LocalSubjectAccessReview": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1.LocalSubjectAccessReview instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1.LocalSubjectAccessReview" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1.NonResourceAttributes": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1.NonResourceAttributes instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1.NonResourceAttributes" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1.ResourceAttributes": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1.ResourceAttributes instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1.ResourceAttributes" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1.SelfSubjectAccessReview": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1.SelfSubjectAccessReview instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1.SelfSubjectAccessReview" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1.SelfSubjectAccessReviewSpec": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1.SelfSubjectAccessReviewSpec instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1.SelfSubjectAccessReviewSpec" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1.SubjectAccessReview": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1.SubjectAccessReview instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1.SubjectAccessReview" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1.SubjectAccessReviewSpec": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1.SubjectAccessReviewSpec instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1.SubjectAccessReviewSpec" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1.SubjectAccessReviewStatus": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1.SubjectAccessReviewStatus instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1.SubjectAccessReviewStatus" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.LocalSubjectAccessReview": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1beta1.LocalSubjectAccessReview instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1beta1.LocalSubjectAccessReview" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.NonResourceAttributes": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1beta1.NonResourceAttributes instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1beta1.NonResourceAttributes" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.ResourceAttributes": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1beta1.ResourceAttributes instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1beta1.ResourceAttributes" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SelfSubjectAccessReview": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1beta1.SelfSubjectAccessReview instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1beta1.SelfSubjectAccessReview" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SelfSubjectAccessReviewSpec": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1beta1.SelfSubjectAccessReviewSpec instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1beta1.SelfSubjectAccessReviewSpec" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SubjectAccessReview": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1beta1.SubjectAccessReview instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1beta1.SubjectAccessReview" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SubjectAccessReviewSpec": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1beta1.SubjectAccessReviewSpec instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1beta1.SubjectAccessReviewSpec" + }, + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SubjectAccessReviewStatus": { + "description": "Deprecated. Please use io.k8s.api.authorization.v1beta1.SubjectAccessReviewStatus instead.", + "$ref": "#/definitions/io.k8s.api.authorization.v1beta1.SubjectAccessReviewStatus" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.CrossVersionObjectReference": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v1.CrossVersionObjectReference instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v1.CrossVersionObjectReference" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.HorizontalPodAutoscaler": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v1.HorizontalPodAutoscaler instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v1.HorizontalPodAutoscaler" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.HorizontalPodAutoscalerList": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v1.HorizontalPodAutoscalerList instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v1.HorizontalPodAutoscalerList" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.HorizontalPodAutoscalerSpec": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v1.HorizontalPodAutoscalerSpec instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v1.HorizontalPodAutoscalerSpec" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.HorizontalPodAutoscalerStatus": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v1.HorizontalPodAutoscalerStatus instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v1.HorizontalPodAutoscalerStatus" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.Scale": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v1.Scale instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v1.Scale" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.ScaleSpec": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v1.ScaleSpec instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v1.ScaleSpec" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.ScaleStatus": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v1.ScaleStatus instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v1.ScaleStatus" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.CrossVersionObjectReference": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.CrossVersionObjectReference instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.CrossVersionObjectReference" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscaler": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscaler instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscaler" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscalerCondition": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerCondition instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerCondition" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscalerList": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerList instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerList" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscalerSpec": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerSpec instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerSpec" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscalerStatus": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerStatus instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerStatus" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.MetricSpec": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.MetricSpec instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.MetricSpec" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.MetricStatus": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.MetricStatus instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.MetricStatus" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.ObjectMetricSource": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.ObjectMetricSource instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.ObjectMetricSource" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.ObjectMetricStatus": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.ObjectMetricStatus instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.ObjectMetricStatus" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.PodsMetricSource": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.PodsMetricSource instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.PodsMetricSource" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.PodsMetricStatus": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.PodsMetricStatus instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.PodsMetricStatus" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.ResourceMetricSource": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.ResourceMetricSource instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.ResourceMetricSource" + }, + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.ResourceMetricStatus": { + "description": "Deprecated. Please use io.k8s.api.autoscaling.v2alpha1.ResourceMetricStatus instead.", + "$ref": "#/definitions/io.k8s.api.autoscaling.v2alpha1.ResourceMetricStatus" + }, + "io.k8s.kubernetes.pkg.apis.batch.v1.Job": { + "description": "Deprecated. Please use io.k8s.api.batch.v1.Job instead.", + "$ref": "#/definitions/io.k8s.api.batch.v1.Job" + }, + "io.k8s.kubernetes.pkg.apis.batch.v1.JobCondition": { + "description": "Deprecated. Please use io.k8s.api.batch.v1.JobCondition instead.", + "$ref": "#/definitions/io.k8s.api.batch.v1.JobCondition" + }, + "io.k8s.kubernetes.pkg.apis.batch.v1.JobList": { + "description": "Deprecated. Please use io.k8s.api.batch.v1.JobList instead.", + "$ref": "#/definitions/io.k8s.api.batch.v1.JobList" + }, + "io.k8s.kubernetes.pkg.apis.batch.v1.JobSpec": { + "description": "Deprecated. Please use io.k8s.api.batch.v1.JobSpec instead.", + "$ref": "#/definitions/io.k8s.api.batch.v1.JobSpec" + }, + "io.k8s.kubernetes.pkg.apis.batch.v1.JobStatus": { + "description": "Deprecated. Please use io.k8s.api.batch.v1.JobStatus instead.", + "$ref": "#/definitions/io.k8s.api.batch.v1.JobStatus" + }, + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.CronJob": { + "description": "Deprecated. Please use io.k8s.api.batch.v2alpha1.CronJob instead.", + "$ref": "#/definitions/io.k8s.api.batch.v2alpha1.CronJob" + }, + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.CronJobList": { + "description": "Deprecated. Please use io.k8s.api.batch.v2alpha1.CronJobList instead.", + "$ref": "#/definitions/io.k8s.api.batch.v2alpha1.CronJobList" + }, + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.CronJobSpec": { + "description": "Deprecated. Please use io.k8s.api.batch.v2alpha1.CronJobSpec instead.", + "$ref": "#/definitions/io.k8s.api.batch.v2alpha1.CronJobSpec" + }, + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.CronJobStatus": { + "description": "Deprecated. Please use io.k8s.api.batch.v2alpha1.CronJobStatus instead.", + "$ref": "#/definitions/io.k8s.api.batch.v2alpha1.CronJobStatus" + }, + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.JobTemplateSpec": { + "description": "Deprecated. Please use io.k8s.api.batch.v2alpha1.JobTemplateSpec instead.", + "$ref": "#/definitions/io.k8s.api.batch.v2alpha1.JobTemplateSpec" + }, + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequest": { + "description": "Deprecated. Please use io.k8s.api.certificates.v1beta1.CertificateSigningRequest instead.", + "$ref": "#/definitions/io.k8s.api.certificates.v1beta1.CertificateSigningRequest" + }, + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequestCondition": { + "description": "Deprecated. Please use io.k8s.api.certificates.v1beta1.CertificateSigningRequestCondition instead.", + "$ref": "#/definitions/io.k8s.api.certificates.v1beta1.CertificateSigningRequestCondition" + }, + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequestList": { + "description": "Deprecated. Please use io.k8s.api.certificates.v1beta1.CertificateSigningRequestList instead.", + "$ref": "#/definitions/io.k8s.api.certificates.v1beta1.CertificateSigningRequestList" + }, + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequestSpec": { + "description": "Deprecated. Please use io.k8s.api.certificates.v1beta1.CertificateSigningRequestSpec instead.", + "$ref": "#/definitions/io.k8s.api.certificates.v1beta1.CertificateSigningRequestSpec" + }, + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequestStatus": { + "description": "Deprecated. Please use io.k8s.api.certificates.v1beta1.CertificateSigningRequestStatus instead.", + "$ref": "#/definitions/io.k8s.api.certificates.v1beta1.CertificateSigningRequestStatus" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSet": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DaemonSet instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DaemonSet" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSetList": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DaemonSetList instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DaemonSetList" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSetSpec": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DaemonSetSpec instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DaemonSetSpec" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSetStatus": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DaemonSetStatus instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DaemonSetStatus" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSetUpdateStrategy": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DaemonSetUpdateStrategy instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DaemonSetUpdateStrategy" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.Deployment": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.Deployment instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.Deployment" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentCondition": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DeploymentCondition instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DeploymentCondition" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentList": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DeploymentList instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DeploymentList" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentRollback": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DeploymentRollback instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DeploymentRollback" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentSpec": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DeploymentSpec instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DeploymentSpec" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentStatus": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DeploymentStatus instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DeploymentStatus" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentStrategy": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.DeploymentStrategy instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DeploymentStrategy" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.FSGroupStrategyOptions": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.FSGroupStrategyOptions instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.FSGroupStrategyOptions" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.HTTPIngressPath": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.HTTPIngressPath instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.HTTPIngressPath" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.HTTPIngressRuleValue": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.HTTPIngressRuleValue instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.HTTPIngressRuleValue" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.HostPortRange": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.HostPortRange instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.HostPortRange" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IDRange": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.IDRange instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.IDRange" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.Ingress": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.Ingress instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.Ingress" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressBackend": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.IngressBackend instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.IngressBackend" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressList": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.IngressList instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.IngressList" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressRule": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.IngressRule instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.IngressRule" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressSpec": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.IngressSpec instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.IngressSpec" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressStatus": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.IngressStatus instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.IngressStatus" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressTLS": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.IngressTLS instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.IngressTLS" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicy": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.NetworkPolicy instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.NetworkPolicy" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicyIngressRule": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.NetworkPolicyIngressRule instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.NetworkPolicyIngressRule" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicyList": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.NetworkPolicyList instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.NetworkPolicyList" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicyPeer": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.NetworkPolicyPeer instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.NetworkPolicyPeer" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicyPort": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.NetworkPolicyPort instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.NetworkPolicyPort" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicySpec": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.NetworkPolicySpec instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.NetworkPolicySpec" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.PodSecurityPolicy": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.PodSecurityPolicy instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.PodSecurityPolicy" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.PodSecurityPolicyList": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.PodSecurityPolicyList instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.PodSecurityPolicyList" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.PodSecurityPolicySpec": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.PodSecurityPolicySpec instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.PodSecurityPolicySpec" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSet": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.ReplicaSet instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.ReplicaSet" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSetCondition": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.ReplicaSetCondition instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.ReplicaSetCondition" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSetList": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.ReplicaSetList instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.ReplicaSetList" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSetSpec": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.ReplicaSetSpec instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.ReplicaSetSpec" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSetStatus": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.ReplicaSetStatus instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.ReplicaSetStatus" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.RollbackConfig": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.RollbackConfig instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.RollbackConfig" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.RollingUpdateDaemonSet": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.RollingUpdateDaemonSet instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.RollingUpdateDaemonSet" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.RollingUpdateDeployment": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.RollingUpdateDeployment instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.RollingUpdateDeployment" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.RunAsUserStrategyOptions": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.RunAsUserStrategyOptions instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.RunAsUserStrategyOptions" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.SELinuxStrategyOptions": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.SELinuxStrategyOptions instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.SELinuxStrategyOptions" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.Scale": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.Scale instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.Scale" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ScaleSpec": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.ScaleSpec instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.ScaleSpec" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ScaleStatus": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.ScaleStatus instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.ScaleStatus" + }, + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.SupplementalGroupsStrategyOptions": { + "description": "Deprecated. Please use io.k8s.api.extensions.v1beta1.SupplementalGroupsStrategyOptions instead.", + "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.SupplementalGroupsStrategyOptions" + }, + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicy": { + "description": "Deprecated. Please use io.k8s.api.networking.v1.NetworkPolicy instead.", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicy" + }, + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicyIngressRule": { + "description": "Deprecated. Please use io.k8s.api.networking.v1.NetworkPolicyIngressRule instead.", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyIngressRule" + }, + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicyList": { + "description": "Deprecated. Please use io.k8s.api.networking.v1.NetworkPolicyList instead.", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyList" + }, + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicyPeer": { + "description": "Deprecated. Please use io.k8s.api.networking.v1.NetworkPolicyPeer instead.", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + }, + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicyPort": { + "description": "Deprecated. Please use io.k8s.api.networking.v1.NetworkPolicyPort instead.", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicyPort" + }, + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicySpec": { + "description": "Deprecated. Please use io.k8s.api.networking.v1.NetworkPolicySpec instead.", + "$ref": "#/definitions/io.k8s.api.networking.v1.NetworkPolicySpec" + }, + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.Eviction": { + "description": "Deprecated. Please use io.k8s.api.policy.v1beta1.Eviction instead.", + "$ref": "#/definitions/io.k8s.api.policy.v1beta1.Eviction" + }, + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.PodDisruptionBudget": { + "description": "Deprecated. Please use io.k8s.api.policy.v1beta1.PodDisruptionBudget instead.", + "$ref": "#/definitions/io.k8s.api.policy.v1beta1.PodDisruptionBudget" + }, + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.PodDisruptionBudgetList": { + "description": "Deprecated. Please use io.k8s.api.policy.v1beta1.PodDisruptionBudgetList instead.", + "$ref": "#/definitions/io.k8s.api.policy.v1beta1.PodDisruptionBudgetList" + }, + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.PodDisruptionBudgetSpec": { + "description": "Deprecated. Please use io.k8s.api.policy.v1beta1.PodDisruptionBudgetSpec instead.", + "$ref": "#/definitions/io.k8s.api.policy.v1beta1.PodDisruptionBudgetSpec" + }, + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.PodDisruptionBudgetStatus": { + "description": "Deprecated. Please use io.k8s.api.policy.v1beta1.PodDisruptionBudgetStatus instead.", + "$ref": "#/definitions/io.k8s.api.policy.v1beta1.PodDisruptionBudgetStatus" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.ClusterRole": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.ClusterRole instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.ClusterRole" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.ClusterRoleBinding": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.ClusterRoleBinding instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.ClusterRoleBinding" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.ClusterRoleBindingList": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.ClusterRoleBindingList instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.ClusterRoleBindingList" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.ClusterRoleList": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.ClusterRoleList instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.ClusterRoleList" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.PolicyRule": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.PolicyRule instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.PolicyRule" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.Role": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.Role instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.Role" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.RoleBinding": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.RoleBinding instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.RoleBinding" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.RoleBindingList": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.RoleBindingList instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.RoleBindingList" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.RoleList": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.RoleList instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.RoleList" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.RoleRef": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.RoleRef instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.RoleRef" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.Subject": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1alpha1.Subject instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1alpha1.Subject" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.ClusterRole": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.ClusterRole instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.ClusterRole" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.ClusterRoleBinding": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.ClusterRoleBinding instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.ClusterRoleBinding" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.ClusterRoleBindingList": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.ClusterRoleBindingList instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.ClusterRoleBindingList" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.ClusterRoleList": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.ClusterRoleList instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.ClusterRoleList" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.PolicyRule": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.PolicyRule instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.PolicyRule" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.Role": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.Role instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.Role" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.RoleBinding": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.RoleBinding instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.RoleBinding" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.RoleBindingList": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.RoleBindingList instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.RoleBindingList" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.RoleList": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.RoleList instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.RoleList" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.RoleRef": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.RoleRef instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.RoleRef" + }, + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.Subject": { + "description": "Deprecated. Please use io.k8s.api.rbac.v1beta1.Subject instead.", + "$ref": "#/definitions/io.k8s.api.rbac.v1beta1.Subject" + }, + "io.k8s.kubernetes.pkg.apis.settings.v1alpha1.PodPreset": { + "description": "Deprecated. Please use io.k8s.api.settings.v1alpha1.PodPreset instead.", + "$ref": "#/definitions/io.k8s.api.settings.v1alpha1.PodPreset" + }, + "io.k8s.kubernetes.pkg.apis.settings.v1alpha1.PodPresetList": { + "description": "Deprecated. Please use io.k8s.api.settings.v1alpha1.PodPresetList instead.", + "$ref": "#/definitions/io.k8s.api.settings.v1alpha1.PodPresetList" + }, + "io.k8s.kubernetes.pkg.apis.settings.v1alpha1.PodPresetSpec": { + "description": "Deprecated. Please use io.k8s.api.settings.v1alpha1.PodPresetSpec instead.", + "$ref": "#/definitions/io.k8s.api.settings.v1alpha1.PodPresetSpec" + }, + "io.k8s.kubernetes.pkg.apis.storage.v1.StorageClass": { + "description": "Deprecated. Please use io.k8s.api.storage.v1.StorageClass instead.", + "$ref": "#/definitions/io.k8s.api.storage.v1.StorageClass" + }, + "io.k8s.kubernetes.pkg.apis.storage.v1.StorageClassList": { + "description": "Deprecated. Please use io.k8s.api.storage.v1.StorageClassList instead.", + "$ref": "#/definitions/io.k8s.api.storage.v1.StorageClassList" + }, + "io.k8s.kubernetes.pkg.apis.storage.v1beta1.StorageClass": { + "description": "Deprecated. Please use io.k8s.api.storage.v1beta1.StorageClass instead.", + "$ref": "#/definitions/io.k8s.api.storage.v1beta1.StorageClass" + }, + "io.k8s.kubernetes.pkg.apis.storage.v1beta1.StorageClassList": { + "description": "Deprecated. Please use io.k8s.api.storage.v1beta1.StorageClassList instead.", + "$ref": "#/definitions/io.k8s.api.storage.v1beta1.StorageClassList" } }, "securityDefinitions": { diff --git a/api/swagger-spec/admissionregistration.k8s.io_v1alpha1.json b/api/swagger-spec/admissionregistration.k8s.io_v1alpha1.json index f6d28b231b0e2..f80aae262e7a3 100644 --- a/api/swagger-spec/admissionregistration.k8s.io_v1alpha1.json +++ b/api/swagger-spec/admissionregistration.k8s.io_v1alpha1.json @@ -1814,10 +1814,6 @@ "$ref": "v1alpha1.Rule" }, "description": "Rules describes what resources/subresources the initializer cares about. The initializer cares about an operation if it matches _any_ Rule. Rule.Resources must not include subresources." - }, - "failurePolicy": { - "$ref": "v1alpha1.FailurePolicyType", - "description": "FailurePolicy defines what happens if the responsible initializer controller fails to takes action. Allowed values are Ignore, or Fail. If \"Ignore\" is set, initializer is removed from the initializers list of an object if the timeout is reached; If \"Fail\" is set, admissionregistration returns timeout error if the timeout is reached." } } }, diff --git a/api/swagger-spec/apps_v1beta1.json b/api/swagger-spec/apps_v1beta1.json index 1f99ebc6bfac7..bde4c4fe8ee06 100644 --- a/api/swagger-spec/apps_v1beta1.json +++ b/api/swagger-spec/apps_v1beta1.json @@ -4135,10 +4135,18 @@ "properties": { "path": { "type": "string", - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "type": { + "$ref": "v1.HostPathType", + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" } } }, + "v1.HostPathType": { + "id": "v1.HostPathType", + "properties": {} + }, "v1.EmptyDirVolumeSource": { "id": "v1.EmptyDirVolumeSource", "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", @@ -4349,6 +4357,10 @@ "secretRef": { "$ref": "v1.LocalObjectReference", "description": "CHAP secret for iSCSI target and initiator authentication" + }, + "initiatorName": { + "type": "string", + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection." } } }, @@ -5231,7 +5243,7 @@ }, "resourceFieldRef": { "$ref": "v1.ResourceFieldSelector", - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." }, "configMapKeyRef": { "$ref": "v1.ConfigMapKeySelector", @@ -5909,7 +5921,7 @@ }, "collisionCount": { "type": "integer", - "format": "int64", + "format": "int32", "description": "Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet." } } @@ -6282,7 +6294,7 @@ }, "collisionCount": { "type": "integer", - "format": "int64", + "format": "int32", "description": "collisionCount is the count of hash collisions for the StatefulSet. The StatefulSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision." } } diff --git a/api/swagger-spec/apps_v1beta2.json b/api/swagger-spec/apps_v1beta2.json index 07913f99aca54..a83f6a8300fef 100644 --- a/api/swagger-spec/apps_v1beta2.json +++ b/api/swagger-spec/apps_v1beta2.json @@ -6273,10 +6273,18 @@ "properties": { "path": { "type": "string", - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "type": { + "$ref": "v1.HostPathType", + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" } } }, + "v1.HostPathType": { + "id": "v1.HostPathType", + "properties": {} + }, "v1.EmptyDirVolumeSource": { "id": "v1.EmptyDirVolumeSource", "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", @@ -6487,6 +6495,10 @@ "secretRef": { "$ref": "v1.LocalObjectReference", "description": "CHAP secret for iSCSI target and initiator authentication" + }, + "initiatorName": { + "type": "string", + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection." } } }, @@ -7369,7 +7381,7 @@ }, "resourceFieldRef": { "$ref": "v1.ResourceFieldSelector", - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." }, "configMapKeyRef": { "$ref": "v1.ConfigMapKeySelector", @@ -8041,7 +8053,7 @@ }, "collisionCount": { "type": "integer", - "format": "int64", + "format": "int32", "description": "Count of hash collisions for the DaemonSet. The DaemonSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision." } } @@ -8216,7 +8228,7 @@ }, "collisionCount": { "type": "integer", - "format": "int64", + "format": "int32", "description": "Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet." } } @@ -8708,7 +8720,7 @@ }, "collisionCount": { "type": "integer", - "format": "int64", + "format": "int32", "description": "collisionCount is the count of hash collisions for the StatefulSet. The StatefulSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision." } } diff --git a/api/swagger-spec/batch_v1.json b/api/swagger-spec/batch_v1.json index 0b4e34e902a1f..043746825a3b1 100644 --- a/api/swagger-spec/batch_v1.json +++ b/api/swagger-spec/batch_v1.json @@ -1717,10 +1717,18 @@ "properties": { "path": { "type": "string", - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "type": { + "$ref": "v1.HostPathType", + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" } } }, + "v1.HostPathType": { + "id": "v1.HostPathType", + "properties": {} + }, "v1.EmptyDirVolumeSource": { "id": "v1.EmptyDirVolumeSource", "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", @@ -1931,6 +1939,10 @@ "secretRef": { "$ref": "v1.LocalObjectReference", "description": "CHAP secret for iSCSI target and initiator authentication" + }, + "initiatorName": { + "type": "string", + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection." } } }, @@ -2813,7 +2825,7 @@ }, "resourceFieldRef": { "$ref": "v1.ResourceFieldSelector", - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." }, "configMapKeyRef": { "$ref": "v1.ConfigMapKeySelector", diff --git a/api/swagger-spec/batch_v1beta1.json b/api/swagger-spec/batch_v1beta1.json index dc10d006f9802..d6ff984f319bc 100644 --- a/api/swagger-spec/batch_v1beta1.json +++ b/api/swagger-spec/batch_v1beta1.json @@ -1772,10 +1772,18 @@ "properties": { "path": { "type": "string", - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "type": { + "$ref": "v1.HostPathType", + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" } } }, + "v1.HostPathType": { + "id": "v1.HostPathType", + "properties": {} + }, "v1.EmptyDirVolumeSource": { "id": "v1.EmptyDirVolumeSource", "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", @@ -1986,6 +1994,10 @@ "secretRef": { "$ref": "v1.LocalObjectReference", "description": "CHAP secret for iSCSI target and initiator authentication" + }, + "initiatorName": { + "type": "string", + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection." } } }, @@ -2868,7 +2880,7 @@ }, "resourceFieldRef": { "$ref": "v1.ResourceFieldSelector", - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." }, "configMapKeyRef": { "$ref": "v1.ConfigMapKeySelector", diff --git a/api/swagger-spec/batch_v2alpha1.json b/api/swagger-spec/batch_v2alpha1.json index 2122d7df850b5..fde83a089d051 100644 --- a/api/swagger-spec/batch_v2alpha1.json +++ b/api/swagger-spec/batch_v2alpha1.json @@ -1772,10 +1772,18 @@ "properties": { "path": { "type": "string", - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "type": { + "$ref": "v1.HostPathType", + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" } } }, + "v1.HostPathType": { + "id": "v1.HostPathType", + "properties": {} + }, "v1.EmptyDirVolumeSource": { "id": "v1.EmptyDirVolumeSource", "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", @@ -1986,6 +1994,10 @@ "secretRef": { "$ref": "v1.LocalObjectReference", "description": "CHAP secret for iSCSI target and initiator authentication" + }, + "initiatorName": { + "type": "string", + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection." } } }, @@ -2868,7 +2880,7 @@ }, "resourceFieldRef": { "$ref": "v1.ResourceFieldSelector", - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." }, "configMapKeyRef": { "$ref": "v1.ConfigMapKeySelector", diff --git a/api/swagger-spec/extensions_v1beta1.json b/api/swagger-spec/extensions_v1beta1.json index 28fbd8d968b61..bc47d4d140279 100644 --- a/api/swagger-spec/extensions_v1beta1.json +++ b/api/swagger-spec/extensions_v1beta1.json @@ -6827,10 +6827,18 @@ "properties": { "path": { "type": "string", - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "type": { + "$ref": "v1.HostPathType", + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" } } }, + "v1.HostPathType": { + "id": "v1.HostPathType", + "properties": {} + }, "v1.EmptyDirVolumeSource": { "id": "v1.EmptyDirVolumeSource", "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", @@ -7041,6 +7049,10 @@ "secretRef": { "$ref": "v1.LocalObjectReference", "description": "CHAP secret for iSCSI target and initiator authentication" + }, + "initiatorName": { + "type": "string", + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection." } } }, @@ -7923,7 +7935,7 @@ }, "resourceFieldRef": { "$ref": "v1.ResourceFieldSelector", - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." }, "configMapKeyRef": { "$ref": "v1.ConfigMapKeySelector", @@ -8594,7 +8606,7 @@ }, "collisionCount": { "type": "integer", - "format": "int64", + "format": "int32", "description": "Count of hash collisions for the DaemonSet. The DaemonSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision." } } @@ -8853,7 +8865,7 @@ }, "collisionCount": { "type": "integer", - "format": "int64", + "format": "int32", "description": "Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet." } } @@ -9293,6 +9305,30 @@ "namespaceSelector": { "$ref": "v1.LabelSelector", "description": "Selects Namespaces using cluster scoped-labels. This matches all pods in all namespaces selected by this label selector. This field follows standard label selector semantics. If present but empty, this selector selects all namespaces." + }, + "ipBlock": { + "$ref": "v1beta1.IPBlock", + "description": "IPBlock defines policy on a particular IPBlock" + } + } + }, + "v1beta1.IPBlock": { + "id": "v1beta1.IPBlock", + "description": "IPBlock describes a particular CIDR (Ex. \"192.168.1.1/24\") that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should not be included within this rule.", + "required": [ + "cidr" + ], + "properties": { + "cidr": { + "type": "string", + "description": "CIDR is a string representing the IP Block Valid examples are \"192.168.1.1/24\"" + }, + "except": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are \"192.168.1.1/24\" Except values will be rejected if they are outside the CIDR range" } } }, diff --git a/api/swagger-spec/networking.k8s.io_v1.json b/api/swagger-spec/networking.k8s.io_v1.json index 79a9ac80cb39b..059a1c639640e 100644 --- a/api/swagger-spec/networking.k8s.io_v1.json +++ b/api/swagger-spec/networking.k8s.io_v1.json @@ -1307,6 +1307,30 @@ "namespaceSelector": { "$ref": "v1.LabelSelector", "description": "Selects Namespaces using cluster scoped-labels. This matches all pods in all namespaces selected by this label selector. This field follows standard label selector semantics. If present but empty, this selector selects all namespaces." + }, + "ipBlock": { + "$ref": "v1.IPBlock", + "description": "IPBlock defines policy on a particular IPBlock" + } + } + }, + "v1.IPBlock": { + "id": "v1.IPBlock", + "description": "IPBlock describes a particular CIDR (Ex. \"192.168.1.1/24\") that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should not be included within this rule.", + "required": [ + "cidr" + ], + "properties": { + "cidr": { + "type": "string", + "description": "CIDR is a string representing the IP Block Valid examples are \"192.168.1.1/24\"" + }, + "except": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are \"192.168.1.1/24\" Except values will be rejected if they are outside the CIDR range" } } }, diff --git a/api/swagger-spec/settings.k8s.io_v1alpha1.json b/api/swagger-spec/settings.k8s.io_v1alpha1.json index 168d97d5434dd..2e3d3b71a30af 100644 --- a/api/swagger-spec/settings.k8s.io_v1alpha1.json +++ b/api/swagger-spec/settings.k8s.io_v1alpha1.json @@ -1305,7 +1305,7 @@ }, "resourceFieldRef": { "$ref": "v1.ResourceFieldSelector", - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." }, "configMapKeyRef": { "$ref": "v1.ConfigMapKeySelector", @@ -1573,10 +1573,18 @@ "properties": { "path": { "type": "string", - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "type": { + "$ref": "v1.HostPathType", + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" } } }, + "v1.HostPathType": { + "id": "v1.HostPathType", + "properties": {} + }, "v1.EmptyDirVolumeSource": { "id": "v1.EmptyDirVolumeSource", "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", @@ -1787,6 +1795,10 @@ "secretRef": { "$ref": "v1.LocalObjectReference", "description": "CHAP secret for iSCSI target and initiator authentication" + }, + "initiatorName": { + "type": "string", + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection." } } }, diff --git a/api/swagger-spec/v1.json b/api/swagger-spec/v1.json index a61f35380794b..6e0ced03e6d85 100644 --- a/api/swagger-spec/v1.json +++ b/api/swagger-spec/v1.json @@ -18865,7 +18865,7 @@ "description": "Cinder represents a cinder volume attached and mounted on kubelets host machine More info: https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md" }, "cephfs": { - "$ref": "v1.CephFSVolumeSource", + "$ref": "v1.CephFSPersistentVolumeSource", "description": "CephFS represents a Ceph FS mount on the host that shares a pod's lifetime" }, "fc": { @@ -18881,7 +18881,7 @@ "description": "FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin. This is an alpha feature and may change in future." }, "azureFile": { - "$ref": "v1.AzureFileVolumeSource", + "$ref": "v1.AzureFilePersistentVolumeSource", "description": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod." }, "vsphereVolume": { @@ -18934,6 +18934,13 @@ "storageClassName": { "type": "string", "description": "Name of StorageClass to which this persistent volume belongs. Empty value means that this volume does not belong to any StorageClass." + }, + "mountOptions": { + "type": "array", + "items": { + "type": "string" + }, + "description": "A list of mount options, e.g. [\"ro\", \"soft\"]. Not validated - mount will simply fail if one is invalid. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options" } } }, @@ -18998,10 +19005,18 @@ "properties": { "path": { "type": "string", - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "type": { + "$ref": "v1.HostPathType", + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" } } }, + "v1.HostPathType": { + "id": "v1.HostPathType", + "properties": {} + }, "v1.GlusterfsVolumeSource": { "id": "v1.GlusterfsVolumeSource", "description": "Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.", @@ -19153,6 +19168,10 @@ "secretRef": { "$ref": "v1.LocalObjectReference", "description": "CHAP secret for iSCSI target and initiator authentication" + }, + "initiatorName": { + "type": "string", + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection." } } }, @@ -19177,8 +19196,8 @@ } } }, - "v1.CephFSVolumeSource": { - "id": "v1.CephFSVolumeSource", + "v1.CephFSPersistentVolumeSource": { + "id": "v1.CephFSPersistentVolumeSource", "description": "Represents a Ceph Filesystem mount that lasts the lifetime of a pod Cephfs volumes do not support ownership management or SELinux relabeling.", "required": [ "monitors" @@ -19204,7 +19223,7 @@ "description": "Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it" }, "secretRef": { - "$ref": "v1.LocalObjectReference", + "$ref": "v1.SecretReference", "description": "Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it" }, "readOnly": { @@ -19213,6 +19232,20 @@ } } }, + "v1.SecretReference": { + "id": "v1.SecretReference", + "description": "SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace", + "properties": { + "name": { + "type": "string", + "description": "Name is unique within a namespace to reference a secret resource." + }, + "namespace": { + "type": "string", + "description": "Namespace defines the space within which the secret name must be unique." + } + } + }, "v1.FCVolumeSource": { "id": "v1.FCVolumeSource", "description": "Represents a Fibre Channel volume. Fibre Channel volumes can only be mounted as read/write once. Fibre Channel volumes support ownership management and SELinux relabeling.", @@ -19289,12 +19322,13 @@ } } }, - "v1.AzureFileVolumeSource": { - "id": "v1.AzureFileVolumeSource", + "v1.AzureFilePersistentVolumeSource": { + "id": "v1.AzureFilePersistentVolumeSource", "description": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", "required": [ "secretName", - "shareName" + "shareName", + "secretNamespace" ], "properties": { "secretName": { @@ -19308,6 +19342,10 @@ "readOnly": { "type": "boolean", "description": "Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts." + }, + "secretNamespace": { + "type": "string", + "description": "the namespace of the secret that contains Azure Storage Account Name and Key default is the same as the Pod" } } }, @@ -19960,6 +19998,42 @@ } } }, + "v1.CephFSVolumeSource": { + "id": "v1.CephFSVolumeSource", + "description": "Represents a Ceph Filesystem mount that lasts the lifetime of a pod Cephfs volumes do not support ownership management or SELinux relabeling.", + "required": [ + "monitors" + ], + "properties": { + "monitors": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Required: Monitors is a collection of Ceph monitors More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it" + }, + "path": { + "type": "string", + "description": "Optional: Used as the mounted root, rather than the full Ceph tree, default is /" + }, + "user": { + "type": "string", + "description": "Optional: User is the rados user name, default is admin More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it" + }, + "secretFile": { + "type": "string", + "description": "Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it" + }, + "secretRef": { + "$ref": "v1.LocalObjectReference", + "description": "Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it" + }, + "readOnly": { + "type": "boolean", + "description": "Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it" + } + } + }, "v1.DownwardAPIVolumeSource": { "id": "v1.DownwardAPIVolumeSource", "description": "DownwardAPIVolumeSource represents a volume containing downward API info. Downward API volumes support ownership management and SELinux relabeling.", @@ -20042,6 +20116,28 @@ } } }, + "v1.AzureFileVolumeSource": { + "id": "v1.AzureFileVolumeSource", + "description": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", + "required": [ + "secretName", + "shareName" + ], + "properties": { + "secretName": { + "type": "string", + "description": "the name of secret that contains Azure Storage Account Name and Key" + }, + "shareName": { + "type": "string", + "description": "Share Name" + }, + "readOnly": { + "type": "boolean", + "description": "Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts." + } + } + }, "v1.ConfigMapVolumeSource": { "id": "v1.ConfigMapVolumeSource", "description": "Adapts a ConfigMap into a volume.\n\nThe contents of the target ConfigMap's Data field will be presented in a volume as files using the keys in the Data field as the file names, unless the items element is populated with specific mappings of keys to paths. ConfigMap volumes support ownership management and SELinux relabeling.", @@ -20404,7 +20500,7 @@ }, "resourceFieldRef": { "$ref": "v1.ResourceFieldSelector", - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." }, "configMapKeyRef": { "$ref": "v1.ConfigMapKeySelector", @@ -21818,6 +21914,10 @@ "publishNotReadyAddresses": { "type": "boolean", "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery. This field will replace the service.alpha.kubernetes.io/tolerate-unready-endpoints when that annotation is deprecated and all clients have been converted to use this field." + }, + "sessionAffinityConfig": { + "$ref": "v1.SessionAffinityConfig", + "description": "sessionAffinityConfig contains the configurations of session affinity." } } }, @@ -21852,6 +21952,27 @@ } } }, + "v1.SessionAffinityConfig": { + "id": "v1.SessionAffinityConfig", + "description": "SessionAffinityConfig represents the configurations of session affinity.", + "properties": { + "clientIP": { + "$ref": "v1.ClientIPConfig", + "description": "clientIP contains the configurations of Client IP based session affinity." + } + } + }, + "v1.ClientIPConfig": { + "id": "v1.ClientIPConfig", + "description": "ClientIPConfig represents the configurations of Client IP based session affinity.", + "properties": { + "timeoutSeconds": { + "type": "integer", + "format": "int32", + "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours)." + } + } + }, "v1.ServiceStatus": { "id": "v1.ServiceStatus", "description": "ServiceStatus represents the current status of a service.", diff --git a/build/BUILD b/build/BUILD index 5f08ab29c54b2..9bed34abc7c10 100644 --- a/build/BUILD +++ b/build/BUILD @@ -25,23 +25,23 @@ filegroup( # in build/common.sh. DOCKERIZED_BINARIES = { "cloud-controller-manager": { - "base": "@official_busybox//image:image.tar", + "base": "@official_busybox//image", "target": "//cmd/cloud-controller-manager:cloud-controller-manager", }, "kube-apiserver": { - "base": "@official_busybox//image:image.tar", + "base": "@official_busybox//image", "target": "//cmd/kube-apiserver:kube-apiserver", }, "kube-controller-manager": { - "base": "@official_busybox//image:image.tar", + "base": "@official_busybox//image", "target": "//cmd/kube-controller-manager:kube-controller-manager", }, "kube-scheduler": { - "base": "@official_busybox//image:image.tar", + "base": "@official_busybox//image", "target": "//plugin/cmd/kube-scheduler:kube-scheduler", }, "kube-proxy": { - "base": "@debian-iptables-amd64//image:image.tar", + "base": "@debian-iptables-amd64//image", "target": "//cmd/kube-proxy:kube-proxy", }, } @@ -137,7 +137,6 @@ filegroup( "//cmd/genyaml", "//cmd/kubemark", # TODO: server platforms only "//cmd/linkcheck", - "//cmd/mungedocs", "//federation/cmd/genfeddocs", "//test/e2e:e2e.test", "//test/e2e_node:e2e_node.test", # TODO: server platforms only diff --git a/build/common.sh b/build/common.sh index 80c6fc7d8a397..2db1fae322668 100755 --- a/build/common.sh +++ b/build/common.sh @@ -654,7 +654,6 @@ function kube::build::stop_rsyncd_container() { function kube::build::rsync { local -a rsync_opts=( --archive - --prune-empty-dirs --password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" ) if (( ${KUBE_VERBOSE} >= 6 )); then @@ -679,16 +678,19 @@ function kube::build::sync_to_container() { # directory and generated files. The '- /' filter prevents rsync # from trying to set the uid/gid/perms on the root of the sync tree. # As an exception, we need to sync generated files in staging/, because - # they will not be re-generated by 'make'. + # they will not be re-generated by 'make'. Note that the 'H' filtered files + # are hidden from rsync so they will be deleted in the target container if + # they exist. This will allow them to be re-created in the container if + # necessary. kube::build::rsync \ --delete \ - --filter='- /.git/' \ + --filter='H /.git/' \ --filter='- /.make/' \ --filter='- /_tmp/' \ --filter='- /_output/' \ --filter='- /' \ - --filter='- zz_generated.*' \ - --filter='- generated.proto' \ + --filter='H zz_generated.*' \ + --filter='H generated.proto' \ "${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" kube::build::stop_rsyncd_container @@ -714,6 +716,7 @@ function kube::build::copy_output() { # We are looking to copy out all of the built binaries along with various # generated files. kube::build::rsync \ + --prune-empty-dirs \ --filter='- /_temp/' \ --filter='+ /vendor/' \ --filter='+ /Godeps/' \ diff --git a/build/debian-hyperkube-base/Makefile b/build/debian-hyperkube-base/Makefile index 3a63ba6ebf211..b6d5183228551 100644 --- a/build/debian-hyperkube-base/Makefile +++ b/build/debian-hyperkube-base/Makefile @@ -51,6 +51,8 @@ endif mkdir -p ${TEMP_DIR}/cni-bin tar -xz -C ${TEMP_DIR}/cni-bin -f "cni-tars/${CNI_TARBALL}" + # Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel + docker run --rm --privileged multiarch/qemu-user-static:register --reset docker build --pull -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR) rm -rf $(TEMP_DIR) diff --git a/build/root/BUILD.root b/build/root/BUILD.root index 56627f257dd76..7411c8cdf53d3 100644 --- a/build/root/BUILD.root +++ b/build/root/BUILD.root @@ -5,7 +5,7 @@ package(default_visibility = ["//visibility:public"]) -load("@io_bazel_rules_go//go:def.bzl", "go_prefix") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_prefix") load("@io_kubernetes_build//defs:build.bzl", "gcs_upload") go_prefix("k8s.io/kubernetes") @@ -80,3 +80,26 @@ genrule( cmd = "grep ^STABLE_BUILD_SCM_REVISION bazel-out/stable-status.txt | awk '{print $$2}' >$@", stamp = 1, ) + +go_library( + name = "go_default_library", + srcs = [ + "generated.pb.go", + "types.go", + ], + deps = [ + "//vendor/github.com/gogo/protobuf/proto:go_default_library", + "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + ], +) + +filegroup( + name = "go_default_library_protos", + srcs = ["generated.proto"], + visibility = ["//visibility:public"], +) diff --git a/build/root/Makefile.generated_files b/build/root/Makefile.generated_files index 08c59d8aa8f09..669230c334067 100644 --- a/build/root/Makefile.generated_files +++ b/build/root/Makefile.generated_files @@ -69,7 +69,7 @@ verify_generated_files: verify_gen_deepcopy \ # # expect one file to be regenerated # make gen_deepcopy # # expect nothing to be rebuilt, finish in O(seconds) -# touch vendor/k8s.io/kube-gen/cmd/deepcopy-gen/main.go +# touch vendor/k8s.io/code-generator/cmd/deepcopy-gen/main.go # make gen_deepcopy # # expect deepcopy-gen is built exactly once # # expect many files to be regenerated @@ -88,7 +88,7 @@ verify_generated_files: verify_gen_deepcopy \ # # expect one file to be regenerated # make gen_conversion # # expect nothing to be rebuilt, finish in O(seconds) -# touch vendor/k8s.io/kube-gen/cmd/conversion-gen/main.go +# touch vendor/k8s.io/code-generator/cmd/conversion-gen/main.go # make gen_conversion # # expect conversion-gen is built exactly once # # expect many files to be regenerated @@ -222,7 +222,7 @@ RUN_GEN_DEEPCOPY = \ --v $(KUBE_VERBOSE) \ --logtostderr \ -i $$(cat $(META_DIR)/$(DEEPCOPY_GEN).todo | paste -sd, -) \ - --bounding-dirs $(PRJ_SRC_PATH),,"k8s.io/api" \ + --bounding-dirs $(PRJ_SRC_PATH),"k8s.io/api" \ -O $(DEEPCOPY_BASENAME) \ "$$@"; \ fi \ @@ -274,7 +274,7 @@ $(META_DIR)/$(DEEPCOPY_GEN).mk: (echo -n "$(DEEPCOPY_GEN): "; \ ./hack/run-in-gopath.sh go list \ -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ - ./vendor/k8s.io/kube-gen/cmd/deepcopy-gen \ + ./vendor/k8s.io/code-generator/cmd/deepcopy-gen \ | grep --color=never "^$(PRJ_SRC_PATH)/" \ | xargs ./hack/run-in-gopath.sh go list \ -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ @@ -299,7 +299,7 @@ sinclude $(META_DIR)/$(DEEPCOPY_GEN).mk # newer than the binary, and try to rebuild it over and over. So we touch it, # and make is happy. $(DEEPCOPY_GEN): - hack/make-rules/build.sh ./vendor/k8s.io/kube-gen/cmd/deepcopy-gen + hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/deepcopy-gen touch $@ # @@ -424,7 +424,7 @@ $(META_DIR)/$(DEFAULTER_GEN).mk: (echo -n "$(DEFAULTER_GEN): "; \ ./hack/run-in-gopath.sh go list \ -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ - ./vendor/k8s.io/kube-gen/cmd/defaulter-gen \ + ./vendor/k8s.io/code-generator/cmd/defaulter-gen \ | grep --color=never "^$(PRJ_SRC_PATH)/" \ | xargs ./hack/run-in-gopath.sh go list \ -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ @@ -449,7 +449,7 @@ sinclude $(META_DIR)/$(DEFAULTER_GEN).mk # newer than the binary, and try to rebuild it over and over. So we touch it, # and make is happy. $(DEFAULTER_GEN): - hack/make-rules/build.sh ./vendor/k8s.io/kube-gen/cmd/defaulter-gen + hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/defaulter-gen touch $@ # @@ -503,15 +503,15 @@ $(foreach dir, $(OPENAPI_DIRS), $(eval \ # How to regenerate open-api code. This emits a single file for all results. $(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(OPENAPI_GEN) function run_gen_openapi() { \ - ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ - --v $(KUBE_VERBOSE) \ - --logtostderr \ - -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g') \ - -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG) \ - -O $(OPENAPI_BASENAME) \ - "$$@"; \ - }; \ - run_gen_openapi + ./hack/run-in-gopath.sh $(OPENAPI_GEN) \ + --v $(KUBE_VERBOSE) \ + --logtostderr \ + -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g') \ + -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG) \ + -O $(OPENAPI_BASENAME) \ + "$$@"; \ + }; \ + run_gen_openapi # This calculates the dependencies for the generator tool, so we only rebuild # it when needed. It is PHONY so that it always runs, but it only updates the @@ -522,7 +522,7 @@ $(META_DIR)/$(OPENAPI_GEN).mk: (echo -n "$(OPENAPI_GEN): "; \ ./hack/run-in-gopath.sh go list \ -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ - ./vendor/k8s.io/kube-gen/cmd/openapi-gen \ + ./vendor/k8s.io/code-generator/cmd/openapi-gen \ | grep --color=never "^$(PRJ_SRC_PATH)/" \ | xargs ./hack/run-in-gopath.sh go list \ -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ @@ -547,7 +547,7 @@ sinclude $(META_DIR)/$(OPENAPI_GEN).mk # newer than the binary, and try to rebuild it over and over. So we touch it, # and make is happy. $(OPENAPI_GEN): - hack/make-rules/build.sh ./vendor/k8s.io/kube-gen/cmd/openapi-gen + hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/openapi-gen touch $@ # @@ -716,7 +716,7 @@ $(META_DIR)/$(CONVERSION_GEN).mk: (echo -n "$(CONVERSION_GEN): "; \ ./hack/run-in-gopath.sh go list \ -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}' \ - ./vendor/k8s.io/kube-gen/cmd/conversion-gen \ + ./vendor/k8s.io/code-generator/cmd/conversion-gen \ | grep --color=never "^$(PRJ_SRC_PATH)/" \ | xargs ./hack/run-in-gopath.sh go list \ -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}' \ @@ -741,5 +741,5 @@ sinclude $(META_DIR)/$(CONVERSION_GEN).mk # newer than the binary, and try to rebuild it over and over. So we touch it, # and make is happy. $(CONVERSION_GEN): - hack/make-rules/build.sh ./vendor/k8s.io/kube-gen/cmd/conversion-gen + hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/conversion-gen touch $@ diff --git a/build/root/WORKSPACE b/build/root/WORKSPACE index 28e0eaf247ef2..f92e09ddedd65 100644 --- a/build/root/WORKSPACE +++ b/build/root/WORKSPACE @@ -7,9 +7,9 @@ http_archive( http_archive( name = "io_kubernetes_build", - sha256 = "2b2cb64b2fd7734c5eb2b79b79fa35d064fcf53b92cb3aaec5b90fc10fc94135", - strip_prefix = "repo-infra-9ba3a24eeffafa3a794b9e7312103424e7da26e9", - urls = ["https://github.com/kubernetes/repo-infra/archive/9ba3a24eeffafa3a794b9e7312103424e7da26e9.tar.gz"], + sha256 = "5da78568ffb9a323410c701618c23da8c93f4bf4aea76eee41ac244dbd8c8f95", + strip_prefix = "repo-infra-4eaf9e671bbb549fb4ec292cf251f921d7ef80ac", + urls = ["https://github.com/kubernetes/repo-infra/archive/4eaf9e671bbb549fb4ec292cf251f921d7ef80ac.tar.gz"], ) ETCD_VERSION = "3.0.17" diff --git a/build/run.sh b/build/run.sh index af6e92dfa3f37..5304f92803743 100755 --- a/build/run.sh +++ b/build/run.sh @@ -25,10 +25,19 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "$KUBE_ROOT/build/common.sh" +KUBE_RUN_COPY_OUTPUT="${KUBE_RUN_COPY_OUTPUT:-y}" + kube::build::verify_prereqs kube::build::build_image + +if [[ ${KUBE_RUN_COPY_OUTPUT} =~ ^[yY]$ ]]; then + kube::log::status "Output from this container will be rsynced out upon completion. Set KUBE_RUN_COPY_OUTPUT=n to disable." +else + kube::log::status "Output from this container will NOT be rsynced out upon completion. Set KUBE_RUN_COPY_OUTPUT=y to enable." +fi + kube::build::run_build_command "$@" -if [[ ${KUBE_RUN_COPY_OUTPUT:-y} =~ ^[yY]$ ]]; then +if [[ ${KUBE_RUN_COPY_OUTPUT} =~ ^[yY]$ ]]; then kube::build::copy_output fi diff --git a/build/shell.sh b/build/shell.sh index 6f546b662de3f..dac2e4949766e 100755 --- a/build/shell.sh +++ b/build/shell.sh @@ -24,8 +24,5 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/build/common.sh" -source "${KUBE_ROOT}/build/lib/release.sh" -kube::build::verify_prereqs -kube::build::build_image -kube::build::run_build_command bash || true +KUBE_RUN_COPY_OUTPUT="${KUBE_RUN_COPY_OUTPUT:-n}" "${KUBE_ROOT}/build/run.sh" bash "$@" diff --git a/build/visible_to/BUILD b/build/visible_to/BUILD index fef19852b3c94..788adc982cbd0 100644 --- a/build/visible_to/BUILD +++ b/build/visible_to/BUILD @@ -30,7 +30,6 @@ package_group( packages = [ "//cmd/gendocs", "//cmd/genman", - "//cmd/genslateyaml", "//cmd/genyaml", ], ) diff --git a/cluster/addons/calico-policy-controller/OWNERS b/cluster/addons/calico-policy-controller/OWNERS index 0ed4b3ecad3a6..04667f7de2856 100644 --- a/cluster/addons/calico-policy-controller/OWNERS +++ b/cluster/addons/calico-policy-controller/OWNERS @@ -1,6 +1,10 @@ approvers: -- caseydavenport +- bowei +- caseydavenport +- dnardo - fasaxc reviewers: +- bowei - caseydavenport +- dnardo - fasaxc diff --git a/cluster/addons/cluster-loadbalancing/OWNERS b/cluster/addons/cluster-loadbalancing/OWNERS new file mode 100644 index 0000000000000..59cd5f76e7876 --- /dev/null +++ b/cluster/addons/cluster-loadbalancing/OWNERS @@ -0,0 +1,6 @@ +approvers: +- bowei +- nicksardo +reviewers: +- bowei +- nicksardo diff --git a/cluster/addons/cluster-monitoring/heapster-rbac.yaml b/cluster/addons/cluster-monitoring/heapster-rbac.yaml index 58fa1b9921b7d..e75c18b614d7a 100644 --- a/cluster/addons/cluster-monitoring/heapster-rbac.yaml +++ b/cluster/addons/cluster-monitoring/heapster-rbac.yaml @@ -1,4 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: heapster-binding @@ -16,7 +16,7 @@ subjects: --- # Heapster's pod_nanny monitors the heapster deployment & its pod(s), and scales # the resources of the deployment if necessary. -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: system:pod-nanny @@ -39,7 +39,7 @@ rules: - get - update --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: heapster-binding diff --git a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml index bc10aedbab5ae..4a938cbe84003 100644 --- a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml @@ -63,12 +63,14 @@ spec: - name: usr-ca-certs mountPath: /usr/share/ca-certificates readOnly: true + # BEGIN_PROMETHEUS_TO_SD - name: prom-to-sd image: gcr.io/google-containers/prometheus-to-sd:v0.2.1 command: - /monitor - --source=heapster:http://localhost:8082?whitelisted=stackdriver_requests_count,stackdriver_timeseries_count - - --stackdriver-prefix=container.googleapis.com/internal/addons + - --stackdriver-prefix={{ prometheus_to_sd_prefix }}/addons + - --api-override={{ prometheus_to_sd_endpoint }} - --pod-id=$(POD_NAME) - --namespace-id=$(POD_NAMESPACE) volumeMounts: @@ -83,6 +85,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # END_PROMETHEUS_TO_SD - image: gcr.io/google_containers/addon-resizer:2.0 name: heapster-nanny resources: diff --git a/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler-rbac.yaml b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler-rbac.yaml index 1550181c7dcb6..9aa518aaf61ab 100644 --- a/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler-rbac.yaml +++ b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler-rbac.yaml @@ -21,7 +21,7 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:kube-dns-autoscaler labels: @@ -43,7 +43,7 @@ rules: verbs: ["get", "create"] --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: system:kube-dns-autoscaler labels: diff --git a/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml b/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml index 43505ea476e86..5bafaf83c47ca 100644 --- a/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml +++ b/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml @@ -10,7 +10,7 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: elasticsearch-logging labels: @@ -28,7 +28,7 @@ rules: - "get" --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: namespace: kube-system name: elasticsearch-logging diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index cea6dac4cc501..ae58d60180072 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -9,7 +9,7 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: fluentd-es labels: @@ -28,7 +28,7 @@ rules: - "list" --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: fluentd-es labels: diff --git a/cluster/addons/fluentd-gcp/event-exporter.yaml b/cluster/addons/fluentd-gcp/event-exporter.yaml index c3fff97484db7..b08deaaa166d0 100644 --- a/cluster/addons/fluentd-gcp/event-exporter.yaml +++ b/cluster/addons/fluentd-gcp/event-exporter.yaml @@ -8,7 +8,7 @@ metadata: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: event-exporter-rb @@ -51,16 +51,19 @@ spec: image: gcr.io/google-containers/event-exporter:v0.1.5 command: - '/event-exporter' + # BEGIN_PROMETHEUS_TO_SD - name: prometheus-to-sd-exporter image: gcr.io/google-containers/prometheus-to-sd:v0.2.1 command: - /monitor - --component=event_exporter - - --stackdriver-prefix=container.googleapis.com/internal/addons + - --stackdriver-prefix={{ prometheus_to_sd_prefix }}/addons + - --api-override={{ prometheus_to_sd_endpoint }} - --whitelisted-metrics=stackdriver_sink_received_entry_count,stackdriver_sink_request_count,stackdriver_sink_successfully_sent_entry_count volumeMounts: - name: ssl-certs mountPath: /etc/ssl/certs + # END_PROMETHEUS_TO_SD terminationGracePeriodSeconds: 30 volumes: - name: ssl-certs diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml index 069ecc4001eb7..51fbc81c06382 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml @@ -80,17 +80,20 @@ spec: then exit 1; fi; + # BEGIN_PROMETHEUS_TO_SD - name: prometheus-to-sd-exporter image: gcr.io/google-containers/prometheus-to-sd:v0.1.3 command: - /monitor - --component=fluentd - --target-port=31337 - - --stackdriver-prefix=container.googleapis.com/internal/addons + - --stackdriver-prefix={{ prometheus_to_sd_prefix }}/addons + - --api-override={{ prometheus_to_sd_endpoint }} - --whitelisted-metrics=stackdriver_successful_requests_count,stackdriver_failed_requests_count,stackdriver_ingested_entries_count,stackdriver_dropped_entries_count volumeMounts: - name: ssl-certs mountPath: /etc/ssl/certs + # END_PROMETHEUS_TO_SD nodeSelector: beta.kubernetes.io/fluentd-ds-ready: "true" tolerations: diff --git a/cluster/addons/ip-masq-agent/OWNERS b/cluster/addons/ip-masq-agent/OWNERS new file mode 100644 index 0000000000000..ddd12a032ae3d --- /dev/null +++ b/cluster/addons/ip-masq-agent/OWNERS @@ -0,0 +1,6 @@ +approvers: +- bowei +- dnardo +reviewers: +- bowei +- dnardo diff --git a/cluster/addons/kube-proxy/kube-proxy-ds.yaml b/cluster/addons/kube-proxy/kube-proxy-ds.yaml new file mode 100644 index 0000000000000..34a1e42bf618f --- /dev/null +++ b/cluster/addons/kube-proxy/kube-proxy-ds.yaml @@ -0,0 +1,78 @@ +# Please keep kube-proxy configuration in-sync with: +# cluster/saltbase/salt/kube-proxy/kube-proxy.manifest + +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-proxy + addonmanager.kubernetes.io/mode: Reconcile + name: kube-proxy + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-proxy + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 10% + template: + metadata: + labels: + k8s-app: kube-proxy + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/kube-proxy-ds-ready: "true" + initContainers: + - name: touch-lock + image: busybox + command: ['/bin/touch', '/run/xtables.lock'] + securityContext: + privileged: true + volumeMounts: + - mountPath: /run + name: run + readOnly: false + containers: + - name: kube-proxy + image: {{pillar['kube_docker_registry']}}/kube-proxy:{{pillar['kube-proxy_docker_tag']}} + resources: + requests: + cpu: {{ cpurequest }} + command: + - /bin/sh + - -c + - echo -998 > /proc/$$$/oom_score_adj && kube-proxy {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1 + {{container_env}} + {{kube_cache_mutation_detector_env_name}} + {{kube_cache_mutation_detector_env_value}} + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/log + name: varlog + readOnly: false + - mountPath: /var/lib/kube-proxy/kubeconfig + name: kubeconfig + readOnly: false + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + volumes: + - name: varlog + hostPath: + path: /var/log + - name: kubeconfig + hostPath: + path: /var/lib/kube-proxy/kubeconfig + - name: xtables-lock + hostPath: + path: /run/xtables.lock + - name: run + hostPath: + path: /run + serviceAccountName: kube-proxy diff --git a/cluster/addons/kube-proxy/kube-proxy-rbac.yaml b/cluster/addons/kube-proxy/kube-proxy-rbac.yaml new file mode 100644 index 0000000000000..a12ef9d3bfe46 --- /dev/null +++ b/cluster/addons/kube-proxy/kube-proxy-rbac.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-proxy + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: system:kube-proxy + labels: + addonmanager.kubernetes.io/mode: Reconcile +subjects: + - kind: ServiceAccount + name: kube-proxy + namespace: kube-system +roleRef: + kind: ClusterRole + name: system:node-proxier + apiGroup: rbac.authorization.k8s.io diff --git a/cluster/addons/node-problem-detector/npd.yaml b/cluster/addons/node-problem-detector/npd.yaml index 0790f6cccb56e..87365ad17f7b6 100644 --- a/cluster/addons/node-problem-detector/npd.yaml +++ b/cluster/addons/node-problem-detector/npd.yaml @@ -7,7 +7,7 @@ metadata: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: npd-binding @@ -76,6 +76,7 @@ spec: - name: localtime hostPath: path: /etc/localtime + type: "FileOrCreate" serviceAccountName: node-problem-detector tolerations: - operator: "Exists" diff --git a/cluster/addons/node-problem-detector/standalone/npd-binding.yaml b/cluster/addons/node-problem-detector/standalone/npd-binding.yaml index 6de93fd300da5..d7d64a6368463 100644 --- a/cluster/addons/node-problem-detector/standalone/npd-binding.yaml +++ b/cluster/addons/node-problem-detector/standalone/npd-binding.yaml @@ -1,4 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: npd-binding diff --git a/cluster/addons/rbac/kube-apiserver-kubelet-api-admin-binding.yaml b/cluster/addons/rbac/kube-apiserver-kubelet-api-admin-binding.yaml index 65f72f15dfb3b..43af6488263e6 100644 --- a/cluster/addons/rbac/kube-apiserver-kubelet-api-admin-binding.yaml +++ b/cluster/addons/rbac/kube-apiserver-kubelet-api-admin-binding.yaml @@ -1,5 +1,5 @@ # This binding gives the kube-apiserver user full access to the kubelet API -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: kube-apiserver-kubelet-api-admin diff --git a/cluster/addons/rbac/kubelet-api-admin-role.yaml b/cluster/addons/rbac/kubelet-api-admin-role.yaml index 09eb1d1b37f29..aab3dfd83f971 100644 --- a/cluster/addons/rbac/kubelet-api-admin-role.yaml +++ b/cluster/addons/rbac/kubelet-api-admin-role.yaml @@ -1,5 +1,5 @@ # This role allows full access to the kubelet API -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: kubelet-api-admin diff --git a/cluster/addons/rbac/kubelet-binding.yaml b/cluster/addons/rbac/kubelet-binding.yaml index 80567a4b61c88..91b978fb49ce8 100644 --- a/cluster/addons/rbac/kubelet-binding.yaml +++ b/cluster/addons/rbac/kubelet-binding.yaml @@ -2,7 +2,7 @@ # identify the system:nodes group. They use the kubelet identity # TODO: remove this once new nodes are granted individual identities and the # NodeAuthorizer is enabled. -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: kubelet-cluster-admin diff --git a/cluster/addons/rbac/kubelet-certificate-management.yaml b/cluster/addons/rbac/kubelet-certificate-management.yaml index 83f1187d575b0..611aaea400fcc 100644 --- a/cluster/addons/rbac/kubelet-certificate-management.yaml +++ b/cluster/addons/rbac/kubelet-certificate-management.yaml @@ -1,4 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gce:beta:kubelet-certificate-bootstrap @@ -14,7 +14,7 @@ subjects: kind: User name: kubelet --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gce:beta:kubelet-certificate-rotation @@ -30,7 +30,7 @@ subjects: kind: Group name: system:nodes --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gce:beta:kubelet-certificate-bootstrap @@ -45,7 +45,7 @@ rules: verbs: - "create" --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gce:beta:kubelet-certificate-rotation diff --git a/cluster/addons/registry/auth/README.md b/cluster/addons/registry/auth/README.md index 8fbd7e4f3bc44..040c54bcb8d1b 100644 --- a/cluster/addons/registry/auth/README.md +++ b/cluster/addons/registry/auth/README.md @@ -83,7 +83,7 @@ Setup proxy or port-forwarding to the kube-registry. Image push/pull should fail ### Configure Nodes to Authenticate with Kube-Registry -By default, nodes assume no authentication is required by kube-registry. Without authentication, nodes cannot pull images from kube-registry. To solve this, more documentation can be found [Here](https://github.com/kubernetes/kubernetes/blob/master/docs/user-guide/images.md#configuring-nodes-to-authenticate-to-a-private-repository) +By default, nodes assume no authentication is required by kube-registry. Without authentication, nodes cannot pull images from kube-registry. To solve this, more documentation can be found [Here](https://github.com/kubernetes/kubernetes.github.io/blob/master/docs/concepts/containers/images.md#configuring-nodes-to-authenticate-to-a-private-repository). diff --git a/cluster/common.sh b/cluster/common.sh index 403eddedf0f6a..669e520483db1 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -638,6 +638,7 @@ DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-}) DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-}) ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote ${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}) KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) +KUBE_PROXY_DAEMONSET: $(yaml-quote ${KUBE_PROXY_DAEMONSET:-false}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote ${NODE_PROBLEM_DETECTOR_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) @@ -668,6 +669,9 @@ ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-f ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote ${ENABLE_PATCH_CONVERSION_DETECTOR:-false}) ADVANCED_AUDIT_BACKEND: $(yaml-quote ${ADVANCED_AUDIT_BACKEND:-log}) GCE_API_ENDPOINT: $(yaml-quote ${GCE_API_ENDPOINT:-}) +PROMETHEUS_TO_SD_ENDPOINT: $(yaml-quote ${PROMETHEUS_TO_SD_ENDPOINT:-}) +PROMETHEUS_TO_SD_PREFIX: $(yaml-quote ${PROMETHEUS_TO_SD_PREFIX:-}) +ENABLE_PROMETHEUS_TO_SD: $(yaml-quote ${ENABLE_PROMETHEUS_TO_SD:-false}) EOF if [ -n "${KUBELET_PORT:-}" ]; then cat >>$file <>$file <>/srv/salt-overlay/pillar/cluster-params.sls @@ -652,14 +653,15 @@ EOF # This should happen both on cluster initialization and node upgrades. # -# - Uses the CA_CERT and KUBE_PROXY_TOKEN to generate a kubeconfig file for -# the kube-proxy to securely connect to the apiserver. +# - When run as static pods, use the CA_CERT and KUBE_PROXY_TOKEN to generate a +# kubeconfig file for the kube-proxy to securely connect to the apiserver. +# - When run as a daemonset, generate a kubeconfig file specific to service account. function create-salt-kubeproxy-auth() { local -r kube_proxy_kubeconfig_file="/srv/salt-overlay/salt/kube-proxy/kubeconfig" + local kubeconfig_content="" if [ ! -e "${kube_proxy_kubeconfig_file}" ]; then - mkdir -p /srv/salt-overlay/salt/kube-proxy - (umask 077; - cat > "${kube_proxy_kubeconfig_file}" < "${kube_proxy_kubeconfig_file}" <>/etc/gce.conf multizone = ${MULTIZONE} +EOF + fi + if [[ -n "${GCE_ALPHA_FEATURES:-}" ]]; then + use_cloud_config="true" + cat <>/etc/gce.conf +alpha-features = ${GCE_ALPHA_FEATURES} EOF fi if [[ "${use_cloud_config}" != "true" ]]; then @@ -387,8 +393,8 @@ function create-master-kubelet-auth { fi } -function create-kubeproxy-kubeconfig { - echo "Creating kube-proxy kubeconfig file" +function create-kubeproxy-user-kubeconfig { + echo "Creating kube-proxy user kubeconfig file" cat </var/lib/kube-proxy/kubeconfig apiVersion: v1 kind: Config @@ -409,6 +415,30 @@ current-context: service-account-context EOF } +function create-kubeproxy-serviceaccount-kubeconfig { + echo "Creating kube-proxy serviceaccount kubeconfig file" + cat </var/lib/kube-proxy/kubeconfig +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://${KUBERNETES_MASTER_NAME} + name: default +contexts: +- context: + cluster: default + namespace: default + user: default + name: default +current-context: default +users: +- name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +EOF +} + function create-kubecontrollermanager-kubeconfig { echo "Creating kube-controller-manager kubeconfig file" mkdir -p /etc/srv/kubernetes/kube-controller-manager @@ -613,8 +643,17 @@ function start-kubelet { if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" fi + local node_labels="" + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${KUBERNETES_MASTER:-}" != "true" ]]; then + # Add kube-proxy daemonset label to node to avoid situation during cluster + # upgrade/downgrade when there are two instances of kube-proxy running on a node. + node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true" + fi if [[ -n "${NODE_LABELS:-}" ]]; then - flags+=" --node-labels=${NODE_LABELS}" + node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" + fi + if [[ -n "${node_labels:-}" ]]; then + flags+=" --node-labels=${node_labels}" fi if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" @@ -666,11 +705,11 @@ function prepare-log-file { chown root:root $1 } -# Starts kube-proxy pod. -function start-kube-proxy { - echo "Start kube-proxy pod" - prepare-log-file /var/log/kube-proxy.log - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" +# Prepares parameters for kube-proxy manifest. +# $1 source path of kube-proxy manifest. +function prepare-kube-proxy-manifest-variables { + local -r src_file=$1; + remove-salt-config-comments "${src_file}" local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig" @@ -689,14 +728,20 @@ function start-kube-proxy { params+=" ${KUBEPROXY_TEST_ARGS}" fi local container_env="" + local kube_cache_mutation_detector_env_name="" + local kube_cache_mutation_detector_env_value="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env="env:\n - name: KUBE_CACHE_MUTATION_DETECTOR\n value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" + container_env="env:" + kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR" + kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" fi sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file} sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file} sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file} sed -i -e "s@{{params}}@${params}@g" ${src_file} sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file} sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file} sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file} if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then @@ -713,6 +758,14 @@ function start-kube-proxy { mount -o remount,rw /sys; " sed -i -e "s@-\\s\\+kube-proxy@- ${extra_workaround_cmd} kube-proxy@g" "${src_file}" fi +} + +# Starts kube-proxy static pod. +function start-kube-proxy { + echo "Start kube-proxy static pod" + prepare-log-file /var/log/kube-proxy.log + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" + prepare-kube-proxy-manifest-variables "$src_file" cp "${src_file}" /etc/kubernetes/manifests } @@ -828,7 +881,7 @@ function compute-master-manifest-variables { CLOUD_CONFIG_MOUNT="" if [[ -f /etc/gce.conf ]]; then CLOUD_CONFIG_OPT="--cloud-config=/etc/gce.conf" - CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\"}}," + CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}}," CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," fi DOCKER_REGISTRY="gcr.io/google_containers" @@ -933,10 +986,10 @@ function start-kube-apiserver { params+=" --admission-control-config-file=/etc/admission_controller.config" # Mount the file to configure admission controllers if ImagePolicyWebhook is set. admission_controller_config_mount="{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"/etc/admission_controller.config\", \"readOnly\": false}," - admission_controller_config_volume="{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"/etc/admission_controller.config\"}}," + admission_controller_config_volume="{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"/etc/admission_controller.config\", \"type\": \"FileOrCreate\"}}," # Mount the file to configure the ImagePolicyWebhook's webhook. image_policy_webhook_config_mount="{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," - image_policy_webhook_config_volume="{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\"}}," + image_policy_webhook_config_volume="{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\", \"type\": \"FileOrCreate\"}}," fi fi @@ -963,7 +1016,7 @@ function start-kube-apiserver { if [[ -n "${GCP_AUTHN_URL:-}" ]]; then params+=" --authentication-token-webhook-config-file=/etc/gcp_authn.config" webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false}," - webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\"}}," + webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\", \"type\": \"FileOrCreate\"}}," fi local authorization_mode="RBAC" @@ -994,7 +1047,7 @@ function start-kube-apiserver { authorization_mode+=",Webhook" params+=" --authorization-webhook-config-file=/etc/gcp_authz.config" webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false}," - webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\"}}," + webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\", \"type\": \"FileOrCreate\"}}," fi params+=" --authorization-mode=${authorization_mode}" @@ -1193,6 +1246,18 @@ function setup-addon-manifests { chmod 644 "${dst_dir}"/* } +# Updates parameters in yaml file for prometheus-to-sd configuration, or +# removes component if it is disabled. +function update-prometheus-to-sd-parameters { + if [[ "${ENABLE_PROMETHEUS_TO_SD:-}" == "true" ]]; then + sed -i -e "s@{{ *prometheus_to_sd_prefix *}}@${PROMETHEUS_TO_SD_PREFIX}@g" "$1" + sed -i -e "s@{{ *prometheus_to_sd_endpoint *}}@${PROMETHEUS_TO_SD_ENDPOINT}@g" "$1" + else + # Removes all lines between two patterns (throws away prometheus-to-sd) + sed -i -e "/# BEGIN_PROMETHEUS_TO_SD/,/# END_PROMETHEUS_TO_SD/d" "$1" + fi +} + # Prepares the manifests of k8s addons, and starts the addon manager. # Vars assumed: # CLUSTER_NAME @@ -1205,6 +1270,10 @@ function start-kube-addons { setup-addon-manifests "addons" "rbac" # Set up manifests of other addons. + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]]; then + prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml" + setup-addon-manifests "addons" "kube-proxy" + fi if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \ @@ -1241,6 +1310,7 @@ function start-kube-addons { sed -i -e "s@{{ *eventer_memory_per_node *}}@${eventer_memory_per_node}@g" "${controller_yaml}" sed -i -e "s@{{ *nanny_memory *}}@${nanny_memory}@g" "${controller_yaml}" sed -i -e "s@{{ *metrics_cpu_per_node *}}@${metrics_cpu_per_node}@g" "${controller_yaml}" + update-prometheus-to-sd-parameters ${controller_yaml} fi if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then setup-addon-manifests "addons" "dns" @@ -1276,6 +1346,10 @@ function start-kube-addons { if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \ [[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then setup-addon-manifests "addons" "fluentd-gcp" + local -r event_exporter_yaml="${dst_dir}/fluentd-gcp/event-exporter.yaml" + local -r fluentd_gcp_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-ds.yaml" + update-prometheus-to-sd-parameters ${event_exporter_yaml} + update-prometheus-to-sd-parameters ${fluentd_gcp_yaml} fi if [[ "${ENABLE_CLUSTER_UI:-}" == "true" ]]; then setup-addon-manifests "addons" "dashboard" @@ -1447,7 +1521,11 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then create-master-etcd-auth else create-kubelet-kubeconfig "https://${KUBERNETES_MASTER_NAME}" - create-kubeproxy-kubeconfig + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + create-kubeproxy-user-kubeconfig + else + create-kubeproxy-serviceaccount-kubeconfig + fi fi if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then @@ -1475,7 +1553,9 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then start-lb-controller start-rescheduler else - start-kube-proxy + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + start-kube-proxy + fi # Kube-registry-proxy. if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then start-kube-registry-proxy diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index f8f7ffc79f6b0..863a2d6bacd52 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -80,7 +80,6 @@ function get-calico-typha-cpu { echo "${typha_cpu}" } - function config-ip-firewall { echo "Configuring IP firewall rules" # The GCI image has host firewall which drop most inbound/forwarded packets. @@ -425,6 +424,12 @@ EOF use_cloud_config="true" cat <>/etc/gce.conf multizone = ${MULTIZONE} +EOF + fi + if [[ -n "${GCE_ALPHA_FEATURES:-}" ]]; then + use_cloud_config="true" + cat <>/etc/gce.conf +alpha-features = ${GCE_ALPHA_FEATURES} EOF fi if [[ "${use_cloud_config}" != "true" ]]; then @@ -653,7 +658,6 @@ clusters: cluster: server: https://${apiserver_address} certificate-authority: ${CA_CERT_BUNDLE_PATH} - server: https://${KUBERNETES_MASTER_NAME} contexts: - context: cluster: local @@ -676,8 +680,8 @@ function create-master-kubelet-auth { fi } -function create-kubeproxy-kubeconfig { - echo "Creating kube-proxy kubeconfig file" +function create-kubeproxy-user-kubeconfig { + echo "Creating kube-proxy user kubeconfig file" cat </var/lib/kube-proxy/kubeconfig apiVersion: v1 kind: Config @@ -698,6 +702,30 @@ current-context: service-account-context EOF } +function create-kubeproxy-serviceaccount-kubeconfig { + echo "Creating kube-proxy serviceaccount kubeconfig file" + cat </var/lib/kube-proxy/kubeconfig +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://${KUBERNETES_MASTER_NAME} + name: default +contexts: +- context: + cluster: default + namespace: default + user: default + name: default +current-context: default +users: +- name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +EOF +} + function create-kubecontrollermanager-kubeconfig { echo "Creating kube-controller-manager kubeconfig file" mkdir -p /etc/srv/kubernetes/kube-controller-manager @@ -922,8 +950,17 @@ function start-kubelet { if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" fi + local node_labels="" + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${KUBERNETES_MASTER:-}" != "true" ]]; then + # Add kube-proxy daemonset label to node to avoid situation during cluster + # upgrade/downgrade when there are two instances of kube-proxy running on a node. + node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true" + fi if [[ -n "${NODE_LABELS:-}" ]]; then - flags+=" --node-labels=${NODE_LABELS}" + node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" + fi + if [[ -n "${node_labels:-}" ]]; then + flags+=" --node-labels=${node_labels}" fi if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" @@ -1004,11 +1041,11 @@ function prepare-log-file { chown root:root $1 } -# Starts kube-proxy pod. -function start-kube-proxy { - echo "Start kube-proxy pod" - prepare-log-file /var/log/kube-proxy.log - local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" +# Prepares parameters for kube-proxy manifest. +# $1 source path of kube-proxy manifest. +function prepare-kube-proxy-manifest-variables { + local -r src_file=$1; + remove-salt-config-comments "${src_file}" local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig" @@ -1027,19 +1064,34 @@ function start-kube-proxy { params+=" ${KUBEPROXY_TEST_ARGS}" fi local container_env="" + local kube_cache_mutation_detector_env_name="" + local kube_cache_mutation_detector_env_value="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then - container_env="env:\n - name: KUBE_CACHE_MUTATION_DETECTOR\n value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" + container_env="env:" + kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR" + kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\"" fi sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file} sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file} sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file} sed -i -e "s@{{params}}@${params}@g" ${src_file} sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file} + sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file} sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file} sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file} if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file} fi +} + +# Starts kube-proxy static pod. +function start-kube-proxy { + echo "Start kube-proxy static pod" + prepare-log-file /var/log/kube-proxy.log + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest" + prepare-kube-proxy-manifest-variables "${src_file}" + cp "${src_file}" /etc/kubernetes/manifests } @@ -1155,7 +1207,7 @@ function compute-master-manifest-variables { CLOUD_CONFIG_MOUNT="" if [[ -f /etc/gce.conf ]]; then CLOUD_CONFIG_OPT="--cloud-config=/etc/gce.conf" - CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\"}}," + CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}}," CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," fi DOCKER_REGISTRY="gcr.io/google_containers" @@ -1283,7 +1335,7 @@ function start-kube-apiserver { # Create the audit policy file, and mount it into the apiserver pod. create-master-audit-policy "${audit_policy_file}" audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": true}," - audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\"}}," + audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\", \"type\": \"FileOrCreate\"}}," if [[ "${ADVANCED_AUDIT_BACKEND:-log}" == *"log"* ]]; then # The advanced audit log backend config matches the basic audit log config. @@ -1305,7 +1357,7 @@ function start-kube-apiserver { params+=" --audit-webhook-config-file=${audit_webhook_config_file}" create-master-audit-webhook-config "${audit_webhook_config_file}" audit_webhook_config_mount="{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"${audit_webhook_config_file}\", \"readOnly\": true}," - audit_webhook_config_volume="{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"${audit_webhook_config_file}\"}}," + audit_webhook_config_volume="{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"${audit_webhook_config_file}\", \"type\": \"FileOrCreate\"}}," fi fi @@ -1323,10 +1375,10 @@ function start-kube-apiserver { params+=" --admission-control-config-file=/etc/admission_controller.config" # Mount the file to configure admission controllers if ImagePolicyWebhook is set. admission_controller_config_mount="{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"/etc/admission_controller.config\", \"readOnly\": false}," - admission_controller_config_volume="{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"/etc/admission_controller.config\"}}," + admission_controller_config_volume="{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"/etc/admission_controller.config\", \"type\": \"FileOrCreate\"}}," # Mount the file to configure the ImagePolicyWebhook's webhook. image_policy_webhook_config_mount="{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," - image_policy_webhook_config_volume="{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\"}}," + image_policy_webhook_config_volume="{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\", \"type\": \"FileOrCreate\"}}," fi fi @@ -1353,7 +1405,7 @@ function start-kube-apiserver { if [[ -n "${GCP_AUTHN_URL:-}" ]]; then params+=" --authentication-token-webhook-config-file=/etc/gcp_authn.config" webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false}," - webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\"}}," + webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\", \"type\": \"FileOrCreate\"}}," fi @@ -1385,7 +1437,7 @@ function start-kube-apiserver { authorization_mode+=",Webhook" params+=" --authorization-webhook-config-file=/etc/gcp_authz.config" webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false}," - webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\"}}," + webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\", \"type\": \"FileOrCreate\"}}," fi params+=" --authorization-mode=${authorization_mode}" @@ -1405,6 +1457,10 @@ function start-kube-apiserver { if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml" + if [[ -n "${GOOGLE_CLOUD_KMS_CONFIG_FILE_NAME:-}" && -n "${GOOGLE_CLOUD_KMS_CONFIG:-}" ]]; then + echo "${GOOGLE_CLOUD_KMS_CONFIG}" | base64 --decode > "${GOOGLE_CLOUD_KMS_CONFIG_FILE_NAME}" + fi + echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}" params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}" fi @@ -1618,6 +1674,18 @@ function start-fluentd-resource-update { wait-for-apiserver-and-update-fluentd & } +# Updates parameters in yaml file for prometheus-to-sd configuration, or +# removes component if it is disabled. +function update-prometheus-to-sd-parameters { + if [[ "${ENABLE_PROMETHEUS_TO_SD:-}" == "true" ]]; then + sed -i -e "s@{{ *prometheus_to_sd_prefix *}}@${PROMETHEUS_TO_SD_PREFIX}@g" "$1" + sed -i -e "s@{{ *prometheus_to_sd_endpoint *}}@${PROMETHEUS_TO_SD_ENDPOINT}@g" "$1" + else + # Removes all lines between two patterns (throws away prometheus-to-sd) + sed -i -e "/# BEGIN_PROMETHEUS_TO_SD/,/# END_PROMETHEUS_TO_SD/d" "$1" + fi +} + # Prepares the manifests of k8s addons, and starts the addon manager. # Vars assumed: # CLUSTER_NAME @@ -1630,6 +1698,10 @@ function start-kube-addons { setup-addon-manifests "addons" "rbac" # Set up manifests of other addons. + if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]]; then + prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml" + setup-addon-manifests "addons" "kube-proxy" + fi if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \ @@ -1666,6 +1738,7 @@ function start-kube-addons { sed -i -e "s@{{ *eventer_memory_per_node *}}@${eventer_memory_per_node}@g" "${controller_yaml}" sed -i -e "s@{{ *nanny_memory *}}@${nanny_memory}@g" "${controller_yaml}" sed -i -e "s@{{ *metrics_cpu_per_node *}}@${metrics_cpu_per_node}@g" "${controller_yaml}" + update-prometheus-to-sd-parameters ${controller_yaml} fi if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then setup-addon-manifests "addons" "dns" @@ -1701,7 +1774,10 @@ function start-kube-addons { if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \ [[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then setup-addon-manifests "addons" "fluentd-gcp" + local -r event_exporter_yaml="${dst_dir}/fluentd-gcp/event-exporter.yaml" local -r fluentd_gcp_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-ds.yaml" + update-prometheus-to-sd-parameters ${event_exporter_yaml} + update-prometheus-to-sd-parameters ${fluentd_gcp_yaml} start-fluentd-resource-update fi if [[ "${ENABLE_CLUSTER_UI:-}" == "true" ]]; then @@ -1874,7 +1950,11 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then else create-node-pki create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME} - create-kubeproxy-kubeconfig + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + create-kubeproxy-user-kubeconfig + else + create-kubeproxy-serviceaccount-kubeconfig + fi if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then create-node-problem-detector-kubeconfig fi @@ -1897,7 +1977,9 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then start-lb-controller start-rescheduler else - start-kube-proxy + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + start-kube-proxy + fi # Kube-registry-proxy. if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then start-kube-registry-proxy diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 4ff33a801aa5e..e5925ba5eaff0 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -336,7 +336,7 @@ function detect-node-names() { INSTANCE_GROUPS+=($(gcloud compute instance-groups managed list \ --zones "${ZONE}" --project "${PROJECT}" \ --regexp "${NODE_INSTANCE_PREFIX}-.+" \ - --format='value(instanceGroup)' || true)) + --format='value(name)' || true)) NODE_NAMES=() if [[ -n "${INSTANCE_GROUPS[@]:-}" ]]; then for group in "${INSTANCE_GROUPS[@]}"; do diff --git a/cluster/images/hyperkube/Makefile b/cluster/images/hyperkube/Makefile index ccb5e8701155b..e17db25e15db0 100644 --- a/cluster/images/hyperkube/Makefile +++ b/cluster/images/hyperkube/Makefile @@ -38,6 +38,8 @@ endif cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile + # Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel + docker run --rm --privileged multiarch/qemu-user-static:register --reset docker build --pull -t ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${TEMP_DIR} rm -rf "${TEMP_DIR}" diff --git a/cluster/juju/bundles/README.md b/cluster/juju/bundles/README.md index 11fca3f598661..b56a40df4a434 100644 --- a/cluster/juju/bundles/README.md +++ b/cluster/juju/bundles/README.md @@ -59,7 +59,7 @@ Now you should have a running Kubernetes. Run `juju status For further reading on [Juju Quickstart](https://pypi.python.org/pypi/juju-quickstart) -Go to the [Getting started with Juju guide](https://github.com/kubernetes/kubernetes/blob/master/docs/getting-started-guides/juju.md) +Go to the [Getting started with Juju guide](https://kubernetes.io/docs/getting-started-guides/ubuntu/installation/#setting-up-kubernetes-with-juju) for more information about deploying a development Kubernetes cluster. ### Using the Kubernetes Client @@ -178,7 +178,7 @@ github. git clone https://github.com/kubernetes/kubernetes.git -Go to the [Getting started with Juju guide](https://github.com/kubernetes/kubernetes/blob/master/docs/getting-started-guides/juju.md) +Go to the [Getting started with Juju guide](https://kubernetes.io/docs/getting-started-guides/ubuntu/installation/#setting-up-kubernetes-with-juju) for more information about the bundle or charms. ## How to contribute diff --git a/cluster/juju/layers/kubernetes-master/config.yaml b/cluster/juju/layers/kubernetes-master/config.yaml index c59d543cfbb3b..5efcac4185733 100644 --- a/cluster/juju/layers/kubernetes-master/config.yaml +++ b/cluster/juju/layers/kubernetes-master/config.yaml @@ -9,7 +9,7 @@ options: description: The local domain for cluster dns service-cidr: type: string - default: 10.152.0.0/16 + default: 10.152.183.0/24 description: CIDR to user for Kubernetes services. Cannot be changed after deployment. allow-privileged: type: string diff --git a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py index 17cd081e9f31d..790a3cbe4a9fb 100644 --- a/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py +++ b/cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py @@ -465,15 +465,6 @@ def send_data(tls): tls.request_server_cert(common_name, sans, certificate_name) -@when('kube-api.connected') -def push_api_data(kube_api): - ''' Send configuration to remote consumer.''' - # Since all relations already have the private ip address, only - # send the port on the relation object to all consumers. - # The kubernetes api-server uses 6443 for the default secure port. - kube_api.set_api_port('6443') - - @when('kubernetes-master.components.started') def configure_cdk_addons(): ''' Configure CDK addons ''' diff --git a/cluster/juju/layers/kubernetes-worker/README.md b/cluster/juju/layers/kubernetes-worker/README.md index 6fddd252522ad..965199eaf75f8 100644 --- a/cluster/juju/layers/kubernetes-worker/README.md +++ b/cluster/juju/layers/kubernetes-worker/README.md @@ -49,7 +49,7 @@ With the "registry" action that is part for the kubernetes-worker charm, you can Create the relevant authentication files. Let's say you want user `userA` to authenticate with the password `passwordA`. Then you'll do : - echo "userA:passwordA" > htpasswd-plain + echo -n "userA:passwordA" > htpasswd-plain htpasswd -c -b -B htpasswd userA passwordA (the `htpasswd` program comes with the `apache2-utils` package) diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh index 20a4b0a26b987..5128c92af86e4 100644 --- a/cluster/kubemark/gce/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -36,7 +36,7 @@ PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-gci} NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-gci} -MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-cos-stable-59-9460-64-0} +MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-cos-stable-60-9592-84-0} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud} NETWORK=${KUBE_GCE_NETWORK:-e2e} diff --git a/cluster/log-dump/log-dump.sh b/cluster/log-dump/log-dump.sh index 537e548a52870..3874e05a54021 100755 --- a/cluster/log-dump/log-dump.sh +++ b/cluster/log-dump/log-dump.sh @@ -30,6 +30,7 @@ readonly logexporter_namespace="${3:-logexporter}" # check for a function named log_dump_custom_get_instances. If it's # defined, we assume the function can me called with one argument, the # role, which is either "master" or "node". +echo "Checking for custom logdump instances, if any" if [[ $(type -t log_dump_custom_get_instances) == "function" ]]; then readonly use_custom_instance_list=yes else @@ -56,14 +57,17 @@ readonly max_scp_processes=25 # This template spits out the external IPs and images for each node in the cluster in a format like so: # 52.32.7.85 gcr.io/google_containers/kube-apiserver:1355c18c32d7bef16125120bce194fad gcr.io/google_containers/kube-controller-manager:46365cdd8d28b8207950c3c21d1f3900 [...] +echo "Obtaining IPs and images for cluster nodes" readonly ips_and_images='{range .items[*]}{@.status.addresses[?(@.type == "ExternalIP")].address} {@.status.images[*].names[*]}{"\n"}{end}' function setup() { if [[ -z "${use_custom_instance_list}" ]]; then KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. : ${KUBE_CONFIG_FILE:="config-test.sh"} + echo "Sourcing kube-util.sh" source "${KUBE_ROOT}/cluster/kube-util.sh" - detect-project &> /dev/null + echo "Detecting project" + detect-project 2>&1 elif [[ -z "${LOG_DUMP_SSH_KEY:-}" ]]; then echo "LOG_DUMP_SSH_KEY not set, but required when using log_dump_custom_get_instances" exit 1 @@ -121,12 +125,15 @@ function copy-logs-from-node() { # Save logs for node $1 into directory $2. Pass in any non-common files in $3. # Pass in any non-common systemd services in $4. # $3 and $4 should be a space-separated list of files. +# Set $5 to true to indicate it is on master. Default to false. # This function shouldn't ever trigger errexit function save-logs() { local -r node_name="${1}" local -r dir="${2}" local files="${3}" local opt_systemd_services="${4:-""}" + local on_master="${5:-"false"}" + if [[ -n "${use_custom_instance_list}" ]]; then if [[ -n "${LOG_DUMP_SAVE_LOGS:-}" ]]; then files="${files} ${LOG_DUMP_SAVE_LOGS:-}" @@ -147,8 +154,13 @@ function save-logs() { local -r services=( ${systemd_services} ${opt_systemd_services} ${LOG_DUMP_SAVE_SERVICES:-} ) if log-dump-ssh "${node_name}" "command -v journalctl" &> /dev/null; then - log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-node-installation.service" > "${dir}/kube-node-installation.log" || true - log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-node-configuration.service" > "${dir}/kube-node-configuration.log" || true + if [[ "${on_master}" == "true" ]]; then + log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-master-installation.service" > "${dir}/kube-master-installation.log" || true + log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-master-configuration.service" > "${dir}/kube-master-configuration.log" || true + else + log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-node-installation.service" > "${dir}/kube-node-installation.log" || true + log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -u kube-node-configuration.service" > "${dir}/kube-node-configuration.log" || true + fi log-dump-ssh "${node_name}" "sudo journalctl --output=short-precise -k" > "${dir}/kern.log" || true for svc in "${services[@]}"; do @@ -189,7 +201,7 @@ function dump_masters() { for master_name in "${master_names[@]}"; do master_dir="${report_dir}/${master_name}" mkdir -p "${master_dir}" - save-logs "${master_name}" "${master_dir}" "${master_logfiles}" & + save-logs "${master_name}" "${master_dir}" "${master_logfiles}" "" "true" & # We don't want to run more than ${max_scp_processes} at a time, so # wait once we hit that many nodes. This isn't ideal, since one might diff --git a/cluster/saltbase/salt/calico/OWNERS b/cluster/saltbase/salt/calico/OWNERS new file mode 100644 index 0000000000000..ddd12a032ae3d --- /dev/null +++ b/cluster/saltbase/salt/calico/OWNERS @@ -0,0 +1,6 @@ +approvers: +- bowei +- dnardo +reviewers: +- bowei +- dnardo diff --git a/cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest b/cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest index 00bcd8ac06548..5b0a0a521008d 100644 --- a/cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest +++ b/cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest @@ -5,7 +5,7 @@ {% if grains.cloud == 'gce' and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} - {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\"}}," -%} + {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\", \"type\": \"FileOrCreate\"}}," -%} {% endif -%} {% set params = pillar['autoscaler_mig_config'] + " " + cloud_config + " " + pillar.get('autoscaler_expander_config', '') -%} @@ -95,7 +95,8 @@ { "name": "logfile", "hostPath": { - "path": "/var/log/cluster-autoscaler.log" + "path": "/var/log/cluster-autoscaler.log", + "type": "FileOrCreate" } } ], diff --git a/cluster/saltbase/salt/cni/OWNERS b/cluster/saltbase/salt/cni/OWNERS new file mode 100644 index 0000000000000..b09f54100b6ba --- /dev/null +++ b/cluster/saltbase/salt/cni/OWNERS @@ -0,0 +1,8 @@ +approvers: +- bowei +- dnardo +- freehan +reviewers: +- bowei +- dnardo +- freehan diff --git a/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest b/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest index 3cf4a928be5e5..137c7d289b797 100644 --- a/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest +++ b/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest @@ -50,9 +50,11 @@ spec: volumes: - hostPath: path: /var/run/docker.sock + type: Socket name: socket - hostPath: path: /usr/bin/docker + type: File name: docker # This pod is really fire-and-forget. restartPolicy: OnFailure diff --git a/cluster/saltbase/salt/etcd/etcd.manifest b/cluster/saltbase/salt/etcd/etcd.manifest index 1f1be4ffe8caa..c5e56b5a51279 100644 --- a/cluster/saltbase/salt/etcd/etcd.manifest +++ b/cluster/saltbase/salt/etcd/etcd.manifest @@ -103,7 +103,8 @@ }, { "name": "varlogetcd", "hostPath": { - "path": "/var/log/etcd{{ suffix }}.log"} + "path": "/var/log/etcd{{ suffix }}.log", + "type": "FileOrCreate"} }, { "name": "etc", "hostPath": { diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index 4f6ad7bdfd684..e3d8f4cac8bea 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -25,7 +25,7 @@ {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} - {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\"}}," -%} + {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\", \"type\": \"FileOrCreate\"}}," -%} {% endif -%} {% if grains.cloud in ['openstack'] -%} @@ -119,7 +119,7 @@ {% if grains.webhook_authentication_config is defined -%} {% set webhook_authentication_config = " --authentication-token-webhook-config-file=" + grains.webhook_authentication_config -%} {% set webhook_authn_config_mount = "{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"" + grains.webhook_authentication_config + "\", \"readOnly\": false}," -%} - {% set webhook_authn_config_volume = "{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authentication_config + "\"}}," -%} + {% set webhook_authn_config_volume = "{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authentication_config + "\", \"type\": \"FileOrCreate\"}}," -%} {% endif -%} {% set webhook_authorization_config = "" -%} @@ -128,7 +128,7 @@ {% if grains.webhook_authorization_config is defined -%} {% set webhook_authorization_config = " --authorization-webhook-config-file=" + grains.webhook_authorization_config -%} {% set webhook_config_mount = "{\"name\": \"webhookconfigmount\",\"mountPath\": \"" + grains.webhook_authorization_config + "\", \"readOnly\": false}," -%} - {% set webhook_config_volume = "{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authorization_config + "\"}}," -%} + {% set webhook_config_volume = "{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authorization_config + "\", \"type\": \"FileOrCreate\"}}," -%} {% set authz_mode = authz_mode + ",Webhook" -%} {% endif -%} @@ -140,9 +140,9 @@ {% if grains.image_review_config is defined -%} {% set image_review_config = " --admission-control-config-file=" + grains.image_review_config -%} {% set admission_controller_config_mount = "{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"" + grains.image_review_config + "\", \"readOnly\": false}," -%} - {% set admission_controller_config_volume = "{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"" + grains.image_review_config + "\"}}," -%} + {% set admission_controller_config_volume = "{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"" + grains.image_review_config + "\", \"type\": \"FileOrCreate\"}}," -%} {% set image_policy_webhook_config_mount = "{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," -%} - {% set image_policy_webhook_config_volume = "{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\"}}," -%} + {% set image_policy_webhook_config_volume = "{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\", \"type\": \"FileOrCreate\"}}," -%} {% endif -%} {% set admission_control = "" -%} @@ -185,14 +185,14 @@ {% elif pillar['enable_apiserver_advanced_audit'] is defined and pillar['enable_apiserver_advanced_audit'] in ['true'] -%} {% set audit_log = "--audit-policy-file=/etc/audit_policy.config" -%} {% set audit_policy_config_mount = "{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"/etc/audit_policy.config\", \"readOnly\": true}," -%} - {% set audit_policy_config_volume = "{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_policy.config\"}}," -%} + {% set audit_policy_config_volume = "{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_policy.config\", \"type\": \"FileOrCreate\"}}," -%} {% if pillar['advanced_audit_backend'] is defined and 'log' in pillar['advanced_audit_backend'] -%} {% set audit_log = audit_log + " --audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%} {% endif %} {% if pillar['advanced_audit_backend'] is defined and 'webhook' in pillar['advanced_audit_backend'] -%} {% set audit_log = audit_log + " --audit-webhook-mode=batch" -%} {% set audit_webhook_config_mount = "{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"/etc/audit_webhook.config\", \"readOnly\": true}," -%} - {% set audit_webhook_config_volume = "{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_webhook.config\"}}," -%} + {% set audit_webhook_config_volume = "{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_webhook.config\", \"type\": \"FileOrCreate\"}}," -%} {% endif %} {% endif -%} @@ -308,11 +308,13 @@ }, { "name": "logfile", "hostPath": { - "path": "/var/log/kube-apiserver.log"} + "path": "/var/log/kube-apiserver.log", + "type": "FileOrCreate"} }, { "name": "auditlogfile", "hostPath": { - "path": "/var/log/kube-apiserver-audit.log"} + "path": "/var/log/kube-apiserver-audit.log", + "type": "FileOrCreate"} }, { "name": "etcssl", "hostPath": { diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index 9911d09b8d3f6..65359afda7b54 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -49,7 +49,7 @@ {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} - {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\"}}," -%} + {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\", \"type\": \"FileOrCreate\"}}," -%} {% endif -%} {% if grains.cloud in ['openstack'] -%} @@ -164,7 +164,8 @@ }, { "name": "logfile", "hostPath": { - "path": "/var/log/kube-controller-manager.log"} + "path": "/var/log/kube-controller-manager.log", + "type": "FileOrCreate"} }, { "name": "etcssl", "hostPath": { diff --git a/cluster/saltbase/salt/kube-proxy/OWNERS b/cluster/saltbase/salt/kube-proxy/OWNERS new file mode 100644 index 0000000000000..b09f54100b6ba --- /dev/null +++ b/cluster/saltbase/salt/kube-proxy/OWNERS @@ -0,0 +1,8 @@ +approvers: +- bowei +- dnardo +- freehan +reviewers: +- bowei +- dnardo +- freehan diff --git a/cluster/saltbase/salt/kube-proxy/init.sls b/cluster/saltbase/salt/kube-proxy/init.sls index 6d65c574c4503..4ae6103dacd93 100644 --- a/cluster/saltbase/salt/kube-proxy/init.sls +++ b/cluster/saltbase/salt/kube-proxy/init.sls @@ -7,6 +7,7 @@ - makedirs: true # kube-proxy in a static pod +{% if pillar.get('kube_proxy_daemonset', '').lower() != 'true' %} /etc/kubernetes/manifests/kube-proxy.manifest: file.managed: - source: salt://kube-proxy/kube-proxy.manifest @@ -24,6 +25,7 @@ - require: - service: docker - service: kubelet +{% endif %} /var/log/kube-proxy.log: file.managed: diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest index b07ff649e36c8..37d96c9b6a0ed 100644 --- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest +++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -1,3 +1,6 @@ +# Please keep kube-proxy configuration in-sync with: +# cluster/addons/kube-proxy/kube-proxy-ds.yaml + {% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%} {% if grains.api_servers is defined -%} {% set api_servers = "--master=https://" + grains.api_servers -%} @@ -35,6 +38,8 @@ {% set params = log_level + " " + throttles + " " + feature_gates + " " + test_args -%} {% set container_env = "" -%} +{% set kube_cache_mutation_detector_env_name = "" -%} +{% set kube_cache_mutation_detector_env_value = "" -%} # kube-proxy podspec apiVersion: v1 @@ -75,6 +80,8 @@ spec: - -c - echo -998 > /proc/$$$/oom_score_adj && kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1 {{container_env}} + {{kube_cache_mutation_detector_env_name}} + {{kube_cache_mutation_detector_env_value}} securityContext: privileged: true volumeMounts: @@ -102,6 +109,7 @@ spec: name: etc-ssl-certs - hostPath: path: /var/lib/kube-proxy/kubeconfig + type: FileOrCreate name: kubeconfig - hostPath: path: /var/log @@ -111,4 +119,5 @@ spec: name: run - hostPath: path: /run/xtables.lock + type: FileOrCreate name: iptableslock diff --git a/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest b/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest index 8a78a6ddc56e1..6f946bd8c864c 100644 --- a/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest +++ b/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest @@ -83,7 +83,7 @@ }, { "name": "logfile", - "hostPath": {"path": "/var/log/kube-scheduler.log"} + "hostPath": {"path": "/var/log/kube-scheduler.log", "type": "FileOrCreate"} } ] }} diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 470760795b841..846935c3d84c8 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -164,10 +164,16 @@ {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %} {% endif -%} -{% set node_labels = "" %} -{% if pillar['node_labels'] is defined -%} - {% set node_labels="--node-labels=" + pillar['node_labels'] %} -{% endif -%} +{% set kube_proxy_ds_label = "" %} +{% if grains['roles'][0] != 'kubernetes-master' and pillar.get('kube_proxy_daemonset', '').lower() == 'true' %} + # Add kube-proxy daemonset label to node to avoid situation during cluster + # upgrade/downgrade when there are two instances of kube-proxy running on a node. + {% set kube_proxy_ds_label = "beta.kubernetes.io/kube-proxy-ds-ready=true," %} +{% endif %} +{% set node_labels = kube_proxy_ds_label + pillar['node_labels'] %} +{% if node_labels != "" %} + {% set node_labels="--node-labels=" + node_labels %} +{% endif %} {% set node_taints = "" %} {% if pillar['node_taints'] is defined -%} diff --git a/cluster/saltbase/salt/l7-gcp/OWNERS b/cluster/saltbase/salt/l7-gcp/OWNERS new file mode 100644 index 0000000000000..59cd5f76e7876 --- /dev/null +++ b/cluster/saltbase/salt/l7-gcp/OWNERS @@ -0,0 +1,6 @@ +approvers: +- bowei +- nicksardo +reviewers: +- bowei +- nicksardo diff --git a/cluster/saltbase/salt/l7-gcp/glbc.manifest b/cluster/saltbase/salt/l7-gcp/glbc.manifest index bb6863d495524..1214a131f3231 100644 --- a/cluster/saltbase/salt/l7-gcp/glbc.manifest +++ b/cluster/saltbase/salt/l7-gcp/glbc.manifest @@ -48,7 +48,9 @@ spec: volumes: - hostPath: path: /etc/gce.conf + type: FileOrCreate name: cloudconfig - hostPath: path: /var/log/glbc.log + type: FileOrCreate name: logfile diff --git a/cluster/saltbase/salt/rescheduler/rescheduler.manifest b/cluster/saltbase/salt/rescheduler/rescheduler.manifest index 8132a7d6fccc2..ef9af1f5f7fac 100644 --- a/cluster/saltbase/salt/rescheduler/rescheduler.manifest +++ b/cluster/saltbase/salt/rescheduler/rescheduler.manifest @@ -32,4 +32,5 @@ spec: volumes: - hostPath: path: /var/log/rescheduler.log + type: FileOrCreate name: logfile diff --git a/cmd/BUILD b/cmd/BUILD index ec7ef8d0beacc..75dcc63d834f6 100644 --- a/cmd/BUILD +++ b/cmd/BUILD @@ -16,7 +16,6 @@ filegroup( "//cmd/gendocs:all-srcs", "//cmd/genkubedocs:all-srcs", "//cmd/genman:all-srcs", - "//cmd/genslateyaml:all-srcs", "//cmd/genswaggertypedocs:all-srcs", "//cmd/genutils:all-srcs", "//cmd/genyaml:all-srcs", @@ -31,7 +30,6 @@ filegroup( "//cmd/kubelet:all-srcs", "//cmd/kubemark:all-srcs", "//cmd/linkcheck:all-srcs", - "//cmd/mungedocs:all-srcs", ], tags = ["automanaged"], ) diff --git a/cmd/cloud-controller-manager/OWNERS b/cmd/cloud-controller-manager/OWNERS new file mode 100644 index 0000000000000..5a58eee8cea76 --- /dev/null +++ b/cmd/cloud-controller-manager/OWNERS @@ -0,0 +1,8 @@ +approvers: +- thockin +- luxas +- wlan0 +reviewers: +- thockin +- luxas +- wlan0 diff --git a/cmd/cloud-controller-manager/app/options/options.go b/cmd/cloud-controller-manager/app/options/options.go index 2d57294ec888e..1dcf104bebf4f 100644 --- a/cmd/cloud-controller-manager/app/options/options.go +++ b/cmd/cloud-controller-manager/app/options/options.go @@ -67,13 +67,13 @@ func NewCloudControllerManagerServer() *CloudControllerManagerServer { // AddFlags adds flags for a specific ExternalCMServer to the specified FlagSet func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) { - fs.Int32Var(&s.Port, "port", s.Port, "The port that the cloud-controller-manager's http service runs on") - fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)") + fs.Int32Var(&s.Port, "port", s.Port, "The port that the cloud-controller-manager's http service runs on.") + fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).") fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider of cloud services. Cannot be empty.") - fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") - fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.") - fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances") - fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod") + fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") + fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.") + fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.") + fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.") fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration, "The period for syncing NodeStatus in NodeController.") fs.DurationVar(&s.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", s.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.") @@ -81,15 +81,15 @@ func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.") fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.") fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.") - fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/") - fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled") + fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/.") + fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.") fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster.") fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.") - fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") + fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.") - fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver") - fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver") + fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.") + fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.") fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.") leaderelectionconfig.BindFlags(&s.LeaderElection, fs) diff --git a/cmd/cloud-controller-manager/controller-manager.go b/cmd/cloud-controller-manager/controller-manager.go index 49f793294b5c8..b8f2425676bc4 100644 --- a/cmd/cloud-controller-manager/controller-manager.go +++ b/cmd/cloud-controller-manager/controller-manager.go @@ -65,7 +65,7 @@ func main() { if s.AllowUntaggedCloud == true { glog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") } else { - glog.Fatalf("no ClusterID Found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") + glog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") } } diff --git a/cmd/genslateyaml/BUILD b/cmd/genslateyaml/BUILD deleted file mode 100644 index 8a840dcafcb38..0000000000000 --- a/cmd/genslateyaml/BUILD +++ /dev/null @@ -1,37 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) - -go_binary( - name = "genslateyaml", - library = ":go_default_library", -) - -go_library( - name = "go_default_library", - srcs = ["gen_slate_yaml.go"], - deps = [ - "//pkg/kubectl/cmd:go_default_library", - "//pkg/kubectl/cmd/util:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/gopkg.in/yaml.v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/genslateyaml/gen_slate_yaml.go b/cmd/genslateyaml/gen_slate_yaml.go deleted file mode 100644 index 7693fa6d890ef..0000000000000 --- a/cmd/genslateyaml/gen_slate_yaml.go +++ /dev/null @@ -1,197 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "io/ioutil" - "os" - "sort" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "gopkg.in/yaml.v2" - - "k8s.io/kubernetes/pkg/kubectl/cmd" - cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" -) - -// gen_slate_yaml creates a yaml representation of the kubectl help commands. This is to be consumed -// by tools to generate documentation. - -var outputFile = flag.String("output", "", "Destination for kubectl yaml representation.") - -func main() { - flag.Parse() - - if len(*outputFile) < 1 { - fmt.Printf("Must specify --output.\n") - os.Exit(1) - } - - // Initialize a kubectl command that we can use to get the help documentation - kubectl := cmd.NewKubectlCommand(cmdutil.NewFactory(nil), os.Stdin, ioutil.Discard, ioutil.Discard) - - // Create the structural representation - spec := NewKubectlSpec(kubectl) - - // Write the spec to a file as yaml - WriteFile(spec) -} - -func WriteFile(spec KubectlSpec) { - // Marshall the yaml - final, err := yaml.Marshal(&spec) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - // Create the file - outFile, err := os.Create(*outputFile) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - defer outFile.Close() - - // Write the file - _, err = outFile.Write(final) - if err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func NewKubectlSpec(c *cobra.Command) KubectlSpec { - return KubectlSpec{ - TopLevelCommandGroups: []TopLevelCommands{NewTopLevelCommands(c.Commands())}, - } -} - -func NewTopLevelCommands(cs []*cobra.Command) TopLevelCommands { - tlc := TopLevelCommands{} - for _, c := range cs { - tlc.Commands = append(tlc.Commands, NewTopLevelCommand(c)) - } - sort.Sort(tlc) - return tlc -} - -func NewTopLevelCommand(c *cobra.Command) TopLevelCommand { - result := TopLevelCommand{ - MainCommand: NewCommand(c, ""), - } - for _, sub := range c.Commands() { - result.SubCommands = append(result.SubCommands, NewSubCommands(sub, "")...) - } - sort.Sort(result.SubCommands) - return result -} - -// Parse the Options -func NewOptions(flags *pflag.FlagSet) Options { - result := Options{} - flags.VisitAll(func(flag *pflag.Flag) { - opt := &Option{ - Name: flag.Name, - Shorthand: flag.Shorthand, - DefaultValue: flag.DefValue, - Usage: flag.Usage, - } - result = append(result, opt) - }) - return result -} - -// Parse the Commands -func NewSubCommands(c *cobra.Command, path string) Commands { - subCommands := Commands{NewCommand(c, path+c.Name())} - for _, subCommand := range c.Commands() { - subCommands = append(subCommands, NewSubCommands(subCommand, path+c.Name()+" ")...) - } - return subCommands -} - -func NewCommand(c *cobra.Command, path string) *Command { - return &Command{ - Name: c.Name(), - Path: path, - Description: c.Long, - Synopsis: c.Short, - Example: c.Example, - Options: NewOptions(c.NonInheritedFlags()), - InheritedOptions: NewOptions(c.InheritedFlags()), - Usage: c.Use, - } -} - -////////////////////////// -// Types -////////////////////////// - -type KubectlSpec struct { - TopLevelCommandGroups []TopLevelCommands `yaml:",omitempty"` -} - -type TopLevelCommands struct { - Commands []TopLevelCommand `yaml:",omitempty"` -} -type TopLevelCommand struct { - MainCommand *Command `yaml:",omitempty"` - SubCommands Commands `yaml:",omitempty"` -} - -type Options []*Option -type Option struct { - Name string `yaml:",omitempty"` - Shorthand string `yaml:",omitempty"` - DefaultValue string `yaml:"default_value,omitempty"` - Usage string `yaml:",omitempty"` -} - -type Commands []*Command -type Command struct { - Name string `yaml:",omitempty"` - Path string `yaml:",omitempty"` - Synopsis string `yaml:",omitempty"` - Description string `yaml:",omitempty"` - Options Options `yaml:",omitempty"` - InheritedOptions Options `yaml:"inherited_options,omitempty"` - Example string `yaml:",omitempty"` - SeeAlso []string `yaml:"see_also,omitempty"` - Usage string `yaml:",omitempty"` -} - -func (a Options) Len() int { return len(a) } -func (a Options) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a Options) Less(i, j int) bool { - return a[i].Name < a[j].Name -} - -func (a TopLevelCommands) Len() int { return len(a.Commands) } -func (a TopLevelCommands) Swap(i, j int) { a.Commands[i], a.Commands[j] = a.Commands[j], a.Commands[i] } -func (a TopLevelCommands) Less(i, j int) bool { - return a.Commands[i].MainCommand.Path < a.Commands[j].MainCommand.Path -} - -func (a Commands) Len() int { return len(a) } -func (a Commands) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a Commands) Less(i, j int) bool { - return a[i].Path < a[j].Path -} diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index 48b7925eff759..563922ce5cbc2 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -10,7 +10,6 @@ go_library( srcs = [ "aggregator.go", "apiextensions.go", - "plugins.go", "server.go", ], deps = [ @@ -24,7 +23,6 @@ go_library( "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/informers/informers_generated/internalversion:go_default_library", "//pkg/cloudprovider:go_default_library", - "//pkg/cloudprovider/providers:go_default_library", "//pkg/controller/serviceaccount:go_default_library", "//pkg/generated/openapi:go_default_library", "//pkg/kubeapiserver:go_default_library", @@ -42,31 +40,6 @@ go_library( "//pkg/util/reflector/prometheus:go_default_library", "//pkg/util/workqueue/prometheus:go_default_library", "//pkg/version:go_default_library", - "//plugin/pkg/admission/admit:go_default_library", - "//plugin/pkg/admission/alwayspullimages:go_default_library", - "//plugin/pkg/admission/antiaffinity:go_default_library", - "//plugin/pkg/admission/defaulttolerationseconds:go_default_library", - "//plugin/pkg/admission/deny:go_default_library", - "//plugin/pkg/admission/exec:go_default_library", - "//plugin/pkg/admission/gc:go_default_library", - "//plugin/pkg/admission/imagepolicy:go_default_library", - "//plugin/pkg/admission/initialization:go_default_library", - "//plugin/pkg/admission/initialresources:go_default_library", - "//plugin/pkg/admission/limitranger:go_default_library", - "//plugin/pkg/admission/namespace/autoprovision:go_default_library", - "//plugin/pkg/admission/namespace/exists:go_default_library", - "//plugin/pkg/admission/noderestriction:go_default_library", - "//plugin/pkg/admission/persistentvolume/label:go_default_library", - "//plugin/pkg/admission/podnodeselector:go_default_library", - "//plugin/pkg/admission/podpreset:go_default_library", - "//plugin/pkg/admission/podtolerationrestriction:go_default_library", - "//plugin/pkg/admission/priority:go_default_library", - "//plugin/pkg/admission/resourcequota:go_default_library", - "//plugin/pkg/admission/security/podsecuritypolicy:go_default_library", - "//plugin/pkg/admission/securitycontext/scdeny:go_default_library", - "//plugin/pkg/admission/serviceaccount:go_default_library", - "//plugin/pkg/admission/storageclass/setdefault:go_default_library", - "//plugin/pkg/admission/webhook:go_default_library", "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/golang/glog:go_default_library", diff --git a/cmd/kube-apiserver/app/options/BUILD b/cmd/kube-apiserver/app/options/BUILD index 6e5b3d199632b..a9fe44b77f61d 100644 --- a/cmd/kube-apiserver/app/options/BUILD +++ b/cmd/kube-apiserver/app/options/BUILD @@ -10,17 +10,45 @@ go_library( name = "go_default_library", srcs = [ "options.go", + "plugins.go", "validation.go", ], deps = [ "//pkg/api:go_default_library", "//pkg/api/validation:go_default_library", + "//pkg/cloudprovider/providers:go_default_library", "//pkg/features:go_default_library", "//pkg/kubeapiserver/options:go_default_library", "//pkg/kubelet/client:go_default_library", "//pkg/master/ports:go_default_library", + "//plugin/pkg/admission/admit:go_default_library", + "//plugin/pkg/admission/alwayspullimages:go_default_library", + "//plugin/pkg/admission/antiaffinity:go_default_library", + "//plugin/pkg/admission/defaulttolerationseconds:go_default_library", + "//plugin/pkg/admission/deny:go_default_library", + "//plugin/pkg/admission/exec:go_default_library", + "//plugin/pkg/admission/gc:go_default_library", + "//plugin/pkg/admission/imagepolicy:go_default_library", + "//plugin/pkg/admission/initialization:go_default_library", + "//plugin/pkg/admission/initialresources:go_default_library", + "//plugin/pkg/admission/limitranger:go_default_library", + "//plugin/pkg/admission/namespace/autoprovision:go_default_library", + "//plugin/pkg/admission/namespace/exists:go_default_library", + "//plugin/pkg/admission/noderestriction:go_default_library", + "//plugin/pkg/admission/persistentvolume/label:go_default_library", + "//plugin/pkg/admission/podnodeselector:go_default_library", + "//plugin/pkg/admission/podpreset:go_default_library", + "//plugin/pkg/admission/podtolerationrestriction:go_default_library", + "//plugin/pkg/admission/priority:go_default_library", + "//plugin/pkg/admission/resourcequota:go_default_library", + "//plugin/pkg/admission/security/podsecuritypolicy:go_default_library", + "//plugin/pkg/admission/securitycontext/scdeny:go_default_library", + "//plugin/pkg/admission/serviceaccount:go_default_library", + "//plugin/pkg/admission/storageclass/setdefault:go_default_library", + "//plugin/pkg/admission/webhook:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", ], @@ -39,6 +67,7 @@ go_test( "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", + "//vendor/k8s.io/client-go/rest:go_default_library", ], ) diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index c9daf9da33762..31cc05294340f 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -113,6 +113,9 @@ func NewServerRunOptions() *ServerRunOptions { } // Overwrite the default for storage data format. s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf" + + // register all admission plugins + RegisterAllAdmissionPlugins(s.Admission.Plugins) // Set the default for admission plugins names s.Admission.PluginNames = []string{"AlwaysAdmit"} return &s diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index b96a878ec2711..93a5d52ed6de1 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -28,13 +28,13 @@ import ( apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/storagebackend" utilconfig "k8s.io/apiserver/pkg/util/flag" + restclient "k8s.io/client-go/rest" kapi "k8s.io/kubernetes/pkg/api" kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" ) -func TestAddFlagsFlag(t *testing.T) { - // TODO: Expand the test to include other flags as well. +func TestAddFlags(t *testing.T) { f := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) s := NewServerRunOptions() s.AddFlags(f) @@ -65,9 +65,24 @@ func TestAddFlagsFlag(t *testing.T) { "--cloud-config=/cloud-config", "--cloud-provider=azure", "--cors-allowed-origins=10.10.10.100,10.10.10.200", + "--contention-profiling=true", "--enable-aggregator-routing=true", "--enable-logs-handler=false", "--enable-swagger-ui=true", + "--etcd-quorum-read=false", + "--etcd-keyfile=/var/run/kubernetes/etcd.key", + "--etcd-certfile=/var/run/kubernetes/etcdce.crt", + "--etcd-cafile=/var/run/kubernetes/etcdca.crt", + "--kubelet-https=true", + "--kubelet-read-only-port=10255", + "--kubelet-timeout=5s", + "--kubelet-client-certificate=/var/run/kubernetes/ceserver.crt", + "--kubelet-client-key=/var/run/kubernetes/server.key", + "--kubelet-certificate-authority=/var/run/kubernetes/caserver.crt", + "--proxy-client-cert-file=/var/run/kubernetes/proxy.crt", + "--proxy-client-key-file=/var/run/kubernetes/proxy.key", + "--request-timeout=2m", + "--storage-backend=etcd2", } f.Parse(args) @@ -81,6 +96,7 @@ func TestAddFlagsFlag(t *testing.T) { CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"}, MaxRequestsInFlight: 400, MaxMutatingRequestsInFlight: 200, + RequestTimeout: time.Duration(2) * time.Minute, MinRequestTimeout: 1800, }, Admission: &apiserveroptions.AdmissionOptions{ @@ -90,10 +106,15 @@ func TestAddFlagsFlag(t *testing.T) { }, Etcd: &apiserveroptions.EtcdOptions{ StorageConfig: storagebackend.Config{ + Type: "etcd2", ServerList: nil, Prefix: "/registry", DeserializationCacheSize: 0, - Copier: kapi.Scheme, + Copier: kapi.Scheme, + Quorum: false, + KeyFile: "/var/run/kubernetes/etcd.key", + CAFile: "/var/run/kubernetes/etcdca.crt", + CertFile: "/var/run/kubernetes/etcdce.crt", }, DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", DeleteCollectionWorkers: 1, @@ -126,6 +147,11 @@ func TestAddFlagsFlag(t *testing.T) { }, EnableHttps: true, HTTPTimeout: time.Duration(5) * time.Second, + TLSClientConfig: restclient.TLSClientConfig{ + CertFile: "/var/run/kubernetes/ceserver.crt", + KeyFile: "/var/run/kubernetes/server.key", + CAFile: "/var/run/kubernetes/caserver.crt", + }, }, Audit: &apiserveroptions.AuditOptions{ LogOptions: apiserveroptions.AuditLogOptions{ @@ -133,7 +159,7 @@ func TestAddFlagsFlag(t *testing.T) { MaxAge: 11, MaxBackups: 12, MaxSize: 13, - Format: "legacy", + Format: "json", }, WebhookOptions: apiserveroptions.AuditWebhookOptions{ Mode: "blocking", @@ -142,8 +168,9 @@ func TestAddFlagsFlag(t *testing.T) { PolicyFile: "/policy", }, Features: &apiserveroptions.FeatureOptions{ - EnableSwaggerUI: true, - EnableProfiling: true, + EnableSwaggerUI: true, + EnableProfiling: true, + EnableContentionProfiling: true, }, Authentication: &kubeoptions.BuiltInAuthenticationOptions{ Anonymous: &kubeoptions.AnonymousAuthenticationOptions{ @@ -190,6 +217,8 @@ func TestAddFlagsFlag(t *testing.T) { }, EnableLogsHandler: false, EnableAggregatorRouting: true, + ProxyClientKeyFile: "/var/run/kubernetes/proxy.key", + ProxyClientCertFile: "/var/run/kubernetes/proxy.crt", } if !reflect.DeepEqual(expected, s) { diff --git a/cmd/kube-apiserver/app/plugins.go b/cmd/kube-apiserver/app/options/plugins.go similarity index 99% rename from cmd/kube-apiserver/app/plugins.go rename to cmd/kube-apiserver/app/options/plugins.go index 06e21d5a5f0ce..7d0e8bb04a304 100644 --- a/cmd/kube-apiserver/app/plugins.go +++ b/cmd/kube-apiserver/app/options/plugins.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package app +package options // This file exists to force the desired plugin implementations to be linked. // This should probably be part of some configuration fed into the build for a diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 7d0e9a1fc2483..df00872b9b15f 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -249,9 +249,6 @@ func CreateNodeDialer(s *options.ServerRunOptions) (tunneler.Tunneler, *http.Tra // CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunneler.Tunneler, proxyTransport http.RoundTripper) (*master.Config, informers.SharedInformerFactory, clientgoinformers.SharedInformerFactory, *kubeserver.InsecureServingInfo, aggregatorapiserver.ServiceResolver, error) { - // register all admission plugins - RegisterAllAdmissionPlugins(s.Admission.Plugins) - // set defaults in the options before trying to create the generic config if err := defaultOptions(s); err != nil { return nil, nil, nil, nil, nil, err @@ -262,6 +259,14 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele return nil, nil, nil, nil, nil, utilerrors.NewAggregate(errs) } + if s.CloudProvider != nil { + // Initialize the cloudprovider once, to give it a chance to register KMS plugins, if any. + _, err := cloudprovider.InitCloudProvider(s.CloudProvider.CloudProvider, s.CloudProvider.CloudConfigFile) + if err != nil { + return nil, nil, nil, nil, nil, err + } + } + genericConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := BuildGenericConfig(s) if err != nil { return nil, nil, nil, nil, nil, err @@ -542,7 +547,8 @@ func BuildAuthorizer(s *options.ServerRunOptions, sharedInformers informers.Shar return authorizationConfig.New() } -// BuildStorageFactory constructs the storage factory +// BuildStorageFactory constructs the storage factory. If encryption at rest is used, it expects +// all supported KMS plugins to be registered in the KMS plugin registry before being called. func BuildStorageFactory(s *options.ServerRunOptions) (*serverstorage.DefaultStorageFactory, error) { storageGroupsToEncodingVersion, err := s.StorageSerialization.StorageGroupsToEncodingVersion() if err != nil { @@ -584,7 +590,7 @@ func BuildStorageFactory(s *options.ServerRunOptions) (*serverstorage.DefaultSto storageFactory.SetEtcdLocation(groupResource, servers) } - if s.Etcd.EncryptionProviderConfigFilepath != "" { + if len(s.Etcd.EncryptionProviderConfigFilepath) != 0 { transformerOverrides, err := encryptionconfig.GetTransformerOverrides(s.Etcd.EncryptionProviderConfigFilepath) if err != nil { return nil, err @@ -669,292 +675,333 @@ func readCAorNil(file string) ([]byte, error) { // PostProcessSpec adds removed definitions for backward compatibility func postProcessOpenAPISpecForBackwardCompatibility(s *spec.Swagger) (*spec.Swagger, error) { compatibilityMap := map[string]string{ - "v1beta1.DeploymentStatus": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DeploymentStatus", - "v1beta1.ReplicaSetList": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.ReplicaSetList", - "v1beta1.Eviction": "k8s.io/kubernetes/pkg/apis/policy/v1beta1.Eviction", - "v1beta1.StatefulSetList": "k8s.io/kubernetes/pkg/apis/apps/v1beta1.StatefulSetList", - "v1beta1.RoleBinding": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.RoleBinding", - "v1beta1.PodSecurityPolicyList": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.PodSecurityPolicyList", - "v1.NodeSpec": "k8s.io/kubernetes/pkg/api/v1.NodeSpec", - "v1.FlockerVolumeSource": "k8s.io/kubernetes/pkg/api/v1.FlockerVolumeSource", - "v1.ContainerState": "k8s.io/kubernetes/pkg/api/v1.ContainerState", - "v1beta1.ClusterRole": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.ClusterRole", - "v1beta1.StorageClass": "k8s.io/kubernetes/pkg/apis/storage/v1beta1.StorageClass", - "v1.FlexVolumeSource": "k8s.io/kubernetes/pkg/api/v1.FlexVolumeSource", - "v1.SecretKeySelector": "k8s.io/kubernetes/pkg/api/v1.SecretKeySelector", - "v1.DeleteOptions": "k8s.io/kubernetes/pkg/api/v1.DeleteOptions", - "v1.PodStatus": "k8s.io/kubernetes/pkg/api/v1.PodStatus", - "v1.NodeStatus": "k8s.io/kubernetes/pkg/api/v1.NodeStatus", - "v1.ServiceSpec": "k8s.io/kubernetes/pkg/api/v1.ServiceSpec", - "v1.AttachedVolume": "k8s.io/kubernetes/pkg/api/v1.AttachedVolume", - "v1.PersistentVolume": "k8s.io/kubernetes/pkg/api/v1.PersistentVolume", - "v1.LimitRangeList": "k8s.io/kubernetes/pkg/api/v1.LimitRangeList", - "v1alpha1.Role": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.Role", - "v1.Affinity": "k8s.io/kubernetes/pkg/api/v1.Affinity", - "v1beta1.PodDisruptionBudget": "k8s.io/kubernetes/pkg/apis/policy/v1beta1.PodDisruptionBudget", - "v1alpha1.RoleBindingList": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.RoleBindingList", - "v1.PodAffinity": "k8s.io/kubernetes/pkg/api/v1.PodAffinity", - "v1beta1.SELinuxStrategyOptions": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.SELinuxStrategyOptions", - "v1.ResourceQuotaList": "k8s.io/kubernetes/pkg/api/v1.ResourceQuotaList", - "v1.PodList": "k8s.io/kubernetes/pkg/api/v1.PodList", - "v1.EnvVarSource": "k8s.io/kubernetes/pkg/api/v1.EnvVarSource", - "v1beta1.TokenReviewStatus": "k8s.io/kubernetes/pkg/apis/authentication/v1beta1.TokenReviewStatus", - "v1.PersistentVolumeClaimList": "k8s.io/kubernetes/pkg/api/v1.PersistentVolumeClaimList", - "v1beta1.RoleList": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.RoleList", - "v1.ListMeta": "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta", - "v1.ObjectMeta": "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta", - "v1.APIGroupList": "k8s.io/apimachinery/pkg/apis/meta/v1.APIGroupList", - "v2alpha1.Job": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.Job", - "v1.EnvFromSource": "k8s.io/kubernetes/pkg/api/v1.EnvFromSource", - "v1beta1.IngressStatus": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.IngressStatus", - "v1.Service": "k8s.io/kubernetes/pkg/api/v1.Service", - "v1beta1.DaemonSetStatus": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DaemonSetStatus", - "v1alpha1.Subject": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.Subject", - "v1.HorizontalPodAutoscaler": "k8s.io/kubernetes/pkg/apis/autoscaling/v1.HorizontalPodAutoscaler", - "v1.StatusCause": "k8s.io/apimachinery/pkg/apis/meta/v1.StatusCause", - "v1.NodeSelectorRequirement": "k8s.io/kubernetes/pkg/api/v1.NodeSelectorRequirement", - "v1beta1.NetworkPolicyIngressRule": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.NetworkPolicyIngressRule", - "v1beta1.ThirdPartyResource": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.ThirdPartyResource", - "v1beta1.PodSecurityPolicy": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.PodSecurityPolicy", - "v1beta1.StatefulSet": "k8s.io/kubernetes/pkg/apis/apps/v1beta1.StatefulSet", - "v1.LabelSelector": "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector", - "v1.ScaleSpec": "k8s.io/kubernetes/pkg/apis/autoscaling/v1.ScaleSpec", - "v1.DownwardAPIVolumeFile": "k8s.io/kubernetes/pkg/api/v1.DownwardAPIVolumeFile", - "v1beta1.HorizontalPodAutoscaler": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.HorizontalPodAutoscaler", - "v1.AWSElasticBlockStoreVolumeSource": "k8s.io/kubernetes/pkg/api/v1.AWSElasticBlockStoreVolumeSource", - "v1.ComponentStatus": "k8s.io/kubernetes/pkg/api/v1.ComponentStatus", - "v2alpha1.JobSpec": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.JobSpec", - "v1.ContainerImage": "k8s.io/kubernetes/pkg/api/v1.ContainerImage", - "v1.ReplicationControllerStatus": "k8s.io/kubernetes/pkg/api/v1.ReplicationControllerStatus", - "v1.ResourceQuota": "k8s.io/kubernetes/pkg/api/v1.ResourceQuota", - "v1beta1.NetworkPolicyList": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.NetworkPolicyList", - "v1beta1.NonResourceAttributes": "k8s.io/kubernetes/pkg/apis/authorization/v1beta1.NonResourceAttributes", - "v1.JobCondition": "k8s.io/kubernetes/pkg/apis/batch/v1.JobCondition", - "v1.LabelSelectorRequirement": "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelectorRequirement", - "v1beta1.Deployment": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.Deployment", - "v1.LoadBalancerIngress": "k8s.io/kubernetes/pkg/api/v1.LoadBalancerIngress", - "v1.SecretList": "k8s.io/kubernetes/pkg/api/v1.SecretList", - "v1beta1.ReplicaSetSpec": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.ReplicaSetSpec", - "v1beta1.RoleBindingList": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.RoleBindingList", - "v1.ServicePort": "k8s.io/kubernetes/pkg/api/v1.ServicePort", - "v1.Namespace": "k8s.io/kubernetes/pkg/api/v1.Namespace", - "v1beta1.NetworkPolicyPeer": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.NetworkPolicyPeer", - "v1.ReplicationControllerList": "k8s.io/kubernetes/pkg/api/v1.ReplicationControllerList", - "v1beta1.ReplicaSetCondition": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.ReplicaSetCondition", - "v1.ReplicationControllerCondition": "k8s.io/kubernetes/pkg/api/v1.ReplicationControllerCondition", - "v1.DaemonEndpoint": "k8s.io/kubernetes/pkg/api/v1.DaemonEndpoint", - "v1beta1.NetworkPolicyPort": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.NetworkPolicyPort", - "v1.NodeSystemInfo": "k8s.io/kubernetes/pkg/api/v1.NodeSystemInfo", - "v1.LimitRangeItem": "k8s.io/kubernetes/pkg/api/v1.LimitRangeItem", - "v1.ConfigMapVolumeSource": "k8s.io/kubernetes/pkg/api/v1.ConfigMapVolumeSource", - "v1beta1.ClusterRoleList": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.ClusterRoleList", - "v1beta1.ResourceAttributes": "k8s.io/kubernetes/pkg/apis/authorization/v1beta1.ResourceAttributes", - "v1.Pod": "k8s.io/kubernetes/pkg/api/v1.Pod", - "v1.FCVolumeSource": "k8s.io/kubernetes/pkg/api/v1.FCVolumeSource", - "v1beta1.SubresourceReference": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.SubresourceReference", - "v1.ResourceQuotaStatus": "k8s.io/kubernetes/pkg/api/v1.ResourceQuotaStatus", - "v1alpha1.RoleBinding": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.RoleBinding", - "v1.PodCondition": "k8s.io/kubernetes/pkg/api/v1.PodCondition", - "v1.GroupVersionForDiscovery": "k8s.io/apimachinery/pkg/apis/meta/v1.GroupVersionForDiscovery", - "v1.NamespaceStatus": "k8s.io/kubernetes/pkg/api/v1.NamespaceStatus", - "v1.Job": "k8s.io/kubernetes/pkg/apis/batch/v1.Job", - "v1.PersistentVolumeClaimVolumeSource": "k8s.io/kubernetes/pkg/api/v1.PersistentVolumeClaimVolumeSource", - "v1.Handler": "k8s.io/kubernetes/pkg/api/v1.Handler", - "v1.ComponentStatusList": "k8s.io/kubernetes/pkg/api/v1.ComponentStatusList", - "v1.ServerAddressByClientCIDR": "k8s.io/apimachinery/pkg/apis/meta/v1.ServerAddressByClientCIDR", - "v1.PodAntiAffinity": "k8s.io/kubernetes/pkg/api/v1.PodAntiAffinity", - "v1.ISCSIVolumeSource": "k8s.io/kubernetes/pkg/api/v1.ISCSIVolumeSource", - "v1.ContainerStateRunning": "k8s.io/kubernetes/pkg/api/v1.ContainerStateRunning", - "v1.WeightedPodAffinityTerm": "k8s.io/kubernetes/pkg/api/v1.WeightedPodAffinityTerm", - "v1beta1.HostPortRange": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.HostPortRange", - "v1.HorizontalPodAutoscalerSpec": "k8s.io/kubernetes/pkg/apis/autoscaling/v1.HorizontalPodAutoscalerSpec", - "v1.HorizontalPodAutoscalerList": "k8s.io/kubernetes/pkg/apis/autoscaling/v1.HorizontalPodAutoscalerList", - "v1beta1.RoleRef": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.RoleRef", - "v1.Probe": "k8s.io/kubernetes/pkg/api/v1.Probe", - "v1beta1.IngressTLS": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.IngressTLS", - "v1beta1.ThirdPartyResourceList": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.ThirdPartyResourceList", - "v1beta1.DaemonSet": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DaemonSet", - "v1.APIGroup": "k8s.io/apimachinery/pkg/apis/meta/v1.APIGroup", - "v1beta1.Subject": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.Subject", - "v1beta1.DeploymentList": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DeploymentList", - "v1.NodeAffinity": "k8s.io/kubernetes/pkg/api/v1.NodeAffinity", - "v1beta1.RollingUpdateDeployment": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.RollingUpdateDeployment", - "v1beta1.APIVersion": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.APIVersion", - "v1alpha1.CertificateSigningRequest": "k8s.io/kubernetes/pkg/apis/certificates/v1alpha1.CertificateSigningRequest", - "v1.CinderVolumeSource": "k8s.io/kubernetes/pkg/api/v1.CinderVolumeSource", - "v1.NamespaceSpec": "k8s.io/kubernetes/pkg/api/v1.NamespaceSpec", - "v1beta1.PodDisruptionBudgetSpec": "k8s.io/kubernetes/pkg/apis/policy/v1beta1.PodDisruptionBudgetSpec", - "v1.Patch": "k8s.io/apimachinery/pkg/apis/meta/v1.Patch", - "v1beta1.ClusterRoleBinding": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.ClusterRoleBinding", - "v1beta1.HorizontalPodAutoscalerSpec": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.HorizontalPodAutoscalerSpec", - "v1.PersistentVolumeClaimSpec": "k8s.io/kubernetes/pkg/api/v1.PersistentVolumeClaimSpec", - "v1.Secret": "k8s.io/kubernetes/pkg/api/v1.Secret", - "v1.NodeCondition": "k8s.io/kubernetes/pkg/api/v1.NodeCondition", - "v1.LocalObjectReference": "k8s.io/kubernetes/pkg/api/v1.LocalObjectReference", - "runtime.RawExtension": "k8s.io/apimachinery/pkg/runtime.RawExtension", - "v1.PreferredSchedulingTerm": "k8s.io/kubernetes/pkg/api/v1.PreferredSchedulingTerm", - "v1.RBDVolumeSource": "k8s.io/kubernetes/pkg/api/v1.RBDVolumeSource", - "v1.KeyToPath": "k8s.io/kubernetes/pkg/api/v1.KeyToPath", - "v1.ScaleStatus": "k8s.io/kubernetes/pkg/apis/autoscaling/v1.ScaleStatus", - "v1alpha1.PolicyRule": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.PolicyRule", - "v1.EndpointPort": "k8s.io/kubernetes/pkg/api/v1.EndpointPort", - "v1beta1.IngressList": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.IngressList", - "v1.EndpointAddress": "k8s.io/kubernetes/pkg/api/v1.EndpointAddress", - "v1.NodeSelector": "k8s.io/kubernetes/pkg/api/v1.NodeSelector", - "v1beta1.StorageClassList": "k8s.io/kubernetes/pkg/apis/storage/v1beta1.StorageClassList", - "v1.ServiceList": "k8s.io/kubernetes/pkg/api/v1.ServiceList", - "v2alpha1.CronJobSpec": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.CronJobSpec", - "v1.ContainerStateTerminated": "k8s.io/kubernetes/pkg/api/v1.ContainerStateTerminated", - "v1beta1.TokenReview": "k8s.io/kubernetes/pkg/apis/authentication/v1beta1.TokenReview", - "v1beta1.IngressBackend": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.IngressBackend", - "v1.Time": "k8s.io/apimachinery/pkg/apis/meta/v1.Time", - "v1beta1.IngressSpec": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.IngressSpec", - "v2alpha1.JobTemplateSpec": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.JobTemplateSpec", - "v1.LimitRange": "k8s.io/kubernetes/pkg/api/v1.LimitRange", - "v1beta1.UserInfo": "k8s.io/kubernetes/pkg/apis/authentication/v1beta1.UserInfo", - "v1.ResourceQuotaSpec": "k8s.io/kubernetes/pkg/api/v1.ResourceQuotaSpec", - "v1.ContainerPort": "k8s.io/kubernetes/pkg/api/v1.ContainerPort", - "v1beta1.HTTPIngressRuleValue": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.HTTPIngressRuleValue", - "v1.AzureFileVolumeSource": "k8s.io/kubernetes/pkg/api/v1.AzureFileVolumeSource", - "v1beta1.NetworkPolicySpec": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.NetworkPolicySpec", - "v1.PodTemplateSpec": "k8s.io/kubernetes/pkg/api/v1.PodTemplateSpec", - "v1.SecretVolumeSource": "k8s.io/kubernetes/pkg/api/v1.SecretVolumeSource", - "v1.PodSpec": "k8s.io/kubernetes/pkg/api/v1.PodSpec", - "v1.CephFSVolumeSource": "k8s.io/kubernetes/pkg/api/v1.CephFSVolumeSource", - "v1beta1.CPUTargetUtilization": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.CPUTargetUtilization", - "v1.Volume": "k8s.io/kubernetes/pkg/api/v1.Volume", - "v1beta1.Ingress": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.Ingress", - "v1beta1.HorizontalPodAutoscalerList": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.HorizontalPodAutoscalerList", - "v1.PersistentVolumeStatus": "k8s.io/kubernetes/pkg/api/v1.PersistentVolumeStatus", - "v1beta1.IDRange": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.IDRange", - "v2alpha1.JobCondition": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.JobCondition", - "v1beta1.IngressRule": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.IngressRule", - "v1alpha1.RoleRef": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.RoleRef", - "v1.PodAffinityTerm": "k8s.io/kubernetes/pkg/api/v1.PodAffinityTerm", - "v1.ObjectReference": "k8s.io/kubernetes/pkg/api/v1.ObjectReference", - "v1.ServiceStatus": "k8s.io/kubernetes/pkg/api/v1.ServiceStatus", - "v1.APIResource": "k8s.io/apimachinery/pkg/apis/meta/v1.APIResource", - "v1beta1.Scale": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.Scale", - "v1.AzureDiskVolumeSource": "k8s.io/kubernetes/pkg/api/v1.AzureDiskVolumeSource", - "v1beta1.SubjectAccessReviewStatus": "k8s.io/kubernetes/pkg/apis/authorization/v1beta1.SubjectAccessReviewStatus", - "v1.ConfigMap": "k8s.io/kubernetes/pkg/api/v1.ConfigMap", - "v1.CrossVersionObjectReference": "k8s.io/kubernetes/pkg/apis/autoscaling/v1.CrossVersionObjectReference", - "v1.APIVersions": "k8s.io/apimachinery/pkg/apis/meta/v1.APIVersions", - "v1alpha1.ClusterRoleList": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.ClusterRoleList", - "v1.Node": "k8s.io/kubernetes/pkg/api/v1.Node", - "resource.Quantity": "k8s.io/kubernetes/pkg/api/resource.Quantity", - "v1.Event": "k8s.io/kubernetes/pkg/api/v1.Event", - "v1.JobStatus": "k8s.io/kubernetes/pkg/apis/batch/v1.JobStatus", - "v1.PersistentVolumeSpec": "k8s.io/kubernetes/pkg/api/v1.PersistentVolumeSpec", - "v1beta1.SubjectAccessReviewSpec": "k8s.io/kubernetes/pkg/apis/authorization/v1beta1.SubjectAccessReviewSpec", - "v1.ResourceFieldSelector": "k8s.io/kubernetes/pkg/api/v1.ResourceFieldSelector", - "v1.EndpointSubset": "k8s.io/kubernetes/pkg/api/v1.EndpointSubset", - "v1alpha1.CertificateSigningRequestSpec": "k8s.io/kubernetes/pkg/apis/certificates/v1alpha1.CertificateSigningRequestSpec", - "v1.HostPathVolumeSource": "k8s.io/kubernetes/pkg/api/v1.HostPathVolumeSource", - "v1.LoadBalancerStatus": "k8s.io/kubernetes/pkg/api/v1.LoadBalancerStatus", - "v1beta1.HTTPIngressPath": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.HTTPIngressPath", - "v1beta1.Role": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.Role", - "v1beta1.DeploymentStrategy": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DeploymentStrategy", - "v1beta1.RunAsUserStrategyOptions": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.RunAsUserStrategyOptions", - "v1beta1.DeploymentSpec": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DeploymentSpec", - "v1.ExecAction": "k8s.io/kubernetes/pkg/api/v1.ExecAction", - "v1beta1.PodSecurityPolicySpec": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.PodSecurityPolicySpec", - "v1.HorizontalPodAutoscalerStatus": "k8s.io/kubernetes/pkg/apis/autoscaling/v1.HorizontalPodAutoscalerStatus", - "v1.PersistentVolumeList": "k8s.io/kubernetes/pkg/api/v1.PersistentVolumeList", - "v1alpha1.ClusterRole": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.ClusterRole", - "v1.JobSpec": "k8s.io/kubernetes/pkg/apis/batch/v1.JobSpec", - "v1beta1.DaemonSetSpec": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DaemonSetSpec", - "v2alpha1.CronJobList": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.CronJobList", - "v1.Endpoints": "k8s.io/kubernetes/pkg/api/v1.Endpoints", - "v1.SELinuxOptions": "k8s.io/kubernetes/pkg/api/v1.SELinuxOptions", - "v1beta1.SelfSubjectAccessReviewSpec": "k8s.io/kubernetes/pkg/apis/authorization/v1beta1.SelfSubjectAccessReviewSpec", - "v1beta1.ScaleStatus": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.ScaleStatus", - "v1.NodeSelectorTerm": "k8s.io/kubernetes/pkg/api/v1.NodeSelectorTerm", - "v1alpha1.CertificateSigningRequestStatus": "k8s.io/kubernetes/pkg/apis/certificates/v1alpha1.CertificateSigningRequestStatus", - "v1.StatusDetails": "k8s.io/apimachinery/pkg/apis/meta/v1.StatusDetails", - "v2alpha1.JobStatus": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.JobStatus", - "v1beta1.DeploymentRollback": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DeploymentRollback", - "v1.GlusterfsVolumeSource": "k8s.io/kubernetes/pkg/api/v1.GlusterfsVolumeSource", - "v1.ServiceAccountList": "k8s.io/kubernetes/pkg/api/v1.ServiceAccountList", - "v1.JobList": "k8s.io/kubernetes/pkg/apis/batch/v1.JobList", - "v1.EventList": "k8s.io/kubernetes/pkg/api/v1.EventList", - "v1.ContainerStateWaiting": "k8s.io/kubernetes/pkg/api/v1.ContainerStateWaiting", - "v1.APIResourceList": "k8s.io/apimachinery/pkg/apis/meta/v1.APIResourceList", - "v1.ContainerStatus": "k8s.io/kubernetes/pkg/api/v1.ContainerStatus", - "v2alpha1.JobList": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.JobList", - "v1.ConfigMapKeySelector": "k8s.io/kubernetes/pkg/api/v1.ConfigMapKeySelector", - "v1.PhotonPersistentDiskVolumeSource": "k8s.io/kubernetes/pkg/api/v1.PhotonPersistentDiskVolumeSource", - "v1.PodTemplateList": "k8s.io/kubernetes/pkg/api/v1.PodTemplateList", - "v1.PersistentVolumeClaimStatus": "k8s.io/kubernetes/pkg/api/v1.PersistentVolumeClaimStatus", - "v1.ServiceAccount": "k8s.io/kubernetes/pkg/api/v1.ServiceAccount", - "v1alpha1.CertificateSigningRequestList": "k8s.io/kubernetes/pkg/apis/certificates/v1alpha1.CertificateSigningRequestList", - "v1beta1.SupplementalGroupsStrategyOptions": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.SupplementalGroupsStrategyOptions", - "v1.HTTPHeader": "k8s.io/kubernetes/pkg/api/v1.HTTPHeader", - "version.Info": "k8s.io/apimachinery/pkg/version.Info", - "v1.EventSource": "k8s.io/kubernetes/pkg/api/v1.EventSource", - "v1alpha1.ClusterRoleBindingList": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.ClusterRoleBindingList", - "v1.OwnerReference": "k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference", - "v1beta1.ClusterRoleBindingList": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.ClusterRoleBindingList", - "v1beta1.ScaleSpec": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.ScaleSpec", - "v1.GitRepoVolumeSource": "k8s.io/kubernetes/pkg/api/v1.GitRepoVolumeSource", - "v1beta1.NetworkPolicy": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.NetworkPolicy", - "v1.ConfigMapEnvSource": "k8s.io/kubernetes/pkg/api/v1.ConfigMapEnvSource", - "v1.PodTemplate": "k8s.io/kubernetes/pkg/api/v1.PodTemplate", - "v1beta1.DeploymentCondition": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DeploymentCondition", - "v1beta1.PodDisruptionBudgetStatus": "k8s.io/kubernetes/pkg/apis/policy/v1beta1.PodDisruptionBudgetStatus", - "v1.EnvVar": "k8s.io/kubernetes/pkg/api/v1.EnvVar", - "v1.LimitRangeSpec": "k8s.io/kubernetes/pkg/api/v1.LimitRangeSpec", - "v1.DownwardAPIVolumeSource": "k8s.io/kubernetes/pkg/api/v1.DownwardAPIVolumeSource", - "v1.NodeDaemonEndpoints": "k8s.io/kubernetes/pkg/api/v1.NodeDaemonEndpoints", - "v1.ComponentCondition": "k8s.io/kubernetes/pkg/api/v1.ComponentCondition", - "v1alpha1.CertificateSigningRequestCondition": "k8s.io/kubernetes/pkg/apis/certificates/v1alpha1.CertificateSigningRequestCondition", - "v1.SecurityContext": "k8s.io/kubernetes/pkg/api/v1.SecurityContext", - "v1beta1.LocalSubjectAccessReview": "k8s.io/kubernetes/pkg/apis/authorization/v1beta1.LocalSubjectAccessReview", - "v1beta1.StatefulSetSpec": "k8s.io/kubernetes/pkg/apis/apps/v1beta1.StatefulSetSpec", - "v1.NodeAddress": "k8s.io/kubernetes/pkg/api/v1.NodeAddress", - "v1.QuobyteVolumeSource": "k8s.io/kubernetes/pkg/api/v1.QuobyteVolumeSource", - "v1.Capabilities": "k8s.io/kubernetes/pkg/api/v1.Capabilities", - "v1.GCEPersistentDiskVolumeSource": "k8s.io/kubernetes/pkg/api/v1.GCEPersistentDiskVolumeSource", - "v1beta1.ReplicaSet": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.ReplicaSet", - "v1beta1.HorizontalPodAutoscalerStatus": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.HorizontalPodAutoscalerStatus", - "v1beta1.PolicyRule": "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.PolicyRule", - "v1.ConfigMapList": "k8s.io/kubernetes/pkg/api/v1.ConfigMapList", - "v1.Lifecycle": "k8s.io/kubernetes/pkg/api/v1.Lifecycle", - "v1beta1.SelfSubjectAccessReview": "k8s.io/kubernetes/pkg/apis/authorization/v1beta1.SelfSubjectAccessReview", - "v2alpha1.CronJob": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.CronJob", - "v2alpha1.CronJobStatus": "k8s.io/kubernetes/pkg/apis/batch/v2alpha1.CronJobStatus", - "v1beta1.SubjectAccessReview": "k8s.io/kubernetes/pkg/apis/authorization/v1beta1.SubjectAccessReview", - "v1.Preconditions": "k8s.io/kubernetes/pkg/api/v1.Preconditions", - "v1beta1.DaemonSetList": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.DaemonSetList", - "v1.PersistentVolumeClaim": "k8s.io/kubernetes/pkg/api/v1.PersistentVolumeClaim", - "v1.Scale": "k8s.io/kubernetes/pkg/apis/autoscaling/v1.Scale", - "v1beta1.StatefulSetStatus": "k8s.io/kubernetes/pkg/apis/apps/v1beta1.StatefulSetStatus", - "v1.NFSVolumeSource": "k8s.io/kubernetes/pkg/api/v1.NFSVolumeSource", - "v1.ObjectFieldSelector": "k8s.io/kubernetes/pkg/api/v1.ObjectFieldSelector", - "v1.ResourceRequirements": "k8s.io/kubernetes/pkg/api/v1.ResourceRequirements", - "v1.WatchEvent": "k8s.io/apimachinery/pkg/apis/meta/v1.WatchEvent", - "v1.ReplicationControllerSpec": "k8s.io/kubernetes/pkg/api/v1.ReplicationControllerSpec", - "v1.HTTPGetAction": "k8s.io/kubernetes/pkg/api/v1.HTTPGetAction", - "v1beta1.RollbackConfig": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.RollbackConfig", - "v1beta1.TokenReviewSpec": "k8s.io/kubernetes/pkg/apis/authentication/v1beta1.TokenReviewSpec", - "v1.PodSecurityContext": "k8s.io/kubernetes/pkg/api/v1.PodSecurityContext", - "v1beta1.PodDisruptionBudgetList": "k8s.io/kubernetes/pkg/apis/policy/v1beta1.PodDisruptionBudgetList", - "v1.VolumeMount": "k8s.io/kubernetes/pkg/api/v1.VolumeMount", - "v1.ReplicationController": "k8s.io/kubernetes/pkg/api/v1.ReplicationController", - "v1.NamespaceList": "k8s.io/kubernetes/pkg/api/v1.NamespaceList", - "v1alpha1.ClusterRoleBinding": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.ClusterRoleBinding", - "v1.TCPSocketAction": "k8s.io/kubernetes/pkg/api/v1.TCPSocketAction", - "v1.Binding": "k8s.io/kubernetes/pkg/api/v1.Binding", - "v1beta1.ReplicaSetStatus": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.ReplicaSetStatus", - "intstr.IntOrString": "k8s.io/kubernetes/pkg/util/intstr.IntOrString", - "v1.EndpointsList": "k8s.io/kubernetes/pkg/api/v1.EndpointsList", - "v1.Container": "k8s.io/kubernetes/pkg/api/v1.Container", - "v1alpha1.RoleList": "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1.RoleList", - "v1.VsphereVirtualDiskVolumeSource": "k8s.io/kubernetes/pkg/api/v1.VsphereVirtualDiskVolumeSource", - "v1.NodeList": "k8s.io/kubernetes/pkg/api/v1.NodeList", - "v1.EmptyDirVolumeSource": "k8s.io/kubernetes/pkg/api/v1.EmptyDirVolumeSource", - "v1beta1.FSGroupStrategyOptions": "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.FSGroupStrategyOptions", - "v1.Status": "k8s.io/apimachinery/pkg/apis/meta/v1.Status", + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SelfSubjectAccessReview": "io.k8s.api.authorization.v1beta1.SelfSubjectAccessReview", + "io.k8s.kubernetes.pkg.api.v1.GitRepoVolumeSource": "io.k8s.api.core.v1.GitRepoVolumeSource", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.ExternalAdmissionHookConfigurationList": "io.k8s.api.admissionregistration.v1alpha1.ExternalAdmissionHookConfigurationList", + "io.k8s.kubernetes.pkg.api.v1.EndpointPort": "io.k8s.api.core.v1.EndpointPort", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.SupplementalGroupsStrategyOptions": "io.k8s.api.extensions.v1beta1.SupplementalGroupsStrategyOptions", + "io.k8s.kubernetes.pkg.api.v1.PodStatus": "io.k8s.api.core.v1.PodStatus", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.RoleBindingList": "io.k8s.api.rbac.v1beta1.RoleBindingList", + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.PodDisruptionBudgetSpec": "io.k8s.api.policy.v1beta1.PodDisruptionBudgetSpec", + "io.k8s.kubernetes.pkg.api.v1.HTTPGetAction": "io.k8s.api.core.v1.HTTPGetAction", + "io.k8s.kubernetes.pkg.apis.authorization.v1.ResourceAttributes": "io.k8s.api.authorization.v1.ResourceAttributes", + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeList": "io.k8s.api.core.v1.PersistentVolumeList", + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.CronJobSpec": "io.k8s.api.batch.v2alpha1.CronJobSpec", + "io.k8s.kubernetes.pkg.api.v1.CephFSVolumeSource": "io.k8s.api.core.v1.CephFSVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.Affinity": "io.k8s.api.core.v1.Affinity", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.PolicyRule": "io.k8s.api.rbac.v1beta1.PolicyRule", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSetSpec": "io.k8s.api.extensions.v1beta1.DaemonSetSpec", + "io.k8s.kubernetes.pkg.api.v1.ProjectedVolumeSource": "io.k8s.api.core.v1.ProjectedVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.TCPSocketAction": "io.k8s.api.core.v1.TCPSocketAction", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSet": "io.k8s.api.extensions.v1beta1.DaemonSet", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressList": "io.k8s.api.extensions.v1beta1.IngressList", + "io.k8s.kubernetes.pkg.api.v1.PodSpec": "io.k8s.api.core.v1.PodSpec", + "io.k8s.kubernetes.pkg.apis.authentication.v1.TokenReview": "io.k8s.api.authentication.v1.TokenReview", + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SubjectAccessReview": "io.k8s.api.authorization.v1beta1.SubjectAccessReview", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.ClusterRoleBinding": "io.k8s.api.rbac.v1alpha1.ClusterRoleBinding", + "io.k8s.kubernetes.pkg.api.v1.Node": "io.k8s.api.core.v1.Node", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.ServiceReference": "io.k8s.api.admissionregistration.v1alpha1.ServiceReference", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentStatus": "io.k8s.api.extensions.v1beta1.DeploymentStatus", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.RoleRef": "io.k8s.api.rbac.v1beta1.RoleRef", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.Scale": "io.k8s.api.apps.v1beta1.Scale", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.InitializerConfiguration": "io.k8s.api.admissionregistration.v1alpha1.InitializerConfiguration", + "io.k8s.kubernetes.pkg.api.v1.PhotonPersistentDiskVolumeSource": "io.k8s.api.core.v1.PhotonPersistentDiskVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.PreferredSchedulingTerm": "io.k8s.api.core.v1.PreferredSchedulingTerm", + "io.k8s.kubernetes.pkg.apis.batch.v1.JobSpec": "io.k8s.api.batch.v1.JobSpec", + "io.k8s.kubernetes.pkg.api.v1.EventSource": "io.k8s.api.core.v1.EventSource", + "io.k8s.kubernetes.pkg.api.v1.Container": "io.k8s.api.core.v1.Container", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.AdmissionHookClientConfig": "io.k8s.api.admissionregistration.v1alpha1.AdmissionHookClientConfig", + "io.k8s.kubernetes.pkg.api.v1.ResourceQuota": "io.k8s.api.core.v1.ResourceQuota", + "io.k8s.kubernetes.pkg.api.v1.SecretList": "io.k8s.api.core.v1.SecretList", + "io.k8s.kubernetes.pkg.api.v1.NodeSystemInfo": "io.k8s.api.core.v1.NodeSystemInfo", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.PolicyRule": "io.k8s.api.rbac.v1alpha1.PolicyRule", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSetSpec": "io.k8s.api.extensions.v1beta1.ReplicaSetSpec", + "io.k8s.kubernetes.pkg.api.v1.NodeStatus": "io.k8s.api.core.v1.NodeStatus", + "io.k8s.kubernetes.pkg.api.v1.ResourceQuotaList": "io.k8s.api.core.v1.ResourceQuotaList", + "io.k8s.kubernetes.pkg.api.v1.HostPathVolumeSource": "io.k8s.api.core.v1.HostPathVolumeSource", + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequest": "io.k8s.api.certificates.v1beta1.CertificateSigningRequest", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressRule": "io.k8s.api.extensions.v1beta1.IngressRule", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicyPeer": "io.k8s.api.extensions.v1beta1.NetworkPolicyPeer", + "io.k8s.kubernetes.pkg.apis.storage.v1.StorageClass": "io.k8s.api.storage.v1.StorageClass", + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicyPeer": "io.k8s.api.networking.v1.NetworkPolicyPeer", + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicyIngressRule": "io.k8s.api.networking.v1.NetworkPolicyIngressRule", + "io.k8s.kubernetes.pkg.api.v1.StorageOSPersistentVolumeSource": "io.k8s.api.core.v1.StorageOSPersistentVolumeSource", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicyIngressRule": "io.k8s.api.extensions.v1beta1.NetworkPolicyIngressRule", + "io.k8s.kubernetes.pkg.api.v1.PodAffinity": "io.k8s.api.core.v1.PodAffinity", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.RollbackConfig": "io.k8s.api.extensions.v1beta1.RollbackConfig", + "io.k8s.kubernetes.pkg.api.v1.PodList": "io.k8s.api.core.v1.PodList", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ScaleStatus": "io.k8s.api.extensions.v1beta1.ScaleStatus", + "io.k8s.kubernetes.pkg.api.v1.ComponentCondition": "io.k8s.api.core.v1.ComponentCondition", + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequestList": "io.k8s.api.certificates.v1beta1.CertificateSigningRequestList", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.ClusterRoleBindingList": "io.k8s.api.rbac.v1alpha1.ClusterRoleBindingList", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscalerCondition": "io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerCondition", + "io.k8s.kubernetes.pkg.api.v1.ServiceList": "io.k8s.api.core.v1.ServiceList", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.PodSecurityPolicy": "io.k8s.api.extensions.v1beta1.PodSecurityPolicy", + "io.k8s.kubernetes.pkg.apis.batch.v1.JobCondition": "io.k8s.api.batch.v1.JobCondition", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentStatus": "io.k8s.api.apps.v1beta1.DeploymentStatus", + "io.k8s.kubernetes.pkg.api.v1.Volume": "io.k8s.api.core.v1.Volume", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.RoleBindingList": "io.k8s.api.rbac.v1alpha1.RoleBindingList", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.Rule": "io.k8s.api.admissionregistration.v1alpha1.Rule", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.InitializerConfigurationList": "io.k8s.api.admissionregistration.v1alpha1.InitializerConfigurationList", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicy": "io.k8s.api.extensions.v1beta1.NetworkPolicy", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.ClusterRoleList": "io.k8s.api.rbac.v1alpha1.ClusterRoleList", + "io.k8s.kubernetes.pkg.api.v1.ObjectFieldSelector": "io.k8s.api.core.v1.ObjectFieldSelector", + "io.k8s.kubernetes.pkg.api.v1.EventList": "io.k8s.api.core.v1.EventList", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.MetricStatus": "io.k8s.api.autoscaling.v2alpha1.MetricStatus", + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicyPort": "io.k8s.api.networking.v1.NetworkPolicyPort", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.RoleList": "io.k8s.api.rbac.v1beta1.RoleList", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.RoleList": "io.k8s.api.rbac.v1alpha1.RoleList", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentStrategy": "io.k8s.api.apps.v1beta1.DeploymentStrategy", + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.CrossVersionObjectReference": "io.k8s.api.autoscaling.v1.CrossVersionObjectReference", + "io.k8s.kubernetes.pkg.api.v1.ConfigMapProjection": "io.k8s.api.core.v1.ConfigMapProjection", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.CrossVersionObjectReference": "io.k8s.api.autoscaling.v2alpha1.CrossVersionObjectReference", + "io.k8s.kubernetes.pkg.api.v1.LoadBalancerStatus": "io.k8s.api.core.v1.LoadBalancerStatus", + "io.k8s.kubernetes.pkg.api.v1.ISCSIVolumeSource": "io.k8s.api.core.v1.ISCSIVolumeSource", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.ControllerRevisionList": "io.k8s.api.apps.v1beta1.ControllerRevisionList", + "io.k8s.kubernetes.pkg.api.v1.EndpointSubset": "io.k8s.api.core.v1.EndpointSubset", + "io.k8s.kubernetes.pkg.api.v1.SELinuxOptions": "io.k8s.api.core.v1.SELinuxOptions", + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaimVolumeSource": "io.k8s.api.core.v1.PersistentVolumeClaimVolumeSource", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.MetricSpec": "io.k8s.api.autoscaling.v2alpha1.MetricSpec", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSetList": "io.k8s.api.apps.v1beta1.StatefulSetList", + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.ResourceAttributes": "io.k8s.api.authorization.v1beta1.ResourceAttributes", + "io.k8s.kubernetes.pkg.api.v1.Capabilities": "io.k8s.api.core.v1.Capabilities", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.Deployment": "io.k8s.api.extensions.v1beta1.Deployment", + "io.k8s.kubernetes.pkg.api.v1.Binding": "io.k8s.api.core.v1.Binding", + "io.k8s.kubernetes.pkg.api.v1.ReplicationControllerList": "io.k8s.api.core.v1.ReplicationControllerList", + "io.k8s.kubernetes.pkg.apis.authorization.v1.SelfSubjectAccessReview": "io.k8s.api.authorization.v1.SelfSubjectAccessReview", + "io.k8s.kubernetes.pkg.apis.authentication.v1beta1.UserInfo": "io.k8s.api.authentication.v1beta1.UserInfo", + "io.k8s.kubernetes.pkg.api.v1.HostAlias": "io.k8s.api.core.v1.HostAlias", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSetUpdateStrategy": "io.k8s.api.apps.v1beta1.StatefulSetUpdateStrategy", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressSpec": "io.k8s.api.extensions.v1beta1.IngressSpec", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentCondition": "io.k8s.api.extensions.v1beta1.DeploymentCondition", + "io.k8s.kubernetes.pkg.api.v1.GCEPersistentDiskVolumeSource": "io.k8s.api.core.v1.GCEPersistentDiskVolumeSource", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.ExternalAdmissionHook": "io.k8s.api.admissionregistration.v1alpha1.ExternalAdmissionHook", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.Scale": "io.k8s.api.extensions.v1beta1.Scale", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscalerStatus": "io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerStatus", + "io.k8s.kubernetes.pkg.api.v1.FlexVolumeSource": "io.k8s.api.core.v1.FlexVolumeSource", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.RollingUpdateDeployment": "io.k8s.api.extensions.v1beta1.RollingUpdateDeployment", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.ObjectMetricStatus": "io.k8s.api.autoscaling.v2alpha1.ObjectMetricStatus", + "io.k8s.kubernetes.pkg.api.v1.Event": "io.k8s.api.core.v1.Event", + "io.k8s.kubernetes.pkg.api.v1.ResourceQuotaSpec": "io.k8s.api.core.v1.ResourceQuotaSpec", + "io.k8s.kubernetes.pkg.api.v1.Handler": "io.k8s.api.core.v1.Handler", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressBackend": "io.k8s.api.extensions.v1beta1.IngressBackend", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.Role": "io.k8s.api.rbac.v1alpha1.Role", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.ObjectMetricSource": "io.k8s.api.autoscaling.v2alpha1.ObjectMetricSource", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.ResourceMetricStatus": "io.k8s.api.autoscaling.v2alpha1.ResourceMetricStatus", + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.HorizontalPodAutoscalerSpec": "io.k8s.api.autoscaling.v1.HorizontalPodAutoscalerSpec", + "io.k8s.kubernetes.pkg.api.v1.Lifecycle": "io.k8s.api.core.v1.Lifecycle", + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequestStatus": "io.k8s.api.certificates.v1beta1.CertificateSigningRequestStatus", + "io.k8s.kubernetes.pkg.api.v1.ContainerStateRunning": "io.k8s.api.core.v1.ContainerStateRunning", + "io.k8s.kubernetes.pkg.api.v1.ServiceAccountList": "io.k8s.api.core.v1.ServiceAccountList", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.HostPortRange": "io.k8s.api.extensions.v1beta1.HostPortRange", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.ControllerRevision": "io.k8s.api.apps.v1beta1.ControllerRevision", + "io.k8s.kubernetes.pkg.api.v1.ReplicationControllerSpec": "io.k8s.api.core.v1.ReplicationControllerSpec", + "io.k8s.kubernetes.pkg.api.v1.ContainerStateTerminated": "io.k8s.api.core.v1.ContainerStateTerminated", + "io.k8s.kubernetes.pkg.api.v1.ReplicationControllerStatus": "io.k8s.api.core.v1.ReplicationControllerStatus", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSetList": "io.k8s.api.extensions.v1beta1.DaemonSetList", + "io.k8s.kubernetes.pkg.apis.authorization.v1.SelfSubjectAccessReviewSpec": "io.k8s.api.authorization.v1.SelfSubjectAccessReviewSpec", + "io.k8s.kubernetes.pkg.api.v1.ComponentStatusList": "io.k8s.api.core.v1.ComponentStatusList", + "io.k8s.kubernetes.pkg.api.v1.ContainerStateWaiting": "io.k8s.api.core.v1.ContainerStateWaiting", + "io.k8s.kubernetes.pkg.api.v1.VolumeMount": "io.k8s.api.core.v1.VolumeMount", + "io.k8s.kubernetes.pkg.api.v1.Secret": "io.k8s.api.core.v1.Secret", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.ClusterRoleList": "io.k8s.api.rbac.v1beta1.ClusterRoleList", + "io.k8s.kubernetes.pkg.api.v1.ConfigMapList": "io.k8s.api.core.v1.ConfigMapList", + "io.k8s.kubernetes.pkg.apis.storage.v1beta1.StorageClassList": "io.k8s.api.storage.v1beta1.StorageClassList", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.HTTPIngressPath": "io.k8s.api.extensions.v1beta1.HTTPIngressPath", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.ClusterRole": "io.k8s.api.rbac.v1alpha1.ClusterRole", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.ResourceMetricSource": "io.k8s.api.autoscaling.v2alpha1.ResourceMetricSource", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentRollback": "io.k8s.api.extensions.v1beta1.DeploymentRollback", + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaimSpec": "io.k8s.api.core.v1.PersistentVolumeClaimSpec", + "io.k8s.kubernetes.pkg.api.v1.ReplicationController": "io.k8s.api.core.v1.ReplicationController", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSetSpec": "io.k8s.api.apps.v1beta1.StatefulSetSpec", + "io.k8s.kubernetes.pkg.api.v1.SecurityContext": "io.k8s.api.core.v1.SecurityContext", + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicySpec": "io.k8s.api.networking.v1.NetworkPolicySpec", + "io.k8s.kubernetes.pkg.api.v1.LocalObjectReference": "io.k8s.api.core.v1.LocalObjectReference", + "io.k8s.kubernetes.pkg.api.v1.RBDVolumeSource": "io.k8s.api.core.v1.RBDVolumeSource", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicySpec": "io.k8s.api.extensions.v1beta1.NetworkPolicySpec", + "io.k8s.kubernetes.pkg.api.v1.KeyToPath": "io.k8s.api.core.v1.KeyToPath", + "io.k8s.kubernetes.pkg.api.v1.WeightedPodAffinityTerm": "io.k8s.api.core.v1.WeightedPodAffinityTerm", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.PodsMetricStatus": "io.k8s.api.autoscaling.v2alpha1.PodsMetricStatus", + "io.k8s.kubernetes.pkg.api.v1.NodeAddress": "io.k8s.api.core.v1.NodeAddress", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.Ingress": "io.k8s.api.extensions.v1beta1.Ingress", + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.PodDisruptionBudget": "io.k8s.api.policy.v1beta1.PodDisruptionBudget", + "io.k8s.kubernetes.pkg.api.v1.ServicePort": "io.k8s.api.core.v1.ServicePort", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IDRange": "io.k8s.api.extensions.v1beta1.IDRange", + "io.k8s.kubernetes.pkg.api.v1.SecretEnvSource": "io.k8s.api.core.v1.SecretEnvSource", + "io.k8s.kubernetes.pkg.api.v1.NodeSelector": "io.k8s.api.core.v1.NodeSelector", + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaimStatus": "io.k8s.api.core.v1.PersistentVolumeClaimStatus", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentSpec": "io.k8s.api.apps.v1beta1.DeploymentSpec", + "io.k8s.kubernetes.pkg.apis.authorization.v1.NonResourceAttributes": "io.k8s.api.authorization.v1.NonResourceAttributes", + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.ScaleStatus": "io.k8s.api.autoscaling.v1.ScaleStatus", + "io.k8s.kubernetes.pkg.api.v1.PodCondition": "io.k8s.api.core.v1.PodCondition", + "io.k8s.kubernetes.pkg.api.v1.PodTemplateSpec": "io.k8s.api.core.v1.PodTemplateSpec", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSet": "io.k8s.api.apps.v1beta1.StatefulSet", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicyPort": "io.k8s.api.extensions.v1beta1.NetworkPolicyPort", + "io.k8s.kubernetes.pkg.apis.authentication.v1beta1.TokenReview": "io.k8s.api.authentication.v1beta1.TokenReview", + "io.k8s.kubernetes.pkg.api.v1.LimitRangeSpec": "io.k8s.api.core.v1.LimitRangeSpec", + "io.k8s.kubernetes.pkg.api.v1.FlockerVolumeSource": "io.k8s.api.core.v1.FlockerVolumeSource", + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.Eviction": "io.k8s.api.policy.v1beta1.Eviction", + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaimList": "io.k8s.api.core.v1.PersistentVolumeClaimList", + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequestCondition": "io.k8s.api.certificates.v1beta1.CertificateSigningRequestCondition", + "io.k8s.kubernetes.pkg.api.v1.DownwardAPIVolumeFile": "io.k8s.api.core.v1.DownwardAPIVolumeFile", + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.LocalSubjectAccessReview": "io.k8s.api.authorization.v1beta1.LocalSubjectAccessReview", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.ScaleStatus": "io.k8s.api.apps.v1beta1.ScaleStatus", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.HTTPIngressRuleValue": "io.k8s.api.extensions.v1beta1.HTTPIngressRuleValue", + "io.k8s.kubernetes.pkg.apis.batch.v1.Job": "io.k8s.api.batch.v1.Job", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.ExternalAdmissionHookConfiguration": "io.k8s.api.admissionregistration.v1alpha1.ExternalAdmissionHookConfiguration", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.RoleBinding": "io.k8s.api.rbac.v1beta1.RoleBinding", + "io.k8s.kubernetes.pkg.api.v1.FCVolumeSource": "io.k8s.api.core.v1.FCVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.EndpointAddress": "io.k8s.api.core.v1.EndpointAddress", + "io.k8s.kubernetes.pkg.api.v1.ContainerPort": "io.k8s.api.core.v1.ContainerPort", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.ClusterRoleBinding": "io.k8s.api.rbac.v1beta1.ClusterRoleBinding", + "io.k8s.kubernetes.pkg.api.v1.GlusterfsVolumeSource": "io.k8s.api.core.v1.GlusterfsVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.ResourceRequirements": "io.k8s.api.core.v1.ResourceRequirements", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.RollingUpdateDeployment": "io.k8s.api.apps.v1beta1.RollingUpdateDeployment", + "io.k8s.kubernetes.pkg.api.v1.NamespaceStatus": "io.k8s.api.core.v1.NamespaceStatus", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.RunAsUserStrategyOptions": "io.k8s.api.extensions.v1beta1.RunAsUserStrategyOptions", + "io.k8s.kubernetes.pkg.api.v1.Namespace": "io.k8s.api.core.v1.Namespace", + "io.k8s.kubernetes.pkg.apis.authorization.v1.SubjectAccessReviewSpec": "io.k8s.api.authorization.v1.SubjectAccessReviewSpec", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscaler": "io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscaler", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSetCondition": "io.k8s.api.extensions.v1beta1.ReplicaSetCondition", + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.HorizontalPodAutoscalerStatus": "io.k8s.api.autoscaling.v1.HorizontalPodAutoscalerStatus", + "io.k8s.kubernetes.pkg.apis.authentication.v1.TokenReviewStatus": "io.k8s.api.authentication.v1.TokenReviewStatus", + "io.k8s.kubernetes.pkg.api.v1.PersistentVolume": "io.k8s.api.core.v1.PersistentVolume", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.FSGroupStrategyOptions": "io.k8s.api.extensions.v1beta1.FSGroupStrategyOptions", + "io.k8s.kubernetes.pkg.api.v1.PodSecurityContext": "io.k8s.api.core.v1.PodSecurityContext", + "io.k8s.kubernetes.pkg.api.v1.PodTemplate": "io.k8s.api.core.v1.PodTemplate", + "io.k8s.kubernetes.pkg.apis.authorization.v1.LocalSubjectAccessReview": "io.k8s.api.authorization.v1.LocalSubjectAccessReview", + "io.k8s.kubernetes.pkg.api.v1.StorageOSVolumeSource": "io.k8s.api.core.v1.StorageOSVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.NodeSelectorTerm": "io.k8s.api.core.v1.NodeSelectorTerm", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.Role": "io.k8s.api.rbac.v1beta1.Role", + "io.k8s.kubernetes.pkg.api.v1.ContainerStatus": "io.k8s.api.core.v1.ContainerStatus", + "io.k8s.kubernetes.pkg.apis.authorization.v1.SubjectAccessReviewStatus": "io.k8s.api.authorization.v1.SubjectAccessReviewStatus", + "io.k8s.kubernetes.pkg.apis.authentication.v1.TokenReviewSpec": "io.k8s.api.authentication.v1.TokenReviewSpec", + "io.k8s.kubernetes.pkg.api.v1.ConfigMap": "io.k8s.api.core.v1.ConfigMap", + "io.k8s.kubernetes.pkg.api.v1.ServiceStatus": "io.k8s.api.core.v1.ServiceStatus", + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SelfSubjectAccessReviewSpec": "io.k8s.api.authorization.v1beta1.SelfSubjectAccessReviewSpec", + "io.k8s.kubernetes.pkg.api.v1.CinderVolumeSource": "io.k8s.api.core.v1.CinderVolumeSource", + "io.k8s.kubernetes.pkg.apis.settings.v1alpha1.PodPresetSpec": "io.k8s.api.settings.v1alpha1.PodPresetSpec", + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.NonResourceAttributes": "io.k8s.api.authorization.v1beta1.NonResourceAttributes", + "io.k8s.kubernetes.pkg.api.v1.ContainerImage": "io.k8s.api.core.v1.ContainerImage", + "io.k8s.kubernetes.pkg.api.v1.ReplicationControllerCondition": "io.k8s.api.core.v1.ReplicationControllerCondition", + "io.k8s.kubernetes.pkg.api.v1.EmptyDirVolumeSource": "io.k8s.api.core.v1.EmptyDirVolumeSource", + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.HorizontalPodAutoscalerList": "io.k8s.api.autoscaling.v1.HorizontalPodAutoscalerList", + "io.k8s.kubernetes.pkg.apis.batch.v1.JobList": "io.k8s.api.batch.v1.JobList", + "io.k8s.kubernetes.pkg.api.v1.NFSVolumeSource": "io.k8s.api.core.v1.NFSVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.Pod": "io.k8s.api.core.v1.Pod", + "io.k8s.kubernetes.pkg.api.v1.ObjectReference": "io.k8s.api.core.v1.ObjectReference", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.Deployment": "io.k8s.api.apps.v1beta1.Deployment", + "io.k8s.kubernetes.pkg.apis.storage.v1.StorageClassList": "io.k8s.api.storage.v1.StorageClassList", + "io.k8s.kubernetes.pkg.api.v1.AttachedVolume": "io.k8s.api.core.v1.AttachedVolume", + "io.k8s.kubernetes.pkg.api.v1.AWSElasticBlockStoreVolumeSource": "io.k8s.api.core.v1.AWSElasticBlockStoreVolumeSource", + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.CronJobList": "io.k8s.api.batch.v2alpha1.CronJobList", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentSpec": "io.k8s.api.extensions.v1beta1.DeploymentSpec", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.PodSecurityPolicyList": "io.k8s.api.extensions.v1beta1.PodSecurityPolicyList", + "io.k8s.kubernetes.pkg.api.v1.PodAffinityTerm": "io.k8s.api.core.v1.PodAffinityTerm", + "io.k8s.kubernetes.pkg.api.v1.HTTPHeader": "io.k8s.api.core.v1.HTTPHeader", + "io.k8s.kubernetes.pkg.api.v1.ConfigMapKeySelector": "io.k8s.api.core.v1.ConfigMapKeySelector", + "io.k8s.kubernetes.pkg.api.v1.SecretKeySelector": "io.k8s.api.core.v1.SecretKeySelector", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentList": "io.k8s.api.extensions.v1beta1.DeploymentList", + "io.k8s.kubernetes.pkg.apis.authentication.v1.UserInfo": "io.k8s.api.authentication.v1.UserInfo", + "io.k8s.kubernetes.pkg.api.v1.LoadBalancerIngress": "io.k8s.api.core.v1.LoadBalancerIngress", + "io.k8s.kubernetes.pkg.api.v1.DaemonEndpoint": "io.k8s.api.core.v1.DaemonEndpoint", + "io.k8s.kubernetes.pkg.api.v1.NodeSelectorRequirement": "io.k8s.api.core.v1.NodeSelectorRequirement", + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.CronJobStatus": "io.k8s.api.batch.v2alpha1.CronJobStatus", + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.Scale": "io.k8s.api.autoscaling.v1.Scale", + "io.k8s.kubernetes.pkg.api.v1.ScaleIOVolumeSource": "io.k8s.api.core.v1.ScaleIOVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.PodAntiAffinity": "io.k8s.api.core.v1.PodAntiAffinity", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.PodSecurityPolicySpec": "io.k8s.api.extensions.v1beta1.PodSecurityPolicySpec", + "io.k8s.kubernetes.pkg.apis.settings.v1alpha1.PodPresetList": "io.k8s.api.settings.v1alpha1.PodPresetList", + "io.k8s.kubernetes.pkg.api.v1.NodeAffinity": "io.k8s.api.core.v1.NodeAffinity", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentCondition": "io.k8s.api.apps.v1beta1.DeploymentCondition", + "io.k8s.kubernetes.pkg.api.v1.NodeSpec": "io.k8s.api.core.v1.NodeSpec", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.StatefulSetStatus": "io.k8s.api.apps.v1beta1.StatefulSetStatus", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.RuleWithOperations": "io.k8s.api.admissionregistration.v1alpha1.RuleWithOperations", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressStatus": "io.k8s.api.extensions.v1beta1.IngressStatus", + "io.k8s.kubernetes.pkg.api.v1.LimitRangeList": "io.k8s.api.core.v1.LimitRangeList", + "io.k8s.kubernetes.pkg.api.v1.AzureDiskVolumeSource": "io.k8s.api.core.v1.AzureDiskVolumeSource", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSetStatus": "io.k8s.api.extensions.v1beta1.ReplicaSetStatus", + "io.k8s.kubernetes.pkg.api.v1.ComponentStatus": "io.k8s.api.core.v1.ComponentStatus", + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.HorizontalPodAutoscaler": "io.k8s.api.autoscaling.v1.HorizontalPodAutoscaler", + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicy": "io.k8s.api.networking.v1.NetworkPolicy", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.RollbackConfig": "io.k8s.api.apps.v1beta1.RollbackConfig", + "io.k8s.kubernetes.pkg.api.v1.NodeCondition": "io.k8s.api.core.v1.NodeCondition", + "io.k8s.kubernetes.pkg.api.v1.DownwardAPIProjection": "io.k8s.api.core.v1.DownwardAPIProjection", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.SELinuxStrategyOptions": "io.k8s.api.extensions.v1beta1.SELinuxStrategyOptions", + "io.k8s.kubernetes.pkg.api.v1.NamespaceSpec": "io.k8s.api.core.v1.NamespaceSpec", + "io.k8s.kubernetes.pkg.apis.certificates.v1beta1.CertificateSigningRequestSpec": "io.k8s.api.certificates.v1beta1.CertificateSigningRequestSpec", + "io.k8s.kubernetes.pkg.api.v1.ServiceSpec": "io.k8s.api.core.v1.ServiceSpec", + "io.k8s.kubernetes.pkg.apis.authorization.v1.SubjectAccessReview": "io.k8s.api.authorization.v1.SubjectAccessReview", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentList": "io.k8s.api.apps.v1beta1.DeploymentList", + "io.k8s.kubernetes.pkg.api.v1.Toleration": "io.k8s.api.core.v1.Toleration", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.NetworkPolicyList": "io.k8s.api.extensions.v1beta1.NetworkPolicyList", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.PodsMetricSource": "io.k8s.api.autoscaling.v2alpha1.PodsMetricSource", + "io.k8s.kubernetes.pkg.api.v1.EnvFromSource": "io.k8s.api.core.v1.EnvFromSource", + "io.k8s.kubernetes.pkg.apis.autoscaling.v1.ScaleSpec": "io.k8s.api.autoscaling.v1.ScaleSpec", + "io.k8s.kubernetes.pkg.api.v1.PodTemplateList": "io.k8s.api.core.v1.PodTemplateList", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscalerSpec": "io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerSpec", + "io.k8s.kubernetes.pkg.api.v1.SecretProjection": "io.k8s.api.core.v1.SecretProjection", + "io.k8s.kubernetes.pkg.api.v1.ResourceFieldSelector": "io.k8s.api.core.v1.ResourceFieldSelector", + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeSpec": "io.k8s.api.core.v1.PersistentVolumeSpec", + "io.k8s.kubernetes.pkg.api.v1.ConfigMapVolumeSource": "io.k8s.api.core.v1.ConfigMapVolumeSource", + "io.k8s.kubernetes.pkg.apis.autoscaling.v2alpha1.HorizontalPodAutoscalerList": "io.k8s.api.autoscaling.v2alpha1.HorizontalPodAutoscalerList", + "io.k8s.kubernetes.pkg.apis.authentication.v1beta1.TokenReviewStatus": "io.k8s.api.authentication.v1beta1.TokenReviewStatus", + "io.k8s.kubernetes.pkg.apis.networking.v1.NetworkPolicyList": "io.k8s.api.networking.v1.NetworkPolicyList", + "io.k8s.kubernetes.pkg.api.v1.Endpoints": "io.k8s.api.core.v1.Endpoints", + "io.k8s.kubernetes.pkg.api.v1.LimitRangeItem": "io.k8s.api.core.v1.LimitRangeItem", + "io.k8s.kubernetes.pkg.api.v1.ServiceAccount": "io.k8s.api.core.v1.ServiceAccount", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ScaleSpec": "io.k8s.api.extensions.v1beta1.ScaleSpec", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.IngressTLS": "io.k8s.api.extensions.v1beta1.IngressTLS", + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.CronJob": "io.k8s.api.batch.v2alpha1.CronJob", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.Subject": "io.k8s.api.rbac.v1alpha1.Subject", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSetStatus": "io.k8s.api.extensions.v1beta1.DaemonSetStatus", + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.PodDisruptionBudgetList": "io.k8s.api.policy.v1beta1.PodDisruptionBudgetList", + "io.k8s.kubernetes.pkg.api.v1.VsphereVirtualDiskVolumeSource": "io.k8s.api.core.v1.VsphereVirtualDiskVolumeSource", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.RoleRef": "io.k8s.api.rbac.v1alpha1.RoleRef", + "io.k8s.kubernetes.pkg.api.v1.PortworxVolumeSource": "io.k8s.api.core.v1.PortworxVolumeSource", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSetList": "io.k8s.api.extensions.v1beta1.ReplicaSetList", + "io.k8s.kubernetes.pkg.api.v1.VolumeProjection": "io.k8s.api.core.v1.VolumeProjection", + "io.k8s.kubernetes.pkg.apis.storage.v1beta1.StorageClass": "io.k8s.api.storage.v1beta1.StorageClass", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.ReplicaSet": "io.k8s.api.extensions.v1beta1.ReplicaSet", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.DeploymentRollback": "io.k8s.api.apps.v1beta1.DeploymentRollback", + "io.k8s.kubernetes.pkg.apis.rbac.v1alpha1.RoleBinding": "io.k8s.api.rbac.v1alpha1.RoleBinding", + "io.k8s.kubernetes.pkg.api.v1.AzureFileVolumeSource": "io.k8s.api.core.v1.AzureFileVolumeSource", + "io.k8s.kubernetes.pkg.apis.policy.v1beta1.PodDisruptionBudgetStatus": "io.k8s.api.policy.v1beta1.PodDisruptionBudgetStatus", + "io.k8s.kubernetes.pkg.apis.authentication.v1beta1.TokenReviewSpec": "io.k8s.api.authentication.v1beta1.TokenReviewSpec", + "io.k8s.kubernetes.pkg.api.v1.EndpointsList": "io.k8s.api.core.v1.EndpointsList", + "io.k8s.kubernetes.pkg.api.v1.ConfigMapEnvSource": "io.k8s.api.core.v1.ConfigMapEnvSource", + "io.k8s.kubernetes.pkg.apis.batch.v2alpha1.JobTemplateSpec": "io.k8s.api.batch.v2alpha1.JobTemplateSpec", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DaemonSetUpdateStrategy": "io.k8s.api.extensions.v1beta1.DaemonSetUpdateStrategy", + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SubjectAccessReviewSpec": "io.k8s.api.authorization.v1beta1.SubjectAccessReviewSpec", + "io.k8s.kubernetes.pkg.api.v1.LocalVolumeSource": "io.k8s.api.core.v1.LocalVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.ContainerState": "io.k8s.api.core.v1.ContainerState", + "io.k8s.kubernetes.pkg.api.v1.Service": "io.k8s.api.core.v1.Service", + "io.k8s.kubernetes.pkg.api.v1.ExecAction": "io.k8s.api.core.v1.ExecAction", + "io.k8s.kubernetes.pkg.api.v1.Taint": "io.k8s.api.core.v1.Taint", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.Subject": "io.k8s.api.rbac.v1beta1.Subject", + "io.k8s.kubernetes.pkg.apis.authorization.v1beta1.SubjectAccessReviewStatus": "io.k8s.api.authorization.v1beta1.SubjectAccessReviewStatus", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.ClusterRoleBindingList": "io.k8s.api.rbac.v1beta1.ClusterRoleBindingList", + "io.k8s.kubernetes.pkg.api.v1.DownwardAPIVolumeSource": "io.k8s.api.core.v1.DownwardAPIVolumeSource", + "io.k8s.kubernetes.pkg.apis.batch.v1.JobStatus": "io.k8s.api.batch.v1.JobStatus", + "io.k8s.kubernetes.pkg.api.v1.ResourceQuotaStatus": "io.k8s.api.core.v1.ResourceQuotaStatus", + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeStatus": "io.k8s.api.core.v1.PersistentVolumeStatus", + "io.k8s.kubernetes.pkg.api.v1.PersistentVolumeClaim": "io.k8s.api.core.v1.PersistentVolumeClaim", + "io.k8s.kubernetes.pkg.api.v1.NodeDaemonEndpoints": "io.k8s.api.core.v1.NodeDaemonEndpoints", + "io.k8s.kubernetes.pkg.api.v1.EnvVar": "io.k8s.api.core.v1.EnvVar", + "io.k8s.kubernetes.pkg.api.v1.SecretVolumeSource": "io.k8s.api.core.v1.SecretVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.EnvVarSource": "io.k8s.api.core.v1.EnvVarSource", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.RollingUpdateStatefulSetStrategy": "io.k8s.api.apps.v1beta1.RollingUpdateStatefulSetStrategy", + "io.k8s.kubernetes.pkg.apis.rbac.v1beta1.ClusterRole": "io.k8s.api.rbac.v1beta1.ClusterRole", + "io.k8s.kubernetes.pkg.apis.admissionregistration.v1alpha1.Initializer": "io.k8s.api.admissionregistration.v1alpha1.Initializer", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.DeploymentStrategy": "io.k8s.api.extensions.v1beta1.DeploymentStrategy", + "io.k8s.kubernetes.pkg.apis.apps.v1beta1.ScaleSpec": "io.k8s.api.apps.v1beta1.ScaleSpec", + "io.k8s.kubernetes.pkg.apis.settings.v1alpha1.PodPreset": "io.k8s.api.settings.v1alpha1.PodPreset", + "io.k8s.kubernetes.pkg.api.v1.Probe": "io.k8s.api.core.v1.Probe", + "io.k8s.kubernetes.pkg.api.v1.NamespaceList": "io.k8s.api.core.v1.NamespaceList", + "io.k8s.kubernetes.pkg.api.v1.QuobyteVolumeSource": "io.k8s.api.core.v1.QuobyteVolumeSource", + "io.k8s.kubernetes.pkg.api.v1.NodeList": "io.k8s.api.core.v1.NodeList", + "io.k8s.kubernetes.pkg.apis.extensions.v1beta1.RollingUpdateDaemonSet": "io.k8s.api.extensions.v1beta1.RollingUpdateDaemonSet", + "io.k8s.kubernetes.pkg.api.v1.LimitRange": "io.k8s.api.core.v1.LimitRange", } for k, v := range compatibilityMap { diff --git a/cmd/kube-apiserver/app/testing/BUILD b/cmd/kube-apiserver/app/testing/BUILD index 7e4e275b7b513..a4e9b6ae21b2b 100644 --- a/cmd/kube-apiserver/app/testing/BUILD +++ b/cmd/kube-apiserver/app/testing/BUILD @@ -22,6 +22,9 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apiserver/pkg/features:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", ], diff --git a/cmd/kube-apiserver/app/testing/server_test.go b/cmd/kube-apiserver/app/testing/server_test.go index cc228e57c24d1..575126426a6d3 100644 --- a/cmd/kube-apiserver/app/testing/server_test.go +++ b/cmd/kube-apiserver/app/testing/server_test.go @@ -33,6 +33,9 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" ) @@ -150,6 +153,8 @@ func TestCRDShadowGroup(t *testing.T) { } func TestCRD(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.Initializers, true)() + config, tearDown := StartTestServerOrDie(t) defer tearDown() diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 5638a0e0afeca..abbbb97943526 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -128,8 +128,14 @@ func StartTestServerOrDie(t *testing.T) (*restclient.Config, TearDownFunc) { // retry test because the bind might fail due to a race with another process // binding to the port. We cannot listen to :0 (then the kernel would give us // a port which is free for sure), so we need this workaround. + + var err error + for retry := 0; retry < 5 && !t.Failed(); retry++ { - config, td, err := StartTestServer(t) + var config *restclient.Config + var td TearDownFunc + + config, td, err = StartTestServer(t) if err == nil { return config, td } @@ -139,7 +145,7 @@ func StartTestServerOrDie(t *testing.T) (*restclient.Config, TearDownFunc) { t.Logf("Bind error, retrying...") } - t.Fatalf("Failed to launch server") + t.Fatalf("Failed to launch server: %v", err) return nil, nil } diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index 1f11c248de8c0..92fadcacf54a3 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -91,6 +91,7 @@ go_library( "//pkg/volume/gce_pd:go_default_library", "//pkg/volume/glusterfs:go_default_library", "//pkg/volume/host_path:go_default_library", + "//pkg/volume/iscsi:go_default_library", "//pkg/volume/local:go_default_library", "//pkg/volume/nfs:go_default_library", "//pkg/volume/photon_pd:go_default_library", diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 2c5e2518fc0de..6560eec44f7c9 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -116,47 +116,15 @@ func Run(s *options.CMServer) error { } else { glog.Errorf("unable to register configz: %s", err) } - kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig) + + kubeClient, leaderElectionClient, kubeconfig, err := createClients(s) if err != nil { return err } - kubeconfig.ContentConfig.ContentType = s.ContentType - // Override kubeconfig qps/burst settings from flags - kubeconfig.QPS = s.KubeAPIQPS - kubeconfig.Burst = int(s.KubeAPIBurst) - kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "controller-manager")) - if err != nil { - glog.Fatalf("Invalid API configuration: %v", err) - } - leaderElectionClient := kubernetes.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election")) + go startHTTP(s) - go func() { - mux := http.NewServeMux() - healthz.InstallHandler(mux) - if s.EnableProfiling { - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - if s.EnableContentionProfiling { - goruntime.SetBlockProfileRate(1) - } - } - configz.InstallHandler(mux) - mux.Handle("/metrics", prometheus.Handler()) - - server := &http.Server{ - Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))), - Handler: mux, - } - glog.Fatal(server.ListenAndServe()) - }() - - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) - eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")}) - recorder := eventBroadcaster.NewRecorder(api.Scheme, v1.EventSource{Component: "controller-manager"}) + recorder := createRecorder(kubeClient) run := func(stop <-chan struct{}) { rootClientBuilder := controller.SimpleControllerClientBuilder{ @@ -230,6 +198,53 @@ func Run(s *options.CMServer) error { panic("unreachable") } +func startHTTP(s *options.CMServer) { + mux := http.NewServeMux() + healthz.InstallHandler(mux) + if s.EnableProfiling { + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + if s.EnableContentionProfiling { + goruntime.SetBlockProfileRate(1) + } + } + configz.InstallHandler(mux) + mux.Handle("/metrics", prometheus.Handler()) + + server := &http.Server{ + Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))), + Handler: mux, + } + glog.Fatal(server.ListenAndServe()) +} + +func createRecorder(kubeClient *clientset.Clientset) record.EventRecorder { + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")}) + return eventBroadcaster.NewRecorder(api.Scheme, v1.EventSource{Component: "controller-manager"}) +} + +func createClients(s *options.CMServer) (*clientset.Clientset, *clientset.Clientset, *restclient.Config, error) { + kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig) + if err != nil { + return nil, nil, nil, err + } + + kubeconfig.ContentConfig.ContentType = s.ContentType + // Override kubeconfig qps/burst settings from flags + kubeconfig.QPS = s.KubeAPIQPS + kubeconfig.Burst = int(s.KubeAPIBurst) + kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "controller-manager")) + if err != nil { + glog.Fatalf("Invalid API configuration: %v", err) + } + leaderElectionClient := kubernetes.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election")) + return kubeClient, leaderElectionClient, kubeconfig, nil +} + type ControllerContext struct { // ClientBuilder will provide a client for this controller to use ClientBuilder controller.ControllerClientBuilder diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index 0d6f3dca1fe73..46c9309f51dc2 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -175,7 +175,8 @@ func startAttachDetachController(ctx ControllerContext) (bool, error) { ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), ctx.InformerFactory.Core().V1().PersistentVolumes(), ctx.Cloud, - ProbeAttachableVolumePlugins(ctx.Options.VolumeConfiguration), + ProbeAttachableVolumePlugins(), + GetDynamicPluginProber(ctx.Options.VolumeConfiguration), ctx.Options.DisableAttachDetachReconcilerSync, ctx.Options.ReconcilerSyncLoopPeriod.Duration, attachdetach.DefaultTimerConfig, diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 5b834261fa74d..e5acccd066b6b 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -69,9 +69,6 @@ func NewCMServer() *CMServer { ConcurrentDeploymentSyncs: 5, ConcurrentNamespaceSyncs: 10, ConcurrentSATokenSyncs: 5, - LookupCacheSizeForRC: 4096, - LookupCacheSizeForRS: 4096, - LookupCacheSizeForDaemonSet: 1024, ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, @@ -144,15 +141,6 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled fs.Int32Var(&s.ConcurrentDeploymentSyncs, "concurrent-deployment-syncs", s.ConcurrentDeploymentSyncs, "The number of deployment objects that are allowed to sync concurrently. Larger number = more responsive deployments, but more CPU (and network) load") fs.Int32Var(&s.ConcurrentNamespaceSyncs, "concurrent-namespace-syncs", s.ConcurrentNamespaceSyncs, "The number of namespace objects that are allowed to sync concurrently. Larger number = more responsive namespace termination, but more CPU (and network) load") fs.Int32Var(&s.ConcurrentSATokenSyncs, "concurrent-serviceaccount-token-syncs", s.ConcurrentSATokenSyncs, "The number of service account token objects that are allowed to sync concurrently. Larger number = more responsive token generation, but more CPU (and network) load") - // TODO(#43388): Remove the following flag 6 months after v1.6.0 is released. - fs.Int32Var(&s.LookupCacheSizeForRC, "replication-controller-lookup-cache-size", s.LookupCacheSizeForRC, "This flag is deprecated and will be removed in future releases. ReplicationController no longer requires a lookup cache.") - fs.MarkDeprecated("replication-controller-lookup-cache-size", "This flag is deprecated and will be removed in future releases. ReplicationController no longer requires a lookup cache.") - // TODO(#43388): Remove the following flag 6 months after v1.6.0 is released. - fs.Int32Var(&s.LookupCacheSizeForRS, "replicaset-lookup-cache-size", s.LookupCacheSizeForRS, "This flag is deprecated and will be removed in future releases. ReplicaSet no longer requires a lookup cache.") - fs.MarkDeprecated("replicaset-lookup-cache-size", "This flag is deprecated and will be removed in future releases. ReplicaSet no longer requires a lookup cache.") - // TODO(#43388): Remove the following flag 6 months after v1.6.0 is released. - fs.Int32Var(&s.LookupCacheSizeForDaemonSet, "daemonset-lookup-cache-size", s.LookupCacheSizeForDaemonSet, "This flag is deprecated and will be removed in future releases. DaemonSet no longer requires a lookup cache.") - fs.MarkDeprecated("daemonset-lookup-cache-size", "This flag is deprecated and will be removed in future releases. DaemonSet no longer requires a lookup cache.") fs.DurationVar(&s.ServiceSyncPeriod.Duration, "service-sync-period", s.ServiceSyncPeriod.Duration, "The period for syncing services with their external load balancers") fs.DurationVar(&s.NodeSyncPeriod.Duration, "node-sync-period", 0, ""+ "This flag is deprecated and will be removed in future releases. See node-monitor-period for Node health checking or "+ diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index 2b036c8b7e65c..f26ab264cefd1 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -47,6 +47,7 @@ import ( "k8s.io/kubernetes/pkg/volume/gce_pd" "k8s.io/kubernetes/pkg/volume/glusterfs" "k8s.io/kubernetes/pkg/volume/host_path" + "k8s.io/kubernetes/pkg/volume/iscsi" "k8s.io/kubernetes/pkg/volume/local" "k8s.io/kubernetes/pkg/volume/nfs" "k8s.io/kubernetes/pkg/volume/photon_pd" @@ -60,18 +61,15 @@ import ( ) // ProbeAttachableVolumePlugins collects all volume plugins for the attach/ -// detach controller. VolumeConfiguration is used ot get FlexVolumePluginDir -// which specifies the directory to search for additional third party volume -// plugins. +// detach controller. // The list of plugins is manually compiled. This code and the plugin // initialization code for kubelet really, really need a through refactor. -func ProbeAttachableVolumePlugins(config componentconfig.VolumeConfiguration) []volume.VolumePlugin { +func ProbeAttachableVolumePlugins() []volume.VolumePlugin { allPlugins := []volume.VolumePlugin{} allPlugins = append(allPlugins, aws_ebs.ProbeVolumePlugins()...) allPlugins = append(allPlugins, gce_pd.ProbeVolumePlugins()...) allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, flexvolume.ProbeVolumePlugins(config.FlexVolumePluginDir)...) allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...) allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) @@ -79,9 +77,17 @@ func ProbeAttachableVolumePlugins(config componentconfig.VolumeConfiguration) [] allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...) allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...) return allPlugins } +// GetDynamicPluginProber gets the probers of dynamically discoverable plugins +// for the attach/detach controller. +// Currently only Flexvolume plugins are dynamically discoverable. +func GetDynamicPluginProber(config componentconfig.VolumeConfiguration) volume.DynamicPluginProber { + return flexvolume.GetDynamicPluginProber(config.FlexVolumePluginDir) +} + // ProbeControllerVolumePlugins collects all persistent volume plugins into an // easy to use list. Only volume plugins that implement any of // provisioner/recycler/deleter interface should be returned. diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 320799a673cc6..6010bad665f1e 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -253,7 +253,7 @@ func (o *Options) writeConfigFile() error { return err } - fmt.Printf("Wrote configuration to: %s\n", o.WriteConfigTo) + glog.Infof("Wrote configuration to: %s\n", o.WriteConfigTo) return nil } diff --git a/cmd/kubeadm/app/BUILD b/cmd/kubeadm/app/BUILD index a54eeb757d338..c3705c11052da 100644 --- a/cmd/kubeadm/app/BUILD +++ b/cmd/kubeadm/app/BUILD @@ -31,6 +31,7 @@ filegroup( "//cmd/kubeadm/app/cmd:all-srcs", "//cmd/kubeadm/app/constants:all-srcs", "//cmd/kubeadm/app/discovery:all-srcs", + "//cmd/kubeadm/app/features:all-srcs", "//cmd/kubeadm/app/images:all-srcs", "//cmd/kubeadm/app/node:all-srcs", "//cmd/kubeadm/app/phases/addons/dns:all-srcs", @@ -45,6 +46,7 @@ filegroup( "//cmd/kubeadm/app/phases/markmaster:all-srcs", "//cmd/kubeadm/app/phases/selfhosting:all-srcs", "//cmd/kubeadm/app/phases/token:all-srcs", + "//cmd/kubeadm/app/phases/upgrade:all-srcs", "//cmd/kubeadm/app/phases/uploadconfig:all-srcs", "//cmd/kubeadm/app/preflight:all-srcs", "//cmd/kubeadm/app/util:all-srcs", diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index 5dedda4e4b5fd..bee5c7d237406 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -40,8 +40,9 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.Etcd.Image = "foo" obj.Etcd.DataDir = "foo" obj.ImageRepository = "foo" + obj.CIImageRepository = "" obj.UnifiedControlPlaneImage = "foo" - obj.FeatureFlags = map[string]bool{} + obj.FeatureGates = map[string]bool{} }, func(obj *kubeadm.NodeConfiguration, c fuzz.Continue) { c.FuzzNoCustom(obj) diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 93ad680334447..7de204380e50e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -17,7 +17,6 @@ limitations under the License. package kubeadm import ( - "fmt" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -50,11 +49,16 @@ type MasterConfiguration struct { // ImageRepository what container registry to pull control plane images from ImageRepository string + + // Container registry for core images generated by CI + // +k8s:conversion-gen=false + CIImageRepository string + // UnifiedControlPlaneImage specifies if a specific container image should be used for all control plane components UnifiedControlPlaneImage string - // FeatureFlags enabled by the user - FeatureFlags map[string]bool + // FeatureGates enabled by the user + FeatureGates map[string]bool } type API struct { @@ -117,6 +121,14 @@ type NodeConfiguration struct { DiscoveryTokenUnsafeSkipCAVerification bool } -func (cfg *MasterConfiguration) GetMasterEndpoint() string { - return fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddress, cfg.API.BindPort) +// GetControlPlaneImageRepository returns name of image repository +// for control plane images (API,Controller Manager,Scheduler and Proxy) +// It will override location with CI registry name in case user requests special +// Kubernetes version from CI build area. +// (See: kubeadmconstants.DefaultCIImageRepository) +func (cfg *MasterConfiguration) GetControlPlaneImageRepository() string { + if cfg.CIImageRepository != "" { + return cfg.CIImageRepository + } + return cfg.ImageRepository } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD index 3feef312b9d62..b75dec730b3f9 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD @@ -12,10 +12,12 @@ go_library( "doc.go", "register.go", "types.go", + "zz_generated.conversion.go", "zz_generated.deepcopy.go", "zz_generated.defaults.go", ], deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library", diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/doc.go index 4609a57d9eb0f..d0a277ec0b28b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/doc.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/doc.go @@ -17,4 +17,5 @@ limitations under the License. // +k8s:defaulter-gen=TypeMeta // +groupName=kubeadm.k8s.io // +k8s:deepcopy-gen=package +// +k8s:conversion-gen=k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm package v1alpha1 // import "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index f7c1cdf7ac194..e96f67ed49e57 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -52,8 +52,8 @@ type MasterConfiguration struct { // UnifiedControlPlaneImage specifies if a specific container image should be used for all control plane components UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"` - // FeatureFlags enabled by the user - FeatureFlags map[string]bool `json:"featureFlags"` + // FeatureGates enabled by the user + FeatureGates map[string]bool `json:"featureGates"` } type API struct { diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go new file mode 100644 index 0000000000000..02a9ea33ec6e3 --- /dev/null +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go @@ -0,0 +1,255 @@ +// +build !ignore_autogenerated + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file was autogenerated by conversion-gen. Do not edit it manually! + +package v1alpha1 + +import ( + conversion "k8s.io/apimachinery/pkg/conversion" + runtime "k8s.io/apimachinery/pkg/runtime" + kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + time "time" + unsafe "unsafe" +) + +func init() { + localSchemeBuilder.Register(RegisterConversions) +} + +// RegisterConversions adds conversion functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterConversions(scheme *runtime.Scheme) error { + return scheme.AddGeneratedConversionFuncs( + Convert_v1alpha1_API_To_kubeadm_API, + Convert_kubeadm_API_To_v1alpha1_API, + Convert_v1alpha1_Etcd_To_kubeadm_Etcd, + Convert_kubeadm_Etcd_To_v1alpha1_Etcd, + Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration, + Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration, + Convert_v1alpha1_Networking_To_kubeadm_Networking, + Convert_kubeadm_Networking_To_v1alpha1_Networking, + Convert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration, + Convert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration, + Convert_v1alpha1_TokenDiscovery_To_kubeadm_TokenDiscovery, + Convert_kubeadm_TokenDiscovery_To_v1alpha1_TokenDiscovery, + ) +} + +func autoConvert_v1alpha1_API_To_kubeadm_API(in *API, out *kubeadm.API, s conversion.Scope) error { + out.AdvertiseAddress = in.AdvertiseAddress + out.BindPort = in.BindPort + return nil +} + +// Convert_v1alpha1_API_To_kubeadm_API is an autogenerated conversion function. +func Convert_v1alpha1_API_To_kubeadm_API(in *API, out *kubeadm.API, s conversion.Scope) error { + return autoConvert_v1alpha1_API_To_kubeadm_API(in, out, s) +} + +func autoConvert_kubeadm_API_To_v1alpha1_API(in *kubeadm.API, out *API, s conversion.Scope) error { + out.AdvertiseAddress = in.AdvertiseAddress + out.BindPort = in.BindPort + return nil +} + +// Convert_kubeadm_API_To_v1alpha1_API is an autogenerated conversion function. +func Convert_kubeadm_API_To_v1alpha1_API(in *kubeadm.API, out *API, s conversion.Scope) error { + return autoConvert_kubeadm_API_To_v1alpha1_API(in, out, s) +} + +func autoConvert_v1alpha1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error { + out.Endpoints = *(*[]string)(unsafe.Pointer(&in.Endpoints)) + out.CAFile = in.CAFile + out.CertFile = in.CertFile + out.KeyFile = in.KeyFile + out.DataDir = in.DataDir + out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.Image = in.Image + return nil +} + +// Convert_v1alpha1_Etcd_To_kubeadm_Etcd is an autogenerated conversion function. +func Convert_v1alpha1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error { + return autoConvert_v1alpha1_Etcd_To_kubeadm_Etcd(in, out, s) +} + +func autoConvert_kubeadm_Etcd_To_v1alpha1_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error { + out.Endpoints = *(*[]string)(unsafe.Pointer(&in.Endpoints)) + out.CAFile = in.CAFile + out.CertFile = in.CertFile + out.KeyFile = in.KeyFile + out.DataDir = in.DataDir + out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.Image = in.Image + return nil +} + +// Convert_kubeadm_Etcd_To_v1alpha1_Etcd is an autogenerated conversion function. +func Convert_kubeadm_Etcd_To_v1alpha1_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error { + return autoConvert_kubeadm_Etcd_To_v1alpha1_Etcd(in, out, s) +} + +func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in *MasterConfiguration, out *kubeadm.MasterConfiguration, s conversion.Scope) error { + if err := Convert_v1alpha1_API_To_kubeadm_API(&in.API, &out.API, s); err != nil { + return err + } + if err := Convert_v1alpha1_Etcd_To_kubeadm_Etcd(&in.Etcd, &out.Etcd, s); err != nil { + return err + } + if err := Convert_v1alpha1_Networking_To_kubeadm_Networking(&in.Networking, &out.Networking, s); err != nil { + return err + } + out.KubernetesVersion = in.KubernetesVersion + out.CloudProvider = in.CloudProvider + out.NodeName = in.NodeName + out.AuthorizationModes = *(*[]string)(unsafe.Pointer(&in.AuthorizationModes)) + out.Token = in.Token + out.TokenTTL = time.Duration(in.TokenTTL) + out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs)) + out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs)) + out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs)) + out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs)) + out.CertificatesDir = in.CertificatesDir + out.ImageRepository = in.ImageRepository + out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + return nil +} + +// Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in *MasterConfiguration, out *kubeadm.MasterConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in, out, s) +} + +func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in *kubeadm.MasterConfiguration, out *MasterConfiguration, s conversion.Scope) error { + if err := Convert_kubeadm_API_To_v1alpha1_API(&in.API, &out.API, s); err != nil { + return err + } + if err := Convert_kubeadm_Etcd_To_v1alpha1_Etcd(&in.Etcd, &out.Etcd, s); err != nil { + return err + } + if err := Convert_kubeadm_Networking_To_v1alpha1_Networking(&in.Networking, &out.Networking, s); err != nil { + return err + } + out.KubernetesVersion = in.KubernetesVersion + out.CloudProvider = in.CloudProvider + out.NodeName = in.NodeName + out.AuthorizationModes = *(*[]string)(unsafe.Pointer(&in.AuthorizationModes)) + out.Token = in.Token + out.TokenTTL = time.Duration(in.TokenTTL) + out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs)) + out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs)) + out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs)) + out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs)) + out.CertificatesDir = in.CertificatesDir + out.ImageRepository = in.ImageRepository + // INFO: in.CIImageRepository opted out of conversion generation + out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + return nil +} + +// Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration is an autogenerated conversion function. +func Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in *kubeadm.MasterConfiguration, out *MasterConfiguration, s conversion.Scope) error { + return autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in, out, s) +} + +func autoConvert_v1alpha1_Networking_To_kubeadm_Networking(in *Networking, out *kubeadm.Networking, s conversion.Scope) error { + out.ServiceSubnet = in.ServiceSubnet + out.PodSubnet = in.PodSubnet + out.DNSDomain = in.DNSDomain + return nil +} + +// Convert_v1alpha1_Networking_To_kubeadm_Networking is an autogenerated conversion function. +func Convert_v1alpha1_Networking_To_kubeadm_Networking(in *Networking, out *kubeadm.Networking, s conversion.Scope) error { + return autoConvert_v1alpha1_Networking_To_kubeadm_Networking(in, out, s) +} + +func autoConvert_kubeadm_Networking_To_v1alpha1_Networking(in *kubeadm.Networking, out *Networking, s conversion.Scope) error { + out.ServiceSubnet = in.ServiceSubnet + out.PodSubnet = in.PodSubnet + out.DNSDomain = in.DNSDomain + return nil +} + +// Convert_kubeadm_Networking_To_v1alpha1_Networking is an autogenerated conversion function. +func Convert_kubeadm_Networking_To_v1alpha1_Networking(in *kubeadm.Networking, out *Networking, s conversion.Scope) error { + return autoConvert_kubeadm_Networking_To_v1alpha1_Networking(in, out, s) +} + +func autoConvert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration(in *NodeConfiguration, out *kubeadm.NodeConfiguration, s conversion.Scope) error { + out.CACertPath = in.CACertPath + out.DiscoveryFile = in.DiscoveryFile + out.DiscoveryToken = in.DiscoveryToken + out.DiscoveryTokenAPIServers = *(*[]string)(unsafe.Pointer(&in.DiscoveryTokenAPIServers)) + out.NodeName = in.NodeName + out.TLSBootstrapToken = in.TLSBootstrapToken + out.Token = in.Token + out.DiscoveryTokenCACertHashes = *(*[]string)(unsafe.Pointer(&in.DiscoveryTokenCACertHashes)) + out.DiscoveryTokenUnsafeSkipCAVerification = in.DiscoveryTokenUnsafeSkipCAVerification + return nil +} + +// Convert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration(in *NodeConfiguration, out *kubeadm.NodeConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration(in, out, s) +} + +func autoConvert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in *kubeadm.NodeConfiguration, out *NodeConfiguration, s conversion.Scope) error { + out.CACertPath = in.CACertPath + out.DiscoveryFile = in.DiscoveryFile + out.DiscoveryToken = in.DiscoveryToken + out.DiscoveryTokenAPIServers = *(*[]string)(unsafe.Pointer(&in.DiscoveryTokenAPIServers)) + out.NodeName = in.NodeName + out.TLSBootstrapToken = in.TLSBootstrapToken + out.Token = in.Token + out.DiscoveryTokenCACertHashes = *(*[]string)(unsafe.Pointer(&in.DiscoveryTokenCACertHashes)) + out.DiscoveryTokenUnsafeSkipCAVerification = in.DiscoveryTokenUnsafeSkipCAVerification + return nil +} + +// Convert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration is an autogenerated conversion function. +func Convert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in *kubeadm.NodeConfiguration, out *NodeConfiguration, s conversion.Scope) error { + return autoConvert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in, out, s) +} + +func autoConvert_v1alpha1_TokenDiscovery_To_kubeadm_TokenDiscovery(in *TokenDiscovery, out *kubeadm.TokenDiscovery, s conversion.Scope) error { + out.ID = in.ID + out.Secret = in.Secret + out.Addresses = *(*[]string)(unsafe.Pointer(&in.Addresses)) + return nil +} + +// Convert_v1alpha1_TokenDiscovery_To_kubeadm_TokenDiscovery is an autogenerated conversion function. +func Convert_v1alpha1_TokenDiscovery_To_kubeadm_TokenDiscovery(in *TokenDiscovery, out *kubeadm.TokenDiscovery, s conversion.Scope) error { + return autoConvert_v1alpha1_TokenDiscovery_To_kubeadm_TokenDiscovery(in, out, s) +} + +func autoConvert_kubeadm_TokenDiscovery_To_v1alpha1_TokenDiscovery(in *kubeadm.TokenDiscovery, out *TokenDiscovery, s conversion.Scope) error { + out.ID = in.ID + out.Secret = in.Secret + out.Addresses = *(*[]string)(unsafe.Pointer(&in.Addresses)) + return nil +} + +// Convert_kubeadm_TokenDiscovery_To_v1alpha1_TokenDiscovery is an autogenerated conversion function. +func Convert_kubeadm_TokenDiscovery_To_v1alpha1_TokenDiscovery(in *kubeadm.TokenDiscovery, out *TokenDiscovery, s conversion.Scope) error { + return autoConvert_kubeadm_TokenDiscovery_To_v1alpha1_TokenDiscovery(in, out, s) +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go index 3750f87b64c1b..57a22cd188c43 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.deepcopy.go @@ -140,8 +140,8 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.FeatureFlags != nil { - in, out := &in.FeatureFlags, &out.FeatureFlags + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates *out = make(map[string]bool, len(*in)) for key, val := range *in { (*out)[key] = val diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/BUILD b/cmd/kubeadm/app/apis/kubeadm/validation/BUILD index c7412cc893841..09eed2d163d16 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/validation/BUILD @@ -22,8 +22,9 @@ go_library( srcs = ["validation.go"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/cmd/features:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/token:go_default_library", "//pkg/api/validation:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index dbdd252dccb9e..7ed09b192170a 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -29,8 +29,9 @@ import ( "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/features" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token" apivalidation "k8s.io/kubernetes/pkg/api/validation" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" @@ -67,7 +68,8 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificates-dir"))...) allErrs = append(allErrs, ValidateNodeName(c.NodeName, field.NewPath("node-name"))...) allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...) - allErrs = append(allErrs, ValidateFeatureFlags(c.FeatureFlags, field.NewPath("feature-flags"))...) + allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("feature-gates"))...) + allErrs = append(allErrs, ValidateAPIEndpoint(c, field.NewPath("api-endpoint"))...) return allErrs } @@ -282,8 +284,8 @@ func ValidateMixedArguments(flag *pflag.FlagSet) error { mixedInvalidFlags := []string{} flag.Visit(func(f *pflag.Flag) { - if f.Name == "config" || strings.HasPrefix(f.Name, "skip-") { - // "--skip-*" flags can be set with --config + if f.Name == "config" || strings.HasPrefix(f.Name, "skip-") || f.Name == "dry-run" || f.Name == "kubeconfig" { + // "--skip-*" flags or other whitelisted flags can be set with --config return } mixedInvalidFlags = append(mixedInvalidFlags, f.Name) @@ -295,17 +297,27 @@ func ValidateMixedArguments(flag *pflag.FlagSet) error { return nil } -func ValidateFeatureFlags(featureFlags map[string]bool, fldPath *field.Path) field.ErrorList { +func ValidateFeatureGates(featureGates map[string]bool, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} validFeatures := features.Keys(features.InitFeatureGates) // check valid feature names are provided - for k := range featureFlags { + for k := range featureGates { if !features.Supports(features.InitFeatureGates, k) { - allErrs = append(allErrs, field.Invalid(fldPath, featureFlags, + allErrs = append(allErrs, field.Invalid(fldPath, featureGates, fmt.Sprintf("%s is not a valid feature name. Valid features are: %s", k, validFeatures))) } } return allErrs } + +func ValidateAPIEndpoint(c *kubeadm.MasterConfiguration, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + endpoint, err := kubeadmutil.GetMasterEndpoint(c) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, endpoint, "Invalid API Endpoint")) + } + return allErrs +} diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go index e4fe08cf57cea..2e0c874f30362 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go @@ -209,6 +209,71 @@ func TestValidateIPNetFromString(t *testing.T) { } } +func TestValidateAPIEndpoint(t *testing.T) { + var tests = []struct { + name string + s *kubeadm.MasterConfiguration + expected bool + }{ + { + name: "Missing configuration", + s: &kubeadm.MasterConfiguration{}, + expected: false, + }, + { + name: "Valid IPv4 address and default port", + s: &kubeadm.MasterConfiguration{ + API: kubeadm.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: 6443, + }, + }, + expected: true, + }, + { + name: "Valid IPv6 address and port", + s: &kubeadm.MasterConfiguration{ + API: kubeadm.API{ + AdvertiseAddress: "2001:db7::1", + BindPort: 3446, + }, + }, + expected: true, + }, + { + name: "Invalid IPv4 address", + s: &kubeadm.MasterConfiguration{ + API: kubeadm.API{ + AdvertiseAddress: "1.2.34", + BindPort: 6443, + }, + }, + expected: false, + }, + { + name: "Invalid IPv6 address", + s: &kubeadm.MasterConfiguration{ + API: kubeadm.API{ + AdvertiseAddress: "2001:db7:1", + BindPort: 3446, + }, + }, + expected: false, + }, + } + for _, rt := range tests { + actual := ValidateAPIEndpoint(rt.s, nil) + if (len(actual) == 0) != rt.expected { + t.Errorf( + "%s test case failed:\n\texpected: %t\n\t actual: %t", + rt.name, + rt.expected, + (len(actual) == 0), + ) + } + } +} + func TestValidateMasterConfiguration(t *testing.T) { nodename := "valid-nodename" var tests = []struct { @@ -220,6 +285,10 @@ func TestValidateMasterConfiguration(t *testing.T) { &kubeadm.MasterConfiguration{}, false}, {"invalid missing token with IPv4 service subnet", &kubeadm.MasterConfiguration{ + API: kubeadm.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: 6443, + }, AuthorizationModes: []string{"Node", "RBAC"}, Networking: kubeadm.Networking{ ServiceSubnet: "10.96.0.1/12", @@ -230,6 +299,10 @@ func TestValidateMasterConfiguration(t *testing.T) { }, false}, {"invalid missing token with IPv6 service subnet", &kubeadm.MasterConfiguration{ + API: kubeadm.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: 6443, + }, AuthorizationModes: []string{"Node", "RBAC"}, Networking: kubeadm.Networking{ ServiceSubnet: "2001:db8::1/98", @@ -240,6 +313,10 @@ func TestValidateMasterConfiguration(t *testing.T) { }, false}, {"invalid missing node name", &kubeadm.MasterConfiguration{ + API: kubeadm.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: 6443, + }, AuthorizationModes: []string{"Node", "RBAC"}, Networking: kubeadm.Networking{ ServiceSubnet: "10.96.0.1/12", @@ -250,6 +327,10 @@ func TestValidateMasterConfiguration(t *testing.T) { }, false}, {"valid master configuration with IPv4 service subnet", &kubeadm.MasterConfiguration{ + API: kubeadm.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: 6443, + }, AuthorizationModes: []string{"Node", "RBAC"}, Networking: kubeadm.Networking{ ServiceSubnet: "10.96.0.1/12", @@ -261,6 +342,10 @@ func TestValidateMasterConfiguration(t *testing.T) { }, true}, {"valid master configuration using IPv6 service subnet", &kubeadm.MasterConfiguration{ + API: kubeadm.API{ + AdvertiseAddress: "1:2:3::4", + BindPort: 3446, + }, AuthorizationModes: []string{"Node", "RBAC"}, Networking: kubeadm.Networking{ ServiceSubnet: "2001:db8::1/98", @@ -351,10 +436,10 @@ func TestValidateMixedArguments(t *testing.T) { } } -func TestValidateFeatureFlags(t *testing.T) { +func TestValidateFeatureGates(t *testing.T) { type featureFlag map[string]bool var tests = []struct { - featureFlags featureFlag + featureGates featureFlag expected bool }{ {featureFlag{"SelfHosting": true}, true}, @@ -364,10 +449,10 @@ func TestValidateFeatureFlags(t *testing.T) { {featureFlag{"Foo": true}, false}, } for _, rt := range tests { - actual := ValidateFeatureFlags(rt.featureFlags, nil) + actual := ValidateFeatureGates(rt.featureGates, nil) if (len(actual) == 0) != rt.expected { t.Errorf( - "failed featureFlags:\n\texpected: %t\n\t actual: %t", + "failed featureGates:\n\texpected: %t\n\t actual: %t", rt.expected, (len(actual) == 0), ) diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go index 7fae52156a9fe..a7a52e94ee832 100644 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go @@ -145,8 +145,8 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.FeatureFlags != nil { - in, out := &in.FeatureFlags, &out.FeatureFlags + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates *out = make(map[string]bool, len(*in)) for key, val := range *in { (*out)[key] = val diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index e80b1b37996be..6c261ac2c68a9 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -11,6 +11,7 @@ go_library( srcs = [ "cmd.go", "completion.go", + "config.go", "init.go", "join.go", "reset.go", @@ -21,15 +22,18 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", - "//cmd/kubeadm/app/cmd/features:go_default_library", "//cmd/kubeadm/app/cmd/phases:go_default_library", + "//cmd/kubeadm/app/cmd/upgrade:go_default_library", + "//cmd/kubeadm/app/cmd/util:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/discovery:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/addons/dns:go_default_library", "//cmd/kubeadm/app/phases/addons/proxy:go_default_library", "//cmd/kubeadm/app/phases/apiconfig:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", + "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", "//cmd/kubeadm/app/phases/controlplane:go_default_library", "//cmd/kubeadm/app/phases/etcd:go_default_library", @@ -41,6 +45,7 @@ go_library( "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", + "//cmd/kubeadm/app/util/dryrun:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//cmd/kubeadm/app/util/pubkeypin:go_default_library", "//cmd/kubeadm/app/util/token:go_default_library", @@ -56,10 +61,12 @@ go_library( "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/github.com/renstrom/dedent:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/version:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", @@ -91,8 +98,9 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", - "//cmd/kubeadm/app/cmd/features:all-srcs", "//cmd/kubeadm/app/cmd/phases:all-srcs", + "//cmd/kubeadm/app/cmd/upgrade:all-srcs", + "//cmd/kubeadm/app/cmd/util:all-srcs", ], tags = ["automanaged"], ) diff --git a/cmd/kubeadm/app/cmd/cmd.go b/cmd/kubeadm/app/cmd/cmd.go index 8b4a247a9c8aa..9546f0d10426e 100644 --- a/cmd/kubeadm/app/cmd/cmd.go +++ b/cmd/kubeadm/app/cmd/cmd.go @@ -24,6 +24,7 @@ import ( "k8s.io/apiserver/pkg/util/flag" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/upgrade" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" ) @@ -68,11 +69,13 @@ func NewKubeadmCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob cmds.SetGlobalNormalizationFunc(flag.WarnWordSepNormalizeFunc) cmds.AddCommand(NewCmdCompletion(out, "")) + cmds.AddCommand(NewCmdConfig(out)) cmds.AddCommand(NewCmdInit(out)) cmds.AddCommand(NewCmdJoin(out)) cmds.AddCommand(NewCmdReset(out)) cmds.AddCommand(NewCmdVersion(out)) cmds.AddCommand(NewCmdToken(out, err)) + cmds.AddCommand(upgrade.NewCmdUpgrade(out)) // Wrap not yet fully supported commands in an alpha subcommand experimentalCmd := &cobra.Command{ diff --git a/cmd/kubeadm/app/cmd/completion.go b/cmd/kubeadm/app/cmd/completion.go index e56b674a89746..9035d80f41ddd 100644 --- a/cmd/kubeadm/app/cmd/completion.go +++ b/cmd/kubeadm/app/cmd/completion.go @@ -186,14 +186,6 @@ __kubeadm_compopt() { true # don't do anything. Not supported by bashcompinit in zsh } -__kubeadm_declare() { - if [ "$1" == "-F" ]; then - whence -w "$@" - else - builtin declare "$@" - fi -} - __kubeadm_ltrim_colon_completions() { if [[ "$1" == *:* && "$COMP_WORDBREAKS" == *:* ]]; then @@ -280,7 +272,7 @@ __kubeadm_convert_bash_to_zsh() { -e "s/${LWORD}__ltrim_colon_completions${RWORD}/__kubeadm_ltrim_colon_completions/g" \ -e "s/${LWORD}compgen${RWORD}/__kubeadm_compgen/g" \ -e "s/${LWORD}compopt${RWORD}/__kubeadm_compopt/g" \ - -e "s/${LWORD}declare${RWORD}/__kubeadm_declare/g" \ + -e "s/${LWORD}declare${RWORD}/builtin declare/g" \ -e "s/\\\$(type${RWORD}/\$(__kubeadm_type/g" \ <<'BASH_COMPLETION_EOF' ` diff --git a/cmd/kubeadm/app/cmd/config.go b/cmd/kubeadm/app/cmd/config.go new file mode 100644 index 0000000000000..c703a97563fd2 --- /dev/null +++ b/cmd/kubeadm/app/cmd/config.go @@ -0,0 +1,188 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "io" + + "github.com/renstrom/dedent" + "github.com/spf13/cobra" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + "k8s.io/kubernetes/pkg/api" +) + +func NewCmdConfig(out io.Writer) *cobra.Command { + + var kubeConfigFile string + cmd := &cobra.Command{ + Use: "config", + Short: "Manage configuration for a kubeadm cluster persisted in a ConfigMap in the cluster.", + Long: fmt.Sprintf(dedent.Dedent(` + There is a ConfigMap in the %s namespace called %q that kubeadm uses to store internal configuration about the + cluster. kubeadm CLI v1.8.0+ automatically creates this ConfigMap with used config on 'kubeadm init', but if you + initialized your cluster using kubeadm v1.7.x or lower, you must use the 'config upload' command to create this + ConfigMap in order for 'kubeadm upgrade' to be able to configure your upgraded cluster correctly. + `), metav1.NamespaceSystem, constants.MasterConfigurationConfigMap), + // Without this callback, if a user runs just the "upload" + // command without a subcommand, or with an invalid subcommand, + // cobra will print usage information, but still exit cleanly. + // We want to return an error code in these cases so that the + // user knows that their command was invalid. + RunE: cmdutil.SubCmdRunE("config"), + } + + cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster") + + cmd.AddCommand(NewCmdConfigUpload(out, &kubeConfigFile)) + cmd.AddCommand(NewCmdConfigView(out, &kubeConfigFile)) + + return cmd +} + +func NewCmdConfigUpload(out io.Writer, kubeConfigFile *string) *cobra.Command { + cmd := &cobra.Command{ + Use: "upload", + Short: "Upload configuration about the current state so 'kubeadm upgrade' later can know how to configure the upgraded cluster", + RunE: cmdutil.SubCmdRunE("upload"), + } + + cmd.AddCommand(NewCmdConfigUploadFromFile(out, kubeConfigFile)) + cmd.AddCommand(NewCmdConfigUploadFromFlags(out, kubeConfigFile)) + return cmd +} + +func NewCmdConfigView(out io.Writer, kubeConfigFile *string) *cobra.Command { + return &cobra.Command{ + Use: "view", + Short: "View the kubeadm configuration stored inside the cluster", + Long: fmt.Sprintf(dedent.Dedent(` + Using this command, you can view the ConfigMap in the cluster where the configuration for kubeadm is located + + The configuration is located in the %q namespace in the %q ConfigMap + `), metav1.NamespaceSystem, constants.MasterConfigurationConfigMap), + Run: func(cmd *cobra.Command, args []string) { + client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + kubeadmutil.CheckErr(err) + + err = RunConfigView(out, client) + kubeadmutil.CheckErr(err) + }, + } +} + +func NewCmdConfigUploadFromFile(out io.Writer, kubeConfigFile *string) *cobra.Command { + var cfgPath string + cmd := &cobra.Command{ + Use: "from-file", + Short: "Upload a configuration file to the in-cluster ConfigMap for kubeadm configuration", + Long: fmt.Sprintf(dedent.Dedent(` + Using from-file, you can upload configuration to the ConfigMap in the cluster using the same config file you gave to kubeadm init. + If you initialized your cluster using a v1.7.x or lower kubeadm client and used the --config option; you need to run this command with the + same config file before upgrading to v1.8 using 'kubeadm upgrade'. + + The configuration is located in the %q namespace in the %q ConfigMap + `), metav1.NamespaceSystem, constants.MasterConfigurationConfigMap), + Run: func(cmd *cobra.Command, args []string) { + if len(cfgPath) == 0 { + kubeadmutil.CheckErr(fmt.Errorf("The --config flag is mandatory")) + } + + client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + kubeadmutil.CheckErr(err) + + // The default configuration is empty; everything should come from the file on disk + defaultcfg := &kubeadmapiext.MasterConfiguration{} + // Upload the configuration using the file; don't care about the defaultcfg really + err = uploadConfiguration(client, cfgPath, defaultcfg) + kubeadmutil.CheckErr(err) + }, + } + cmd.Flags().StringVar(&cfgPath, "config", "", "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + return cmd +} + +func NewCmdConfigUploadFromFlags(out io.Writer, kubeConfigFile *string) *cobra.Command { + cfg := &kubeadmapiext.MasterConfiguration{} + api.Scheme.Default(cfg) + + var featureGatesString string + + cmd := &cobra.Command{ + Use: "from-flags", + Short: "Create the in-cluster configuration file for the first time from using flags", + Long: fmt.Sprintf(dedent.Dedent(` + Using from-flags, you can upload configuration to the ConfigMap in the cluster using the same flags you'd give to kubeadm init. + If you initialized your cluster using a v1.7.x or lower kubeadm client and set some flag; you need to run this command with the + same flags before upgrading to v1.8 using 'kubeadm upgrade'. + + The configuration is located in the %q namespace in the %q ConfigMap + `), metav1.NamespaceSystem, constants.MasterConfigurationConfigMap), + Run: func(cmd *cobra.Command, args []string) { + var err error + if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { + kubeadmutil.CheckErr(err) + } + + client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + kubeadmutil.CheckErr(err) + + // Default both statically and dynamically, convert to internal API type, and validate everything + // The cfgPath argument is unset here as we shouldn't load a config file from disk, just go with cfg + err = uploadConfiguration(client, "", cfg) + kubeadmutil.CheckErr(err) + }, + } + AddInitConfigFlags(cmd.PersistentFlags(), cfg, &featureGatesString) + return cmd +} + +// RunConfigView gets the configuration persisted in the cluster +func RunConfigView(out io.Writer, client clientset.Interface) error { + + cfgConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.MasterConfigurationConfigMap, metav1.GetOptions{}) + if err != nil { + return err + } + // No need to append \n as that already exists in the ConfigMap + fmt.Fprintf(out, "%s", cfgConfigMap.Data[constants.MasterConfigurationConfigMapKey]) + return nil +} + +// uploadConfiguration handles the uploading of the configuration internally +func uploadConfiguration(client clientset.Interface, cfgPath string, defaultcfg *kubeadmapiext.MasterConfiguration) error { + + // Default both statically and dynamically, convert to internal API type, and validate everything + // First argument is unset here as we shouldn't load a config file from disk + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, defaultcfg) + if err != nil { + return err + } + + // Then just call the uploadconfig phase to do the rest of the work + return uploadconfig.UploadConfiguration(internalcfg, client) +} diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 46d613b2e89af..c099d35f8d004 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -20,25 +20,29 @@ import ( "fmt" "io" "io/ioutil" - "strconv" + "os" + "path/filepath" + "strings" "text/template" "time" "github.com/renstrom/dedent" "github.com/spf13/cobra" + flag "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/features" - cmdphases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" dnsaddonphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" proxyaddonphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" apiconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/apiconfig" clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" @@ -50,6 +54,7 @@ import ( kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin" "k8s.io/kubernetes/pkg/api" @@ -73,7 +78,7 @@ var ( You can now join any number of machines by running the following on each node as root: - kubeadm join --token {{.Token}} {{.MasterIP}}:{{.MasterPort}} --discovery-token-ca-cert-hash {{.CAPubKeyPin}} + kubeadm join --token {{.Token}} {{.MasterHostPort}} --discovery-token-ca-cert-hash {{.CAPubKeyPin}} `))) ) @@ -86,15 +91,23 @@ func NewCmdInit(out io.Writer) *cobra.Command { var cfgPath string var skipPreFlight bool var skipTokenPrint bool + var dryRun bool + var featureGatesString string + cmd := &cobra.Command{ Use: "init", Short: "Run this in order to set up the Kubernetes master", Run: func(cmd *cobra.Command, args []string) { + var err error + if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { + kubeadmutil.CheckErr(err) + } + api.Scheme.Default(cfg) internalcfg := &kubeadmapi.MasterConfiguration{} api.Scheme.Convert(cfg, internalcfg, nil) - i, err := NewInit(cfgPath, internalcfg, skipPreFlight, skipTokenPrint) + i, err := NewInit(cfgPath, internalcfg, skipPreFlight, skipTokenPrint, dryRun) kubeadmutil.CheckErr(err) kubeadmutil.CheckErr(i.Validate(cmd)) @@ -107,68 +120,86 @@ func NewCmdInit(out io.Writer) *cobra.Command { }, } - cmd.PersistentFlags().StringVar( + AddInitConfigFlags(cmd.PersistentFlags(), cfg, &featureGatesString) + AddInitOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &skipTokenPrint, &dryRun) + + return cmd +} + +// AddInitConfigFlags adds init flags bound to the config to the specified flagset +func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.MasterConfiguration, featureGatesString *string) { + flagSet.StringVar( &cfg.API.AdvertiseAddress, "apiserver-advertise-address", cfg.API.AdvertiseAddress, "The IP address the API Server will advertise it's listening on. 0.0.0.0 means the default network interface's address.", ) - cmd.PersistentFlags().Int32Var( + flagSet.Int32Var( &cfg.API.BindPort, "apiserver-bind-port", cfg.API.BindPort, "Port for the API Server to bind to", ) - cmd.PersistentFlags().StringVar( + flagSet.StringVar( &cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet, "Use alternative range of IP address for service VIPs", ) - cmd.PersistentFlags().StringVar( + flagSet.StringVar( &cfg.Networking.PodSubnet, "pod-network-cidr", cfg.Networking.PodSubnet, "Specify range of IP addresses for the pod network; if set, the control plane will automatically allocate CIDRs for every node", ) - cmd.PersistentFlags().StringVar( + flagSet.StringVar( &cfg.Networking.DNSDomain, "service-dns-domain", cfg.Networking.DNSDomain, `Use alternative domain for services, e.g. "myorg.internal"`, ) - cmd.PersistentFlags().StringVar( + flagSet.StringVar( &cfg.KubernetesVersion, "kubernetes-version", cfg.KubernetesVersion, `Choose a specific Kubernetes version for the control plane`, ) - cmd.PersistentFlags().StringVar( + flagSet.StringVar( &cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where to save and store the certificates`, ) - cmd.PersistentFlags().StringSliceVar( + flagSet.StringSliceVar( &cfg.APIServerCertSANs, "apiserver-cert-extra-sans", cfg.APIServerCertSANs, `Optional extra altnames to use for the API Server serving cert. Can be both IP addresses and dns names.`, ) - cmd.PersistentFlags().StringVar( + flagSet.StringVar( &cfg.NodeName, "node-name", cfg.NodeName, `Specify the node name`, ) + flagSet.StringVar( + &cfg.Token, "token", cfg.Token, + "The token to use for establishing bidirectional trust between nodes and masters.", + ) + flagSet.DurationVar( + &cfg.TokenTTL, "token-ttl", cfg.TokenTTL, + "The duration before the bootstrap token is automatically deleted. 0 means 'never expires'.", + ) + flagSet.StringVar(featureGatesString, "feature-gates", *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ + "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) +} - cmd.PersistentFlags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") - +// AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset +func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight, skipTokenPrint, dryRun *bool) { + flagSet.StringVar( + cfgPath, "config", *cfgPath, + "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)", + ) // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go - cmd.PersistentFlags().BoolVar( - &skipPreFlight, "skip-preflight-checks", skipPreFlight, + flagSet.BoolVar( + skipPreFlight, "skip-preflight-checks", *skipPreFlight, "Skip preflight checks normally run before modifying the system", ) // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go - cmd.PersistentFlags().BoolVar( - &skipTokenPrint, "skip-token-print", skipTokenPrint, + flagSet.BoolVar( + skipTokenPrint, "skip-token-print", *skipTokenPrint, "Skip printing of the default bootstrap token generated by 'kubeadm init'", ) - - cmd.PersistentFlags().StringVar( - &cfg.Token, "token", cfg.Token, - "The token to use for establishing bidirectional trust between nodes and masters.") - - cmd.PersistentFlags().DurationVar( - &cfg.TokenTTL, "token-ttl", cfg.TokenTTL, - "The duration before the bootstrap token is automatically deleted. 0 means 'never expires'.") - - return cmd + // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go + flagSet.BoolVar( + dryRun, "dry-run", *dryRun, + "Don't apply any changes; just output what would be done", + ) } -func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight, skipTokenPrint bool) (*Init, error) { +func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight, skipTokenPrint, dryRun bool) (*Init, error) { fmt.Println("[kubeadm] WARNING: kubeadm is in beta, please do not use it for production clusters.") @@ -189,7 +220,7 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight, } fmt.Printf("[init] Using Kubernetes version: %s\n", cfg.KubernetesVersion) - fmt.Printf("[init] Using Authorization mode: %v\n", cfg.AuthorizationModes) + fmt.Printf("[init] Using Authorization modes: %v\n", cfg.AuthorizationModes) // Warn about the limitations with the current cloudprovider solution. if cfg.CloudProvider != "" { @@ -210,12 +241,13 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight, fmt.Println("[preflight] Skipping pre-flight checks") } - return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint}, nil + return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint, dryRun: dryRun}, nil } type Init struct { cfg *kubeadmapi.MasterConfiguration skipTokenPrint bool + dryRun bool } // Validate validates configuration passed to "kubeadm init" @@ -234,35 +266,71 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("couldn't parse kubernetes version %q: %v", i.cfg.KubernetesVersion, err) } + // Get directories to write files to; can be faked if we're dry-running + realCertsDir := i.cfg.CertificatesDir + certsDirToWriteTo, kubeConfigDir, manifestDir, err := getDirectoriesToUse(i.dryRun, i.cfg.CertificatesDir) + if err != nil { + return err + } + // certsDirToWriteTo is gonna equal cfg.CertificatesDir in the normal case, but gonna be a temp directory if dryrunning + i.cfg.CertificatesDir = certsDirToWriteTo + + adminKubeConfigPath := filepath.Join(kubeConfigDir, kubeadmconstants.AdminKubeConfigFileName) + // PHASE 1: Generate certificates - if err := cmdphases.CreatePKIAssets(i.cfg); err != nil { + if err := certsphase.CreatePKIAssets(i.cfg); err != nil { return err } // PHASE 2: Generate kubeconfig files for the admin and the kubelet - if err := kubeconfigphase.CreateInitKubeConfigFiles(kubeadmconstants.KubernetesDir, i.cfg); err != nil { + if err := kubeconfigphase.CreateInitKubeConfigFiles(kubeConfigDir, i.cfg); err != nil { return err } + // Temporarily set cfg.CertificatesDir to the "real value" when writing controlplane manifests + // This is needed for writing the right kind of manifests + i.cfg.CertificatesDir = realCertsDir + // PHASE 3: Bootstrap the control plane - manifestPath := kubeadmconstants.GetStaticPodDirectory() - if err := controlplanephase.CreateInitStaticPodManifestFiles(manifestPath, i.cfg); err != nil { + if err := controlplanephase.CreateInitStaticPodManifestFiles(manifestDir, i.cfg); err != nil { return err } // Add etcd static pod spec only if external etcd is not configured if len(i.cfg.Etcd.Endpoints) == 0 { - if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(manifestPath, i.cfg); err != nil { + if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(manifestDir, i.cfg); err != nil { return err } } - client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath()) + // Revert the earlier CertificatesDir assignment to the directory that can be written to + i.cfg.CertificatesDir = certsDirToWriteTo + + // If we're dry-running, print the generated manifests + if err := printFilesIfDryRunning(i.dryRun, manifestDir); err != nil { + return err + } + + // Create a kubernetes client and wait for the API server to be healthy (if not dryrunning) + client, err := createClient(i.cfg, i.dryRun) if err != nil { return err } + // waiter holds the apiclient.Waiter implementation of choice, responsible for querying the API server in various ways and waiting for conditions to be fulfilled + waiter := getWaiter(i.dryRun, client) + fmt.Printf("[init] Waiting for the kubelet to boot up the control plane as Static Pods from directory %q\n", kubeadmconstants.GetStaticPodDirectory()) - if err := apiclient.WaitForAPI(client, 30*time.Minute); err != nil { + fmt.Println("[init] This process often takes about a minute to perform or longer if the control plane images have to be pulled...") + // TODO: Adjust this timeout or start polling the kubelet API + // TODO: Make this timeout more realistic when we do create some more complex logic about the interaction with the kubelet + if err := waiter.WaitForAPI(); err != nil { + return err + } + + // Upload currently used configuration to the cluster + // Note: This is done right in the beginning of cluster initialization; as we might want to make other phases + // depend on centralized information from this source in the future + if err := uploadconfigphase.UploadConfiguration(i.cfg, client); err != nil { return err } @@ -273,12 +341,12 @@ func (i *Init) Run(out io.Writer) error { // PHASE 5: Set up the node bootstrap tokens if !i.skipTokenPrint { - fmt.Printf("[token] Using token: %s\n", i.cfg.Token) + fmt.Printf("[bootstraptoken] Using token: %s\n", i.cfg.Token) } // Create the default node bootstrap token tokenDescription := "The default bootstrap token generated by 'kubeadm init'." - if err := nodebootstraptokenphase.UpdateOrCreateToken(client, i.cfg.Token, false, i.cfg.TokenTTL, kubeadmconstants.DefaultTokenUsages, tokenDescription); err != nil { + if err := nodebootstraptokenphase.UpdateOrCreateToken(client, i.cfg.Token, false, i.cfg.TokenTTL, kubeadmconstants.DefaultTokenUsages, []string{}, tokenDescription); err != nil { return err } // Create RBAC rules that makes the bootstrap tokens able to post CSRs @@ -291,7 +359,7 @@ func (i *Init) Run(out io.Writer) error { } // Create the cluster-info ConfigMap with the associated RBAC rules - if err := clusterinfophase.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil { + if err := clusterinfophase.CreateBootstrapConfigMapIfNotExists(client, adminKubeConfigPath); err != nil { return err } if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil { @@ -300,16 +368,11 @@ func (i *Init) Run(out io.Writer) error { // PHASE 6: Install and deploy all addons, and configure things as necessary - // Upload currently used configuration to the cluster - if err := uploadconfigphase.UploadConfiguration(i.cfg, client); err != nil { - return err - } - if err := apiconfigphase.CreateRBACRules(client, k8sVersion); err != nil { return err } - if err := dnsaddonphase.EnsureDNSAddon(i.cfg, client); err != nil { + if err := dnsaddonphase.EnsureDNSAddon(i.cfg, client, k8sVersion); err != nil { return err } @@ -318,28 +381,35 @@ func (i *Init) Run(out io.Writer) error { } // PHASE 7: Make the control plane self-hosted if feature gate is enabled - if features.Enabled(i.cfg.FeatureFlags, features.SelfHosting) { + if features.Enabled(i.cfg.FeatureGates, features.SelfHosting) { // Temporary control plane is up, now we create our self hosted control // plane components and remove the static manifests: fmt.Println("[self-hosted] Creating self-hosted control plane...") - if err := selfhostingphase.CreateSelfHostedControlPlane(i.cfg, client); err != nil { + if err := selfhostingphase.CreateSelfHostedControlPlane(manifestDir, kubeConfigDir, i.cfg, client, waiter); err != nil { return err } } + // Exit earlier if we're dryrunning + if i.dryRun { + fmt.Println("[dryrun] Finished dry-running successfully; above are the resources that would be created.") + return nil + } + // Load the CA certificate from so we can pin its public key caCert, err := pkiutil.TryLoadCertFromDisk(i.cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + + // Generate the Master host/port pair used by initDoneTempl + masterHostPort, err := kubeadmutil.GetMasterHostPort(i.cfg) if err != nil { return err } ctx := map[string]string{ - "KubeConfigPath": kubeadmconstants.GetAdminKubeConfigPath(), - "KubeConfigName": kubeadmconstants.AdminKubeConfigFileName, + "KubeConfigPath": adminKubeConfigPath, "Token": i.cfg.Token, "CAPubKeyPin": pubkeypin.Hash(caCert), - "MasterIP": i.cfg.API.AdvertiseAddress, - "MasterPort": strconv.Itoa(int(i.cfg.API.BindPort)), + "MasterHostPort": masterHostPort, } if i.skipTokenPrint { ctx["Token"] = "" @@ -347,3 +417,60 @@ func (i *Init) Run(out io.Writer) error { return initDoneTempl.Execute(out, ctx) } + +// createClient creates a clientset.Interface object +func createClient(cfg *kubeadmapi.MasterConfiguration, dryRun bool) (clientset.Interface, error) { + if dryRun { + // If we're dry-running; we should create a faked client that answers some GETs in order to be able to do the full init flow and just logs the rest of requests + dryRunGetter := apiclient.NewInitDryRunGetter(cfg.NodeName, cfg.Networking.ServiceSubnet) + return apiclient.NewDryRunClient(dryRunGetter, os.Stdout), nil + } + + // If we're acting for real, we should create a connection to the API server and wait for it to come up + return kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath()) +} + +// getDirectoriesToUse returns the (in order) certificates, kubeconfig and Static Pod manifest directories, followed by a possible error +// This behaves differently when dry-running vs the normal flow +func getDirectoriesToUse(dryRun bool, defaultPkiDir string) (string, string, string, error) { + if dryRun { + dryRunDir, err := ioutil.TempDir("", "kubeadm-init-dryrun") + if err != nil { + return "", "", "", fmt.Errorf("couldn't create a temporary directory: %v", err) + } + // Use the same temp dir for all + return dryRunDir, dryRunDir, dryRunDir, nil + } + + return defaultPkiDir, kubeadmconstants.KubernetesDir, kubeadmconstants.GetStaticPodDirectory(), nil +} + +// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup +func printFilesIfDryRunning(dryRun bool, manifestDir string) error { + if !dryRun { + return nil + } + + fmt.Printf("[dryrun] Wrote certificates, kubeconfig files and control plane manifests to %q\n", manifestDir) + fmt.Println("[dryrun] Won't print certificates or kubeconfig files due to the sensitive nature of them") + fmt.Printf("[dryrun] Please go and examine the %q directory for details about what would be written\n", manifestDir) + + // Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests + files := []dryrunutil.FileToPrint{} + for _, component := range kubeadmconstants.MasterComponents { + realPath := kubeadmconstants.GetStaticPodFilepath(component, manifestDir) + outputPath := kubeadmconstants.GetStaticPodFilepath(component, kubeadmconstants.GetStaticPodDirectory()) + files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) + } + + return dryrunutil.PrintDryRunFiles(files, os.Stdout) +} + +// getWaiter gets the right waiter implementation +func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter { + if dryRun { + return dryrunutil.NewWaiter() + } + // TODO: Adjust this timeout slightly? + return apiclient.NewKubeWaiter(client, 30*time.Minute, os.Stdout) +} diff --git a/cmd/kubeadm/app/cmd/phases/BUILD b/cmd/kubeadm/app/cmd/phases/BUILD index 8f683403e7e67..7a673cf283e91 100644 --- a/cmd/kubeadm/app/cmd/phases/BUILD +++ b/cmd/kubeadm/app/cmd/phases/BUILD @@ -25,11 +25,12 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", + "//cmd/kubeadm/app/cmd/util:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", "//cmd/kubeadm/app/phases/controlplane:go_default_library", "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", @@ -38,6 +39,7 @@ go_library( "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//pkg/api:go_default_library", @@ -54,7 +56,6 @@ go_test( "controlplane_test.go", "etcd_test.go", "kubeconfig_test.go", - "phase_test.go", ], library = ":go_default_library", deps = [ diff --git a/cmd/kubeadm/app/cmd/phases/bootstraptoken.go b/cmd/kubeadm/app/cmd/phases/bootstraptoken.go index f9b4535872000..40a5f1d0a0265 100644 --- a/cmd/kubeadm/app/cmd/phases/bootstraptoken.go +++ b/cmd/kubeadm/app/cmd/phases/bootstraptoken.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" clientset "k8s.io/client-go/kubernetes" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -36,7 +37,7 @@ func NewCmdBootstrapToken() *cobra.Command { Use: "bootstrap-token", Short: "Manage kubeadm-specific Bootstrap Token functions.", Aliases: []string{"bootstraptoken"}, - RunE: subCmdRunE("bootstrap-token"), + RunE: cmdutil.SubCmdRunE("bootstrap-token"), } cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster") @@ -55,7 +56,7 @@ func NewSubCmdClusterInfo(kubeConfigFile *string) *cobra.Command { Short: "Uploads and exposes the cluster-info ConfigMap publicly from the given cluster-info file", Aliases: []string{"clusterinfo"}, Run: func(cmd *cobra.Command, args []string) { - err := validateExactArgNumber(args, []string{"clusterinfo-file"}) + err := cmdutil.ValidateExactArgNumber(args, []string{"clusterinfo-file"}) kubeadmutil.CheckErr(err) client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) @@ -81,7 +82,7 @@ func NewSubCmdNodeBootstrapToken(kubeConfigFile *string) *cobra.Command { Use: "node", Short: "Manages Node Bootstrap Tokens", Aliases: []string{"clusterinfo"}, - RunE: subCmdRunE("node"), + RunE: cmdutil.SubCmdRunE("node"), } cmd.AddCommand(NewSubCmdNodeBootstrapTokenPostCSRs(kubeConfigFile)) diff --git a/cmd/kubeadm/app/cmd/phases/certs.go b/cmd/kubeadm/app/cmd/phases/certs.go index aa8c23ce1be67..6749552a1c9fb 100644 --- a/cmd/kubeadm/app/cmd/phases/certs.go +++ b/cmd/kubeadm/app/cmd/phases/certs.go @@ -17,17 +17,13 @@ limitations under the License. package phases import ( - "crypto/rsa" - "crypto/x509" - "fmt" - "github.com/spf13/cobra" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" "k8s.io/kubernetes/pkg/api" @@ -39,7 +35,7 @@ func NewCmdCerts() *cobra.Command { Use: "certs", Aliases: []string{"certificates"}, Short: "Generate certificates for a Kubernetes cluster.", - RunE: subCmdRunE("certs"), + RunE: cmdutil.SubCmdRunE("certs"), } cmd.AddCommand(getCertsSubCommands()...) @@ -64,37 +60,37 @@ func getCertsSubCommands() []*cobra.Command { { use: "all", short: "Generate all PKI assets necessary to establish the control plane", - cmdFunc: CreatePKIAssets, + cmdFunc: certsphase.CreatePKIAssets, }, { use: "ca", short: "Generate CA certificate and key for a Kubernetes cluster.", - cmdFunc: createOrUseCACertAndKey, + cmdFunc: certsphase.CreateCACertAndKeyfiles, }, { use: "apiserver", short: "Generate API Server serving certificate and key.", - cmdFunc: createOrUseAPIServerCertAndKey, + cmdFunc: certsphase.CreateAPIServerCertAndKeyFiles, }, { use: "apiserver-kubelet-client", short: "Generate a client certificate for the API Server to connect to the kubelets securely.", - cmdFunc: createOrUseAPIServerKubeletClientCertAndKey, + cmdFunc: certsphase.CreateAPIServerKubeletClientCertAndKeyFiles, }, { use: "sa", short: "Generate a private key for signing service account tokens along with its public key.", - cmdFunc: createOrUseServiceAccountKeyAndPublicKey, + cmdFunc: certsphase.CreateServiceAccountKeyAndPublicKeyFiles, }, { use: "front-proxy-ca", short: "Generate front proxy CA certificate and key for a Kubernetes cluster.", - cmdFunc: createOrUseFrontProxyCACertAndKey, + cmdFunc: certsphase.CreateFrontProxyCACertAndKeyFiles, }, { use: "front-proxy-client", short: "Generate front proxy CA client certificate and key for a Kubernetes cluster.", - cmdFunc: createOrUseFrontProxyClientCertAndKey, + cmdFunc: certsphase.CreateFrontProxyClientCertAndKeyFiles, }, } @@ -131,6 +127,9 @@ func runCmdFunc(cmdFunc func(cfg *kubeadmapi.MasterConfiguration) error, cfgPath // are shared between sub commands and gets access to current value e.g. flags value. return func(cmd *cobra.Command, args []string) { + if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { + kubeadmutil.CheckErr(err) + } // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(*cfgPath, cfg) @@ -141,228 +140,3 @@ func runCmdFunc(cmdFunc func(cfg *kubeadmapi.MasterConfiguration) error, cfgPath kubeadmutil.CheckErr(err) } } - -// CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane. -// Please note that this action is a bulk action calling all the atomic certphase actions -func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error { - - certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{ - createOrUseCACertAndKey, - createOrUseAPIServerCertAndKey, - createOrUseAPIServerKubeletClientCertAndKey, - createOrUseServiceAccountKeyAndPublicKey, - createOrUseFrontProxyCACertAndKey, - createOrUseFrontProxyClientCertAndKey, - } - - for _, action := range certActions { - err := action(cfg) - if err != nil { - return err - } - } - - fmt.Printf("[certificates] Valid certificates and keys now exist in %q\n", cfg.CertificatesDir) - - return nil -} - -// createOrUseCACertAndKey create a new self signed CA, or use the existing one. -func createOrUseCACertAndKey(cfg *kubeadmapi.MasterConfiguration) error { - - return createOrUseCertificateAuthorithy( - cfg.CertificatesDir, - kubeadmconstants.CACertAndKeyBaseName, - "CA", - certphase.NewCACertAndKey, - ) -} - -// createOrUseAPIServerCertAndKey create a new CA certificate for apiserver, or use the existing one. -// It assumes the CA certificates should exists into the CertificatesDir -func createOrUseAPIServerCertAndKey(cfg *kubeadmapi.MasterConfiguration) error { - - return createOrUseSignedCertificate( - cfg.CertificatesDir, - kubeadmconstants.CACertAndKeyBaseName, - kubeadmconstants.APIServerCertAndKeyBaseName, - "API server", - func(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) { - return certphase.NewAPIServerCertAndKey(cfg, caCert, caKey) - }, - ) -} - -// create a new CA certificate for kubelets calling apiserver, or use the existing one -// It assumes the CA certificates should exists into the CertificatesDir -func createOrUseAPIServerKubeletClientCertAndKey(cfg *kubeadmapi.MasterConfiguration) error { - - return createOrUseSignedCertificate( - cfg.CertificatesDir, - kubeadmconstants.CACertAndKeyBaseName, - kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, - "API server kubelet client", - certphase.NewAPIServerKubeletClientCertAndKey, - ) -} - -// createOrUseServiceAccountKeyAndPublicKey create a new public/private key pairs for signing service account user, or use the existing one. -func createOrUseServiceAccountKeyAndPublicKey(cfg *kubeadmapi.MasterConfiguration) error { - - return createOrUseKeyAndPublicKey( - cfg.CertificatesDir, - kubeadmconstants.ServiceAccountKeyBaseName, - "service account", - certphase.NewServiceAccountSigningKey, - ) -} - -// createOrUseFrontProxyCACertAndKey create a new self signed front proxy CA, or use the existing one. -func createOrUseFrontProxyCACertAndKey(cfg *kubeadmapi.MasterConfiguration) error { - - return createOrUseCertificateAuthorithy( - cfg.CertificatesDir, - kubeadmconstants.FrontProxyCACertAndKeyBaseName, - "front-proxy CA", - certphase.NewFrontProxyCACertAndKey, - ) -} - -// createOrUseFrontProxyClientCertAndKey create a new certificate for proxy server client, or use the existing one. -// It assumes the front proxy CA certificates should exists into the CertificatesDir -func createOrUseFrontProxyClientCertAndKey(cfg *kubeadmapi.MasterConfiguration) error { - - return createOrUseSignedCertificate( - cfg.CertificatesDir, - kubeadmconstants.FrontProxyCACertAndKeyBaseName, - kubeadmconstants.FrontProxyClientCertAndKeyBaseName, - "front-proxy client", - certphase.NewFrontProxyClientCertAndKey, - ) -} - -// createOrUseCertificateAuthorithy is a generic function that will create a new certificate Authorithy using the given newFunc, -// assign file names according to the given baseName, or use the existing one already present in pkiDir. -func createOrUseCertificateAuthorithy(pkiDir string, baseName string, UXName string, newFunc func() (*x509.Certificate, *rsa.PrivateKey, error)) error { - - // If cert or key exists, we should try to load them - if pkiutil.CertOrKeyExist(pkiDir, baseName) { - - // Try to load .crt and .key from the PKI directory - caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName) - if err != nil { - return fmt.Errorf("failure loading %s certificate: %v", UXName, err) - } - - // Check if the existing cert is a CA - if !caCert.IsCA { - return fmt.Errorf("certificate %s is not a CA", UXName) - } - - fmt.Printf("[certificates] Using the existing %s certificate and key.\n", UXName) - } else { - // The certificate and the key did NOT exist, let's generate them now - caCert, caKey, err := newFunc() - if err != nil { - return fmt.Errorf("failure while generating %s certificate and key: %v", UXName, err) - } - - // Write .crt and .key files to disk - if err = pkiutil.WriteCertAndKey(pkiDir, baseName, caCert, caKey); err != nil { - return fmt.Errorf("failure while saving %s certificate and key: %v", UXName, err) - } - - fmt.Printf("[certificates] Generated %s certificate and key.\n", UXName) - } - return nil -} - -// createOrUseSignedCertificate is a generic function that will create a new signed certificate using the given newFunc, -// assign file names according to the given baseName, or use the existing one already present in pkiDir. -func createOrUseSignedCertificate(pkiDir string, CABaseName string, baseName string, UXName string, newFunc func(*x509.Certificate, *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error)) error { - - // Checks if certificate authorithy exists in the PKI directory - if !pkiutil.CertOrKeyExist(pkiDir, CABaseName) { - return fmt.Errorf("couldn't load certificate authorithy for %s from certificate dir", UXName) - } - - // Try to load certificate authorithy .crt and .key from the PKI directory - caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, CABaseName) - if err != nil { - return fmt.Errorf("failure loading certificate authorithy for %s: %v", UXName, err) - } - - // Make sure the loaded CA cert actually is a CA - if !caCert.IsCA { - return fmt.Errorf("certificate authorithy for %s is not a CA", UXName) - } - - // Checks if the signed certificate exists in the PKI directory - if pkiutil.CertOrKeyExist(pkiDir, baseName) { - // Try to load signed certificate .crt and .key from the PKI directory - signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName) - if err != nil { - return fmt.Errorf("failure loading %s certificate: %v", UXName, err) - } - - // Check if the existing cert is signed by the given CA - if err := signedCert.CheckSignatureFrom(caCert); err != nil { - return fmt.Errorf("certificate %s is not signed by corresponding CA", UXName) - } - - fmt.Printf("[certificates] Using the existing %s certificate and key.\n", UXName) - } else { - // The certificate and the key did NOT exist, let's generate them now - signedCert, signedKey, err := newFunc(caCert, caKey) - if err != nil { - return fmt.Errorf("failure while generating %s key and certificate: %v", UXName, err) - } - - // Write .crt and .key files to disk - if err = pkiutil.WriteCertAndKey(pkiDir, baseName, signedCert, signedKey); err != nil { - return fmt.Errorf("failure while saving %s certificate and key: %v", UXName, err) - } - - fmt.Printf("[certificates] Generated %s certificate and key.\n", UXName) - if pkiutil.HasServerAuth(signedCert) { - fmt.Printf("[certificates] %s serving cert is signed for DNS names %v and IPs %v\n", UXName, signedCert.DNSNames, signedCert.IPAddresses) - } - } - - return nil -} - -// createOrUseKeyAndPublicKey is a generic function that will create a new public/private key pairs using the given newFunc, -// assign file names according to the given baseName, or use the existing one already present in pkiDir. -func createOrUseKeyAndPublicKey(pkiDir string, baseName string, UXName string, newFunc func() (*rsa.PrivateKey, error)) error { - - // Checks if the key exists in the PKI directory - if pkiutil.CertOrKeyExist(pkiDir, baseName) { - - // Try to load .key from the PKI directory - _, err := pkiutil.TryLoadKeyFromDisk(pkiDir, baseName) - if err != nil { - return fmt.Errorf("%s key existed but they could not be loaded properly: %v", UXName, err) - } - - fmt.Printf("[certificates] Using the existing %s key.\n", UXName) - } else { - // The key does NOT exist, let's generate it now - key, err := newFunc() - if err != nil { - return fmt.Errorf("failure while generating %s key: %v", UXName, err) - } - - // Write .key and .pub files to disk - if err = pkiutil.WriteKey(pkiDir, baseName, key); err != nil { - return fmt.Errorf("failure while saving %s key: %v", UXName, err) - } - - if err = pkiutil.WritePublicKey(pkiDir, baseName, &key.PublicKey); err != nil { - return fmt.Errorf("failure while saving %s public key: %v", UXName, err) - } - fmt.Printf("[certificates] Generated %s key and public key.\n", UXName) - } - - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/certs_test.go b/cmd/kubeadm/app/cmd/phases/certs_test.go index 8cb5e7414e200..abf67f6992302 100644 --- a/cmd/kubeadm/app/cmd/phases/certs_test.go +++ b/cmd/kubeadm/app/cmd/phases/certs_test.go @@ -33,7 +33,61 @@ import ( cmdtestutil "k8s.io/kubernetes/cmd/kubeadm/test/cmd" ) -func TestSubCmdCertsCreateFiles(t *testing.T) { +func TestCertsSubCommandsHasFlags(t *testing.T) { + + subCmds := getCertsSubCommands() + + commonFlags := []string{ + "cert-dir", + "config", + } + + var tests = []struct { + command string + additionalFlags []string + }{ + { + command: "all", + additionalFlags: []string{ + "apiserver-advertise-address", + "apiserver-cert-extra-sans", + "service-cidr", + "service-dns-domain", + }, + }, + { + command: "ca", + }, + { + command: "apiserver", + additionalFlags: []string{ + "apiserver-advertise-address", + "apiserver-cert-extra-sans", + "service-cidr", + "service-dns-domain", + }, + }, + { + command: "apiserver-kubelet-client", + }, + { + command: "sa", + }, + { + command: "front-proxy-ca", + }, + { + command: "front-proxy-client", + }, + } + + for _, test := range tests { + expectedFlags := append(commonFlags, test.additionalFlags...) + cmdtestutil.AssertSubCommandHasFlags(t, subCmds, test.command, expectedFlags...) + } +} + +func TestSubCmdCertsCreateFilesWithFlags(t *testing.T) { subCmds := getCertsSubCommands() @@ -53,25 +107,13 @@ func TestSubCmdCertsCreateFiles(t *testing.T) { }, }, { - subCmds: []string{"ca"}, - expectedFiles: []string{kubeadmconstants.CACertName, kubeadmconstants.CAKeyName}, - }, - { - subCmds: []string{"ca", "apiserver"}, - expectedFiles: []string{kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName}, - }, - { - subCmds: []string{"ca", "apiserver-kubelet-client"}, - expectedFiles: []string{kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName}, + subCmds: []string{"ca", "apiserver", "apiserver-kubelet-client"}, + expectedFiles: []string{kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName, kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName}, }, { subCmds: []string{"sa"}, expectedFiles: []string{kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName}, }, - { - subCmds: []string{"front-proxy-ca"}, - expectedFiles: []string{kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName}, - }, { subCmds: []string{"front-proxy-ca", "front-proxy-client"}, expectedFiles: []string{kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName, kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName}, @@ -94,7 +136,7 @@ func TestSubCmdCertsCreateFiles(t *testing.T) { } } -func TestSubCmdApiServerFlags(t *testing.T) { +func TestSubCmdCertsApiServerForwardsFlags(t *testing.T) { subCmds := getCertsSubCommands() @@ -116,6 +158,7 @@ func TestSubCmdApiServerFlags(t *testing.T) { } cmdtestutil.RunSubCommand(t, subCmds, "apiserver", apiserverFlags...) + // asserts created cert has values from CLI flags APIserverCert, err := pkiutil.TryLoadCertFromDisk(tmpdir, kubeadmconstants.APIServerCertAndKeyBaseName) if err != nil { t.Fatalf("Error loading API server certificate: %v", err) @@ -135,29 +178,36 @@ func TestSubCmdApiServerFlags(t *testing.T) { } } -func TestSubCmdCertsReadsConfig(t *testing.T) { +func TestSubCmdCertsCreateFilesWithConfigFile(t *testing.T) { subCmds := getCertsSubCommands() var tests = []struct { - subCmds []string - expectedFileCount int + subCmds []string + expectedFiles []string }{ { - subCmds: []string{"sa"}, - expectedFileCount: 2, + subCmds: []string{"all"}, + expectedFiles: []string{ + kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, + kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName, + kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName, + kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName, + kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName, + kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName, + }, }, { - subCmds: []string{"front-proxy-ca", "front-proxy-client"}, - expectedFileCount: 4, + subCmds: []string{"ca", "apiserver", "apiserver-kubelet-client"}, + expectedFiles: []string{kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName, kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName}, }, { - subCmds: []string{"ca", "apiserver", "apiserver-kubelet-client"}, - expectedFileCount: 6, + subCmds: []string{"front-proxy-ca", "front-proxy-client"}, + expectedFiles: []string{kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName, kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName}, }, { - subCmds: []string{"all"}, - expectedFileCount: 12, + subCmds: []string{"sa"}, + expectedFiles: []string{kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName}, }, } @@ -182,6 +232,6 @@ func TestSubCmdCertsReadsConfig(t *testing.T) { } // verify expected files are there - testutil.AssertFilesCount(t, tmpdir, test.expectedFileCount) + testutil.AssertFileExists(t, tmpdir, test.expectedFiles...) } } diff --git a/cmd/kubeadm/app/cmd/phases/controlplane.go b/cmd/kubeadm/app/cmd/phases/controlplane.go index bc9bc5c9dec23..06571fed3948a 100644 --- a/cmd/kubeadm/app/cmd/phases/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/controlplane.go @@ -21,6 +21,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" "k8s.io/kubernetes/pkg/api" @@ -31,7 +32,7 @@ func NewCmdControlplane() *cobra.Command { cmd := &cobra.Command{ Use: "controlplane", Short: "Generate all static pod manifest files necessary to establish the control plane.", - RunE: subCmdRunE("controlplane"), + RunE: cmdutil.SubCmdRunE("controlplane"), } manifestPath := kubeadmconstants.GetStaticPodDirectory() diff --git a/cmd/kubeadm/app/cmd/phases/etcd.go b/cmd/kubeadm/app/cmd/phases/etcd.go index 7d205160fe518..f3b9ab865f2f6 100644 --- a/cmd/kubeadm/app/cmd/phases/etcd.go +++ b/cmd/kubeadm/app/cmd/phases/etcd.go @@ -21,6 +21,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" "k8s.io/kubernetes/pkg/api" @@ -31,7 +32,7 @@ func NewCmdEtcd() *cobra.Command { cmd := &cobra.Command{ Use: "etcd", Short: "Generate static pod manifest file for etcd.", - RunE: subCmdRunE("etcd"), + RunE: cmdutil.SubCmdRunE("etcd"), } manifestPath := kubeadmconstants.GetStaticPodDirectory() diff --git a/cmd/kubeadm/app/cmd/phases/kubeconfig.go b/cmd/kubeadm/app/cmd/phases/kubeconfig.go index a8b144563ea35..47b88e80f2eae 100644 --- a/cmd/kubeadm/app/cmd/phases/kubeconfig.go +++ b/cmd/kubeadm/app/cmd/phases/kubeconfig.go @@ -24,6 +24,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" "k8s.io/kubernetes/pkg/api" @@ -34,7 +35,7 @@ func NewCmdKubeConfig(out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "kubeconfig", Short: "Generate all kubeconfig files necessary to establish the control plane and the admin kubeconfig file.", - RunE: subCmdRunE("kubeconfig"), + RunE: cmdutil.SubCmdRunE("kubeconfig"), } cmd.AddCommand(getKubeConfigSubCommands(out, kubeadmconstants.KubernetesDir)...) diff --git a/cmd/kubeadm/app/cmd/phases/markmaster.go b/cmd/kubeadm/app/cmd/phases/markmaster.go index 3cdd00c8d60f0..89457fdfc7640 100644 --- a/cmd/kubeadm/app/cmd/phases/markmaster.go +++ b/cmd/kubeadm/app/cmd/phases/markmaster.go @@ -17,10 +17,9 @@ limitations under the License. package phases import ( - "fmt" - "github.com/spf13/cobra" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" @@ -34,15 +33,13 @@ func NewCmdMarkMaster() *cobra.Command { Short: "Mark a node as master.", Aliases: []string{"markmaster"}, RunE: func(_ *cobra.Command, args []string) error { - err := validateExactArgNumber(args, []string{"node-name"}) + err := cmdutil.ValidateExactArgNumber(args, []string{"node-name"}) kubeadmutil.CheckErr(err) client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) kubeadmutil.CheckErr(err) nodeName := args[0] - fmt.Printf("[markmaster] Will mark node %s as master by adding a label and a taint\n", nodeName) - return markmasterphase.MarkMaster(client, nodeName) }, } diff --git a/cmd/kubeadm/app/cmd/phases/phase.go b/cmd/kubeadm/app/cmd/phases/phase.go index fd6edbfed03d8..f0a33c2b59e6c 100644 --- a/cmd/kubeadm/app/cmd/phases/phase.go +++ b/cmd/kubeadm/app/cmd/phases/phase.go @@ -17,10 +17,10 @@ limitations under the License. package phases import ( - "fmt" "io" "github.com/spf13/cobra" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" ) // NewCmdPhase returns the cobra command for the "kubeadm phase" command (currently alpha-gated) @@ -28,7 +28,7 @@ func NewCmdPhase(out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "phase", Short: "Invoke subsets of kubeadm functions separately for a manual install.", - RunE: subCmdRunE("phase"), + RunE: cmdutil.SubCmdRunE("phase"), } cmd.AddCommand(NewCmdBootstrapToken()) @@ -43,37 +43,3 @@ func NewCmdPhase(out io.Writer) *cobra.Command { return cmd } - -// subCmdRunE returns a function that handles a case where a subcommand must be specified -// Without this callback, if a user runs just the command without a subcommand, -// or with an invalid subcommand, cobra will print usage information, but still exit cleanly. -// We want to return an error code in these cases so that the -// user knows that their command was invalid. -func subCmdRunE(name string) func(*cobra.Command, []string) error { - return func(_ *cobra.Command, args []string) error { - if len(args) < 1 { - return fmt.Errorf("missing subcommand; %q is not meant to be run on its own", name) - } - - return fmt.Errorf("invalid subcommand: %q", args[0]) - } -} - -// validateExactArgNumber validates that the required top-level arguments are specified -func validateExactArgNumber(args []string, supportedArgs []string) error { - validArgs := 0 - // Disregard possible "" arguments; they are invalid - for _, arg := range args { - if len(arg) > 0 { - validArgs++ - } - } - - if validArgs < len(supportedArgs) { - return fmt.Errorf("missing one or more required arguments. Required arguments: %v", supportedArgs) - } - if validArgs > len(supportedArgs) { - return fmt.Errorf("too many arguments, only %d argument(s) supported: %v", validArgs, supportedArgs) - } - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/preflight.go b/cmd/kubeadm/app/cmd/phases/preflight.go index b16a8e01e6ca1..f47e35126c891 100644 --- a/cmd/kubeadm/app/cmd/phases/preflight.go +++ b/cmd/kubeadm/app/cmd/phases/preflight.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/cobra" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" ) @@ -27,7 +28,7 @@ func NewCmdPreFlight() *cobra.Command { cmd := &cobra.Command{ Use: "preflight", Short: "Run pre-flight checks", - RunE: subCmdRunE("preflight"), + RunE: cmdutil.SubCmdRunE("preflight"), } cmd.AddCommand(NewCmdPreFlightMaster()) diff --git a/cmd/kubeadm/app/cmd/phases/selfhosting.go b/cmd/kubeadm/app/cmd/phases/selfhosting.go index 2cd4ca1475963..7011a9b776176 100644 --- a/cmd/kubeadm/app/cmd/phases/selfhosting.go +++ b/cmd/kubeadm/app/cmd/phases/selfhosting.go @@ -17,36 +17,87 @@ limitations under the License. package phases import ( + "os" + "strings" + "time" + "github.com/spf13/cobra" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/api" ) // NewCmdSelfhosting returns the self-hosting Cobra command func NewCmdSelfhosting() *cobra.Command { - var kubeConfigFile string - cfg := &kubeadmapiext.MasterConfiguration{} cmd := &cobra.Command{ Use: "selfhosting", Aliases: []string{"selfhosted"}, Short: "Make a kubeadm cluster self-hosted.", + RunE: cmdutil.SubCmdRunE("selfhosting"), + } + + cmd.AddCommand(getSelfhostingSubCommand()) + return cmd +} + +// getSelfhostingSubCommand returns sub commands for Selfhosting phase +func getSelfhostingSubCommand() *cobra.Command { + + cfg := &kubeadmapiext.MasterConfiguration{} + // Default values for the cobra help text + api.Scheme.Default(cfg) + + var cfgPath, kubeConfigFile, featureGatesString string + + // Creates the UX Command + cmd := &cobra.Command{ + Use: "convert-from-staticpods", + Aliases: []string{"from-staticpods"}, + Short: "Converts a Static Pod-hosted control plane into a self-hosted one.", Run: func(cmd *cobra.Command, args []string) { - api.Scheme.Default(cfg) - internalcfg := &kubeadmapi.MasterConfiguration{} - api.Scheme.Convert(cfg, internalcfg, nil) + var err error + if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { + kubeadmutil.CheckErr(err) + } + + if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { + kubeadmutil.CheckErr(err) + } + + // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) + kubeadmutil.CheckErr(err) + + // Gets the kubernetes client client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) kubeadmutil.CheckErr(err) - err = selfhosting.CreateSelfHostedControlPlane(internalcfg, client) + // Converts the Static Pod-hosted control plane into a self-hosted one + waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout) + err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter) kubeadmutil.CheckErr(err) }, } + // Add flags to the command + // flags bound to the configuration object + cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where certificates are stored`) + cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + cmd.Flags().StringVar(&featureGatesString, "feature-gates", featureGatesString, "A set of key=value pairs that describe feature gates for various features."+ + "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) + + // flags that are not bound to the configuration object + // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster") + return cmd } diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index 779b290d6942a..b55bc917622b0 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -113,7 +113,7 @@ func (r *Reset) Run(out io.Writer) error { fmt.Println("[reset] docker doesn't seem to be running, skipping the removal of running kubernetes containers") } - dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim"} + dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"} // Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user // provided external etcd endpoints. In that case, it is his own responsibility to reset etcd diff --git a/cmd/kubeadm/app/cmd/token.go b/cmd/kubeadm/app/cmd/token.go index 730356605bb6b..f21b22934b2c4 100644 --- a/cmd/kubeadm/app/cmd/token.go +++ b/cmd/kubeadm/app/cmd/token.go @@ -17,9 +17,9 @@ limitations under the License. package cmd import ( - "errors" "fmt" "io" + "os" "sort" "strings" "text/tabwriter" @@ -31,11 +31,14 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token" "k8s.io/kubernetes/pkg/api" @@ -46,6 +49,7 @@ import ( func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { var kubeConfigFile string + var dryRun bool tokenCmd := &cobra.Command{ Use: "token", Short: "Manage bootstrap tokens.", @@ -75,19 +79,16 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { // cobra will print usage information, but still exit cleanly. // We want to return an error code in these cases so that the // user knows that their command was invalid. - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return errors.New("missing subcommand; 'token' is not meant to be run on its own") - } else { - return fmt.Errorf("invalid subcommand: %s", args[0]) - } - }, + RunE: cmdutil.SubCmdRunE("token"), } tokenCmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster") + tokenCmd.PersistentFlags().BoolVar(&dryRun, + "dry-run", dryRun, "Whether to enable dry-run mode or not") var usages []string + var extraGroups []string var tokenDuration time.Duration var description string createCmd := &cobra.Command{ @@ -106,7 +107,7 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { if len(args) != 0 { token = args[0] } - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) + client, err := getClientset(kubeConfigFile, dryRun) kubeadmutil.CheckErr(err) // TODO: remove this warning in 1.9 @@ -115,7 +116,7 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { fmt.Fprintln(errW, "[kubeadm] WARNING: starting in 1.8, tokens expire after 24 hours by default (if you require a non-expiring token use --ttl 0)") } - err = RunCreateToken(out, client, token, tokenDuration, usages, description) + err = RunCreateToken(out, client, token, tokenDuration, usages, extraGroups, description) kubeadmutil.CheckErr(err) }, } @@ -123,6 +124,9 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { "ttl", kubeadmconstants.DefaultTokenDuration, "The duration before the token is automatically deleted (e.g. 1s, 2m, 3h). 0 means 'never expires'.") createCmd.Flags().StringSliceVar(&usages, "usages", kubeadmconstants.DefaultTokenUsages, "The ways in which this token can be used. Valid options: [signing,authentication].") + createCmd.Flags().StringSliceVar(&extraGroups, + "groups", []string{}, + fmt.Sprintf("Extra groups that this token will authenticate as when used for authentication. Must match %q.", bootstrapapi.BootstrapGroupPattern)) createCmd.Flags().StringVar(&description, "description", "", "A human friendly description of how this token is used.") tokenCmd.AddCommand(createCmd) @@ -136,7 +140,7 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { This command will list all Bootstrap Tokens for you. `), Run: func(tokenCmd *cobra.Command, args []string) { - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) + client, err := getClientset(kubeConfigFile, dryRun) kubeadmutil.CheckErr(err) err = RunListTokens(out, errW, client) @@ -158,7 +162,7 @@ func NewCmdToken(out io.Writer, errW io.Writer) *cobra.Command { if len(args) < 1 { kubeadmutil.CheckErr(fmt.Errorf("missing subcommand; 'token delete' is missing token of form [%q]", tokenutil.TokenIDRegexpString)) } - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) + client, err := getClientset(kubeConfigFile, dryRun) kubeadmutil.CheckErr(err) err = RunDeleteToken(out, client, args[0]) @@ -193,7 +197,7 @@ func NewCmdTokenGenerate(out io.Writer) *cobra.Command { } // RunCreateToken generates a new bootstrap token and stores it as a secret on the server. -func RunCreateToken(out io.Writer, client clientset.Interface, token string, tokenDuration time.Duration, usages []string, description string) error { +func RunCreateToken(out io.Writer, client clientset.Interface, token string, tokenDuration time.Duration, usages []string, extraGroups []string, description string) error { if len(token) == 0 { var err error @@ -208,8 +212,22 @@ func RunCreateToken(out io.Writer, client clientset.Interface, token string, tok } } + // adding groups only makes sense for authentication + var usagesSet sets.String + usagesSet.Insert(usages...) + if len(extraGroups) > 0 && !usagesSet.Has("authentication") { + return fmt.Errorf("--groups cannot be specified unless --usages includes \"authentication\"") + } + + // validate any extra group names + for _, group := range extraGroups { + if err := bootstrapapi.ValidateBootstrapGroupName(group); err != nil { + return err + } + } + // TODO: Validate usages here so we don't allow something unsupported - err := tokenphase.CreateNewToken(client, token, tokenDuration, usages, description) + err := tokenphase.CreateNewToken(client, token, tokenDuration, usages, extraGroups, description) if err != nil { return err } @@ -247,7 +265,7 @@ func RunListTokens(out io.Writer, errW io.Writer, client clientset.Interface) er } w := tabwriter.NewWriter(out, 10, 4, 3, ' ', 0) - fmt.Fprintln(w, "TOKEN\tTTL\tEXPIRES\tUSAGES\tDESCRIPTION") + fmt.Fprintln(w, "TOKEN\tTTL\tEXPIRES\tUSAGES\tDESCRIPTION\tEXTRA GROUPS") for _, secret := range secrets.Items { tokenId := getSecretString(&secret, bootstrapapi.BootstrapTokenIDKey) if len(tokenId) == 0 { @@ -305,7 +323,12 @@ func RunListTokens(out io.Writer, errW io.Writer, client clientset.Interface) er if len(description) == 0 { description = "" } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", tokenutil.BearerToken(td), ttl, expires, usageString, description) + + groups := getSecretString(&secret, bootstrapapi.BootstrapTokenExtraGroupsKey) + if len(groups) == 0 { + groups = "" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", tokenutil.BearerToken(td), ttl, expires, usageString, description, groups) } w.Flush() return nil @@ -338,3 +361,15 @@ func getSecretString(secret *v1.Secret, key string) string { } return "" } + +func getClientset(file string, dryRun bool) (clientset.Interface, error) { + if dryRun { + dryRunGetter, err := apiclient.NewClientBackedDryRunGetterFromKubeconfig(file) + if err != nil { + return nil, err + } + return apiclient.NewDryRunClient(dryRunGetter, os.Stdout), nil + } + client, err := kubeconfigutil.ClientSetFromFile(file) + return client, err +} diff --git a/cmd/kubeadm/app/cmd/upgrade/BUILD b/cmd/kubeadm/app/cmd/upgrade/BUILD new file mode 100644 index 0000000000000..dc34c1134b00e --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/BUILD @@ -0,0 +1,55 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "apply.go", + "common.go", + "plan.go", + "upgrade.go", + ], + visibility = ["//visibility:public"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", + "//cmd/kubeadm/app/cmd/util:go_default_library", + "//cmd/kubeadm/app/phases/upgrade:go_default_library", + "//cmd/kubeadm/app/preflight:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", + "//cmd/kubeadm/app/util/kubeconfig:go_default_library", + "//pkg/api:go_default_library", + "//pkg/util/version:go_default_library", + "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/github.com/spf13/cobra:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "apply_test.go", + "common_test.go", + "plan_test.go", + ], + library = ":go_default_library", + deps = [ + "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", + "//cmd/kubeadm/app/phases/upgrade:go_default_library", + "//pkg/util/version:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go new file mode 100644 index 0000000000000..319029ce07806 --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -0,0 +1,213 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" + + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/util/version" +) + +// applyFlags holds the information about the flags that can be passed to apply +type applyFlags struct { + nonInteractiveMode bool + force bool + dryRun bool + newK8sVersionStr string + newK8sVersion *version.Version + imagePullTimeout time.Duration + parent *cmdUpgradeFlags +} + +// SessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run) +func (f *applyFlags) SessionIsInteractive() bool { + return !f.nonInteractiveMode +} + +// NewCmdApply returns the cobra command for `kubeadm upgrade apply` +func NewCmdApply(parentFlags *cmdUpgradeFlags) *cobra.Command { + flags := &applyFlags{ + parent: parentFlags, + imagePullTimeout: 15 * time.Minute, + } + + cmd := &cobra.Command{ + Use: "apply [version]", + Short: "Upgrade your Kubernetes cluster to the specified version", + Run: func(cmd *cobra.Command, args []string) { + // Ensure the user is root + err := runPreflightChecks(flags.parent.skipPreFlight) + kubeadmutil.CheckErr(err) + + err = cmdutil.ValidateExactArgNumber(args, []string{"version"}) + kubeadmutil.CheckErr(err) + + // It's safe to use args[0] here as the slice has been validated above + flags.newK8sVersionStr = args[0] + + // Default the flags dynamically, based on each others' value + err = SetImplicitFlags(flags) + kubeadmutil.CheckErr(err) + + err = RunApply(flags) + kubeadmutil.CheckErr(err) + }, + } + + // Specify the valid flags specific for apply + cmd.Flags().BoolVarP(&flags.nonInteractiveMode, "yes", "y", flags.nonInteractiveMode, "Perform the upgrade and do not prompt for confirmation (non-interactive mode).") + cmd.Flags().BoolVarP(&flags.force, "force", "f", flags.force, "Force upgrading although some requirements might not be met. This also implies non-interactive mode.") + cmd.Flags().BoolVar(&flags.dryRun, "dry-run", flags.dryRun, "Do not change any state, just output what actions would be applied.") + cmd.Flags().DurationVar(&flags.imagePullTimeout, "image-pull-timeout", flags.imagePullTimeout, "The maximum amount of time to wait for the control plane pods to be downloaded.") + + return cmd +} + +// RunApply takes care of the actual upgrade functionality +// It does the following things: +// - Checks if the cluster is healthy +// - Gets the configuration from the kubeadm-config ConfigMap in the cluster +// - Enforces all version skew policies +// - Asks the user if they really want to upgrade +// - Makes sure the control plane images are available locally on the master(s) +// - Upgrades the control plane components +// - Applies the other resources that'd be created with kubeadm init as well, like +// - Creating the RBAC rules for the Bootstrap Tokens and the cluster-info ConfigMap +// - Applying new kube-dns and kube-proxy manifests +// - Uploads the newly used configuration to the cluster ConfigMap +func RunApply(flags *applyFlags) error { + + // Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap) + upgradeVars, err := enforceRequirements(flags.parent.kubeConfigPath, flags.parent.cfgPath, flags.parent.printConfig) + if err != nil { + return err + } + + // Set the upgraded version on the external config object now + upgradeVars.cfg.KubernetesVersion = flags.newK8sVersionStr + + // Grab the external, versioned configuration and convert it to the internal type for usage here later + internalcfg := &kubeadmapi.MasterConfiguration{} + api.Scheme.Convert(upgradeVars.cfg, internalcfg, nil) + + // Enforce the version skew policies + if err := EnforceVersionPolicies(flags, upgradeVars.versionGetter); err != nil { + return fmt.Errorf("[upgrade/version] FATAL: %v", err) + } + + // If the current session is interactive, ask the user whether they really want to upgrade + if flags.SessionIsInteractive() { + if err := InteractivelyConfirmUpgrade("Are you sure you want to proceed with the upgrade?"); err != nil { + return err + } + } + + // TODO: Implement a prepulling mechanism here + + // Now; perform the upgrade procedure + if err := PerformControlPlaneUpgrade(flags, upgradeVars.client, internalcfg); err != nil { + return fmt.Errorf("[upgrade/apply] FATAL: %v", err) + } + + // Upgrade RBAC rules and addons. Optionally, if needed, perform some extra task for a specific version + if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg, flags.newK8sVersion); err != nil { + return fmt.Errorf("[upgrade/postupgrade] FATAL: %v", err) + } + + fmt.Println("") + fmt.Printf("[upgrade/successful] SUCCESS! Your cluster was upgraded to %q. Enjoy!\n", flags.newK8sVersionStr) + fmt.Println("") + fmt.Println("[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets in turn.") + + return nil +} + +// SetImplicitFlags handles dynamically defaulting flags based on each other's value +func SetImplicitFlags(flags *applyFlags) error { + // If we are in dry-run or force mode; we should automatically execute this command non-interactively + if flags.dryRun || flags.force { + flags.nonInteractiveMode = true + } + + k8sVer, err := version.ParseSemantic(flags.newK8sVersionStr) + if err != nil { + return fmt.Errorf("couldn't parse version %q as a semantic version", flags.newK8sVersionStr) + } + flags.newK8sVersion = k8sVer + + // Automatically add the "v" prefix to the string representation in case it doesn't exist + if !strings.HasPrefix(flags.newK8sVersionStr, "v") { + flags.newK8sVersionStr = fmt.Sprintf("v%s", flags.newK8sVersionStr) + } + + return nil +} + +// EnforceVersionPolicies makes sure that the version the user specified is valid to upgrade to +// There are both fatal and skippable (with --force) errors +func EnforceVersionPolicies(flags *applyFlags, versionGetter upgrade.VersionGetter) error { + fmt.Printf("[upgrade/version] You have chosen to upgrade to version %q\n", flags.newK8sVersionStr) + + versionSkewErrs := upgrade.EnforceVersionPolicies(versionGetter, flags.newK8sVersionStr, flags.newK8sVersion, flags.parent.allowExperimentalUpgrades, flags.parent.allowRCUpgrades) + if versionSkewErrs != nil { + + if len(versionSkewErrs.Mandatory) > 0 { + return fmt.Errorf("The --version argument is invalid due to these fatal errors: %v", versionSkewErrs.Mandatory) + } + + if len(versionSkewErrs.Skippable) > 0 { + // Return the error if the user hasn't specified the --force flag + if !flags.force { + return fmt.Errorf("The --version argument is invalid due to these errors: %v. Can be bypassed if you pass the --force flag", versionSkewErrs.Mandatory) + } + // Soft errors found, but --force was specified + fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: %v\n", len(versionSkewErrs.Skippable), versionSkewErrs.Skippable) + } + } + return nil +} + +// PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted) +func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, internalcfg *kubeadmapi.MasterConfiguration) error { + + // Check if the cluster is self-hosted and act accordingly + if upgrade.IsControlPlaneSelfHosted(client) { + fmt.Printf("[upgrade/apply] Upgrading your Self-Hosted control plane to version %q...\n", flags.newK8sVersionStr) + + // Upgrade a self-hosted cluster + // TODO(luxas): Implement this later when we have the new upgrade strategy + return fmt.Errorf("not implemented") + } + + // OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster + fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", flags.newK8sVersionStr) + + if err := upgrade.PerformStaticPodControlPlaneUpgrade(client, internalcfg, flags.newK8sVersion); err != nil { + return err + } + return nil +} diff --git a/cmd/kubeadm/app/cmd/upgrade/apply_test.go b/cmd/kubeadm/app/cmd/upgrade/apply_test.go new file mode 100644 index 0000000000000..8f8b15845352b --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/apply_test.go @@ -0,0 +1,194 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "reflect" + "testing" + + "k8s.io/kubernetes/pkg/util/version" +) + +func TestSetImplicitFlags(t *testing.T) { + var tests = []struct { + flags *applyFlags + expectedFlags applyFlags + errExpected bool + }{ + { // if not dryRun or force is set; the nonInteractiveMode field should not be touched + flags: &applyFlags{ + newK8sVersionStr: "v1.8.0", + dryRun: false, + force: false, + nonInteractiveMode: false, + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "v1.8.0", + newK8sVersion: version.MustParseSemantic("v1.8.0"), + dryRun: false, + force: false, + nonInteractiveMode: false, + }, + }, + { // if not dryRun or force is set; the nonInteractiveMode field should not be touched + flags: &applyFlags{ + newK8sVersionStr: "v1.8.0", + dryRun: false, + force: false, + nonInteractiveMode: true, + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "v1.8.0", + newK8sVersion: version.MustParseSemantic("v1.8.0"), + dryRun: false, + force: false, + nonInteractiveMode: true, + }, + }, + { // if dryRun or force is set; the nonInteractiveMode field should be set to true + flags: &applyFlags{ + newK8sVersionStr: "v1.8.0", + dryRun: true, + force: false, + nonInteractiveMode: false, + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "v1.8.0", + newK8sVersion: version.MustParseSemantic("v1.8.0"), + dryRun: true, + force: false, + nonInteractiveMode: true, + }, + }, + { // if dryRun or force is set; the nonInteractiveMode field should be set to true + flags: &applyFlags{ + newK8sVersionStr: "v1.8.0", + dryRun: false, + force: true, + nonInteractiveMode: false, + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "v1.8.0", + newK8sVersion: version.MustParseSemantic("v1.8.0"), + dryRun: false, + force: true, + nonInteractiveMode: true, + }, + }, + { // if dryRun or force is set; the nonInteractiveMode field should be set to true + flags: &applyFlags{ + newK8sVersionStr: "v1.8.0", + dryRun: true, + force: true, + nonInteractiveMode: false, + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "v1.8.0", + newK8sVersion: version.MustParseSemantic("v1.8.0"), + dryRun: true, + force: true, + nonInteractiveMode: true, + }, + }, + { // if dryRun or force is set; the nonInteractiveMode field should be set to true + flags: &applyFlags{ + newK8sVersionStr: "v1.8.0", + dryRun: true, + force: true, + nonInteractiveMode: true, + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "v1.8.0", + newK8sVersion: version.MustParseSemantic("v1.8.0"), + dryRun: true, + force: true, + nonInteractiveMode: true, + }, + }, + { // if the new version is empty; it should error out + flags: &applyFlags{ + newK8sVersionStr: "", + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "", + }, + errExpected: true, + }, + { // if the new version is invalid; it should error out + flags: &applyFlags{ + newK8sVersionStr: "foo", + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "foo", + }, + errExpected: true, + }, + { // if the new version is valid but without the "v" prefix; it parse and prepend v + flags: &applyFlags{ + newK8sVersionStr: "1.8.0", + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "v1.8.0", + newK8sVersion: version.MustParseSemantic("v1.8.0"), + }, + errExpected: false, + }, + { // valid version should succeed + flags: &applyFlags{ + newK8sVersionStr: "v1.8.1", + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "v1.8.1", + newK8sVersion: version.MustParseSemantic("v1.8.1"), + }, + errExpected: false, + }, + { // valid version should succeed + flags: &applyFlags{ + newK8sVersionStr: "1.8.0-alpha.3", + }, + expectedFlags: applyFlags{ + newK8sVersionStr: "v1.8.0-alpha.3", + newK8sVersion: version.MustParseSemantic("v1.8.0-alpha.3"), + }, + errExpected: false, + }, + } + for _, rt := range tests { + actualErr := SetImplicitFlags(rt.flags) + + // If an error was returned; make newK8sVersion nil so it's easy to match using reflect.DeepEqual later (instead of a random pointer) + if actualErr != nil { + rt.flags.newK8sVersion = nil + } + + if !reflect.DeepEqual(*rt.flags, rt.expectedFlags) { + t.Errorf( + "failed SetImplicitFlags:\n\texpected flags: %v\n\t actual: %v", + rt.expectedFlags, + *rt.flags, + ) + } + if (actualErr != nil) != rt.errExpected { + t.Errorf( + "failed SetImplicitFlags:\n\texpected error: %t\n\t actual: %t", + rt.errExpected, + (actualErr != nil), + ) + } + } +} diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go new file mode 100644 index 0000000000000..4a64d37011bfa --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -0,0 +1,120 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "strings" + + "github.com/ghodss/yaml" + + clientset "k8s.io/client-go/kubernetes" + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + "k8s.io/kubernetes/cmd/kubeadm/app/preflight" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" +) + +// upgradeVariables holds variables needed for performing an upgrade or planning to do so +// TODO - Restructure or rename upgradeVariables +type upgradeVariables struct { + client clientset.Interface + cfg *kubeadmapiext.MasterConfiguration + versionGetter upgrade.VersionGetter +} + +// enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure +func enforceRequirements(kubeConfigPath, cfgPath string, printConfig bool) (*upgradeVariables, error) { + client, err := kubeconfigutil.ClientSetFromFile(kubeConfigPath) + if err != nil { + return nil, fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", kubeConfigPath, err) + } + + // Run healthchecks against the cluster + if err := upgrade.CheckClusterHealth(client); err != nil { + return nil, fmt.Errorf("[upgrade/health] FATAL: %v", err) + } + + // Fetch the configuration from a file or ConfigMap and validate it + cfg, err := upgrade.FetchConfiguration(client, os.Stdout, cfgPath) + if err != nil { + return nil, fmt.Errorf("[upgrade/config] FATAL: %v", err) + } + + // If the user told us to print this information out; do it! + if printConfig { + printConfiguration(cfg, os.Stdout) + } + + return &upgradeVariables{ + client: client, + cfg: cfg, + // Use a real version getter interface that queries the API server, the kubeadm client and the Kubernetes CI system for latest versions + versionGetter: upgrade.NewKubeVersionGetter(client, os.Stdout), + }, nil +} + +// printConfiguration prints the external version of the API to yaml +func printConfiguration(cfg *kubeadmapiext.MasterConfiguration, w io.Writer) { + // Short-circuit if cfg is nil, so we can safely get the value of the pointer below + if cfg == nil { + return + } + + cfgYaml, err := yaml.Marshal(*cfg) + if err == nil { + fmt.Fprintln(w, "[upgrade/config] Configuration used:") + + scanner := bufio.NewScanner(bytes.NewReader(cfgYaml)) + for scanner.Scan() { + fmt.Fprintf(w, "\t%s\n", scanner.Text()) + } + } +} + +// runPreflightChecks runs the root preflight check +func runPreflightChecks(skipPreFlight bool) error { + if skipPreFlight { + fmt.Println("[preflight] Skipping pre-flight checks") + return nil + } + + fmt.Println("[preflight] Running pre-flight checks") + return preflight.RunRootCheckOnly() +} + +// InteractivelyConfirmUpgrade asks the user whether they _really_ want to upgrade. +func InteractivelyConfirmUpgrade(question string) error { + + fmt.Printf("[upgrade/confirm] %s [y/N]: ", question) + + scanner := bufio.NewScanner(os.Stdin) + scanner.Scan() + if err := scanner.Err(); err != nil { + return fmt.Errorf("couldn't read from standard input: %v", err) + } + answer := scanner.Text() + if strings.ToLower(answer) == "y" || strings.ToLower(answer) == "yes" { + return nil + } + + return fmt.Errorf("won't proceed; the user didn't answer (Y|y) in order to continue") +} diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go new file mode 100644 index 0000000000000..5904d3b9a20e1 --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/common_test.go @@ -0,0 +1,124 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "bytes" + "testing" + + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" +) + +func TestPrintConfiguration(t *testing.T) { + var tests = []struct { + cfg *kubeadmapiext.MasterConfiguration + buf *bytes.Buffer + expectedBytes []byte + }{ + { + cfg: nil, + expectedBytes: []byte(""), + }, + { + cfg: &kubeadmapiext.MasterConfiguration{ + KubernetesVersion: "v1.7.1", + }, + expectedBytes: []byte(`[upgrade/config] Configuration used: + api: + advertiseAddress: "" + bindPort: 0 + apiServerCertSANs: null + apiServerExtraArgs: null + authorizationModes: null + certificatesDir: "" + cloudProvider: "" + controllerManagerExtraArgs: null + etcd: + caFile: "" + certFile: "" + dataDir: "" + endpoints: null + extraArgs: null + image: "" + keyFile: "" + featureGates: null + imageRepository: "" + kubernetesVersion: v1.7.1 + networking: + dnsDomain: "" + podSubnet: "" + serviceSubnet: "" + nodeName: "" + schedulerExtraArgs: null + token: "" + tokenTTL: 0 + unifiedControlPlaneImage: "" +`), + }, + { + cfg: &kubeadmapiext.MasterConfiguration{ + KubernetesVersion: "v1.7.1", + Networking: kubeadmapiext.Networking{ + ServiceSubnet: "10.96.0.1/12", + }, + }, + expectedBytes: []byte(`[upgrade/config] Configuration used: + api: + advertiseAddress: "" + bindPort: 0 + apiServerCertSANs: null + apiServerExtraArgs: null + authorizationModes: null + certificatesDir: "" + cloudProvider: "" + controllerManagerExtraArgs: null + etcd: + caFile: "" + certFile: "" + dataDir: "" + endpoints: null + extraArgs: null + image: "" + keyFile: "" + featureGates: null + imageRepository: "" + kubernetesVersion: v1.7.1 + networking: + dnsDomain: "" + podSubnet: "" + serviceSubnet: 10.96.0.1/12 + nodeName: "" + schedulerExtraArgs: null + token: "" + tokenTTL: 0 + unifiedControlPlaneImage: "" +`), + }, + } + for _, rt := range tests { + rt.buf = bytes.NewBufferString("") + printConfiguration(rt.cfg, rt.buf) + actualBytes := rt.buf.Bytes() + if !bytes.Equal(actualBytes, rt.expectedBytes) { + t.Errorf( + "failed PrintConfiguration:\n\texpected: %q\n\t actual: %q", + string(rt.expectedBytes), + string(actualBytes), + ) + } + } +} diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go new file mode 100644 index 0000000000000..545b362ae85a8 --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -0,0 +1,142 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "fmt" + "io" + "os" + "sort" + "text/tabwriter" + + "github.com/spf13/cobra" + + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" +) + +// NewCmdPlan returns the cobra command for `kubeadm upgrade plan` +func NewCmdPlan(parentFlags *cmdUpgradeFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "plan", + Short: "Check which versions are available to upgrade to and validate whether your current cluster is upgradeable", + Run: func(_ *cobra.Command, _ []string) { + // Ensure the user is root + err := runPreflightChecks(parentFlags.skipPreFlight) + kubeadmutil.CheckErr(err) + + err = RunPlan(parentFlags) + kubeadmutil.CheckErr(err) + }, + } + + return cmd +} + +// RunPlan takes care of outputting available versions to upgrade to for the user +func RunPlan(parentFlags *cmdUpgradeFlags) error { + + // Start with the basics, verify that the cluster is healthy, build a client and a versionGetter. + upgradeVars, err := enforceRequirements(parentFlags.kubeConfigPath, parentFlags.cfgPath, parentFlags.printConfig) + if err != nil { + return err + } + + // Compute which upgrade possibilities there are + availUpgrades, err := upgrade.GetAvailableUpgrades(upgradeVars.versionGetter, parentFlags.allowExperimentalUpgrades, parentFlags.allowRCUpgrades) + if err != nil { + return err + } + + // Tell the user which upgrades are available + printAvailableUpgrades(availUpgrades, os.Stdout) + return nil +} + +// printAvailableUpgrades prints a UX-friendly overview of what versions are available to upgrade to +// TODO look into columnize or some other formatter when time permits instead of using the tabwriter +func printAvailableUpgrades(upgrades []upgrade.Upgrade, w io.Writer) { + + // Return quickly if no upgrades can be made + if len(upgrades) == 0 { + fmt.Fprintln(w, "Awesome, you're up-to-date! Enjoy!") + return + } + // The tab writer writes to the "real" writer w + tabw := tabwriter.NewWriter(w, 10, 4, 3, ' ', 0) + + // Loop through the upgrade possibilities and output text to the command line + for _, upgrade := range upgrades { + + if upgrade.CanUpgradeKubelets() { + fmt.Fprintln(w, "Components that must be upgraded manually after you've upgraded the control plane with 'kubeadm upgrade apply':") + fmt.Fprintln(tabw, "COMPONENT\tCURRENT\tAVAILABLE") + firstPrinted := false + + // The map is of the form :. Here all the keys are put into a slice and sorted + // in order to always get the right order. Then the map value is extracted separately + for _, oldVersion := range sortedSliceFromStringIntMap(upgrade.Before.KubeletVersions) { + nodeCount := upgrade.Before.KubeletVersions[oldVersion] + if !firstPrinted { + // Output the Kubelet header only on the first version pair + fmt.Fprintf(tabw, "Kubelet\t%d x %s\t%s\n", nodeCount, oldVersion, upgrade.After.KubeVersion) + firstPrinted = true + continue + } + fmt.Fprintf(tabw, "\t\t%d x %s\t%s\n", nodeCount, oldVersion, upgrade.After.KubeVersion) + } + // We should flush the writer here at this stage; as the columns will now be of the right size, adjusted to the above content + tabw.Flush() + fmt.Fprintln(w, "") + } + + fmt.Fprintf(w, "Upgrade to the latest %s:\n", upgrade.Description) + fmt.Fprintln(w, "") + fmt.Fprintln(tabw, "COMPONENT\tCURRENT\tAVAILABLE") + fmt.Fprintf(tabw, "API Server\t%s\t%s\n", upgrade.Before.KubeVersion, upgrade.After.KubeVersion) + fmt.Fprintf(tabw, "Controller Manager\t%s\t%s\n", upgrade.Before.KubeVersion, upgrade.After.KubeVersion) + fmt.Fprintf(tabw, "Scheduler\t%s\t%s\n", upgrade.Before.KubeVersion, upgrade.After.KubeVersion) + fmt.Fprintf(tabw, "Kube Proxy\t%s\t%s\n", upgrade.Before.KubeVersion, upgrade.After.KubeVersion) + fmt.Fprintf(tabw, "Kube DNS\t%s\t%s\n", upgrade.Before.DNSVersion, upgrade.After.DNSVersion) + + // The tabwriter should be flushed at this stage as we have now put in all the required content for this time. This is required for the tabs' size to be correct. + tabw.Flush() + fmt.Fprintln(w, "") + fmt.Fprintln(w, "You can now apply the upgrade by executing the following command:") + fmt.Fprintln(w, "") + fmt.Fprintf(w, "\tkubeadm upgrade apply %s\n", upgrade.After.KubeVersion) + fmt.Fprintln(w, "") + + if upgrade.Before.KubeadmVersion != upgrade.After.KubeadmVersion { + fmt.Fprintf(w, "Note: Before you do can perform this upgrade, you have to update kubeadm to %s\n", upgrade.After.KubeadmVersion) + fmt.Fprintln(w, "") + } + + fmt.Fprintln(w, "_____________________________________________________________________") + fmt.Fprintln(w, "") + } +} + +// sortedSliceFromStringIntMap returns a slice of the keys in the map sorted alphabetically +func sortedSliceFromStringIntMap(strMap map[string]uint16) []string { + strSlice := []string{} + for k := range strMap { + strSlice = append(strSlice, k) + } + sort.Strings(strSlice) + return strSlice +} diff --git a/cmd/kubeadm/app/cmd/upgrade/plan_test.go b/cmd/kubeadm/app/cmd/upgrade/plan_test.go new file mode 100644 index 0000000000000..bddef5c393cef --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/plan_test.go @@ -0,0 +1,329 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "bytes" + "reflect" + "testing" + + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" +) + +func TestSortedSliceFromStringIntMap(t *testing.T) { + var tests = []struct { + strMap map[string]uint16 + expectedSlice []string + }{ // The returned slice should be alphabetically sorted based on the string keys in the map + { + strMap: map[string]uint16{"foo": 1, "bar": 2}, + expectedSlice: []string{"bar", "foo"}, + }, + { // The int value should not affect this func + strMap: map[string]uint16{"foo": 2, "bar": 1}, + expectedSlice: []string{"bar", "foo"}, + }, + { + strMap: map[string]uint16{"b": 2, "a": 1, "cb": 0, "ca": 1000}, + expectedSlice: []string{"a", "b", "ca", "cb"}, + }, + { // This should work for version numbers as well; and the lowest version should come first + strMap: map[string]uint16{"v1.7.0": 1, "v1.6.1": 1, "v1.6.2": 1, "v1.8.0": 1, "v1.8.0-alpha.1": 1}, + expectedSlice: []string{"v1.6.1", "v1.6.2", "v1.7.0", "v1.8.0", "v1.8.0-alpha.1"}, + }, + } + for _, rt := range tests { + actualSlice := sortedSliceFromStringIntMap(rt.strMap) + if !reflect.DeepEqual(actualSlice, rt.expectedSlice) { + t.Errorf( + "failed SortedSliceFromStringIntMap:\n\texpected: %v\n\t actual: %v", + rt.expectedSlice, + actualSlice, + ) + } + } +} + +// TODO Think about modifying this test to be less verbose checking b/c it can be brittle. +func TestPrintAvailableUpgrades(t *testing.T) { + var tests = []struct { + upgrades []upgrade.Upgrade + buf *bytes.Buffer + expectedBytes []byte + }{ + { + upgrades: []upgrade.Upgrade{}, + expectedBytes: []byte(`Awesome, you're up-to-date! Enjoy! +`), + }, + { + upgrades: []upgrade.Upgrade{ + { + Description: "version in the v1.7 series", + Before: upgrade.ClusterState{ + KubeVersion: "v1.7.1", + KubeletVersions: map[string]uint16{ + "v1.7.1": 1, + }, + KubeadmVersion: "v1.7.2", + DNSVersion: "1.14.4", + }, + After: upgrade.ClusterState{ + KubeVersion: "v1.7.3", + KubeadmVersion: "v1.7.3", + DNSVersion: "1.14.4", + }, + }, + }, + expectedBytes: []byte(`Components that must be upgraded manually after you've upgraded the control plane with 'kubeadm upgrade apply': +COMPONENT CURRENT AVAILABLE +Kubelet 1 x v1.7.1 v1.7.3 + +Upgrade to the latest version in the v1.7 series: + +COMPONENT CURRENT AVAILABLE +API Server v1.7.1 v1.7.3 +Controller Manager v1.7.1 v1.7.3 +Scheduler v1.7.1 v1.7.3 +Kube Proxy v1.7.1 v1.7.3 +Kube DNS 1.14.4 1.14.4 + +You can now apply the upgrade by executing the following command: + + kubeadm upgrade apply v1.7.3 + +Note: Before you do can perform this upgrade, you have to update kubeadm to v1.7.3 + +_____________________________________________________________________ + +`), + }, + { + upgrades: []upgrade.Upgrade{ + { + Description: "stable version", + Before: upgrade.ClusterState{ + KubeVersion: "v1.7.3", + KubeletVersions: map[string]uint16{ + "v1.7.3": 1, + }, + KubeadmVersion: "v1.8.0", + DNSVersion: "1.14.4", + }, + After: upgrade.ClusterState{ + KubeVersion: "v1.8.0", + KubeadmVersion: "v1.8.0", + DNSVersion: "1.14.4", + }, + }, + }, + expectedBytes: []byte(`Components that must be upgraded manually after you've upgraded the control plane with 'kubeadm upgrade apply': +COMPONENT CURRENT AVAILABLE +Kubelet 1 x v1.7.3 v1.8.0 + +Upgrade to the latest stable version: + +COMPONENT CURRENT AVAILABLE +API Server v1.7.3 v1.8.0 +Controller Manager v1.7.3 v1.8.0 +Scheduler v1.7.3 v1.8.0 +Kube Proxy v1.7.3 v1.8.0 +Kube DNS 1.14.4 1.14.4 + +You can now apply the upgrade by executing the following command: + + kubeadm upgrade apply v1.8.0 + +_____________________________________________________________________ + +`), + }, + { + upgrades: []upgrade.Upgrade{ + { + Description: "version in the v1.7 series", + Before: upgrade.ClusterState{ + KubeVersion: "v1.7.3", + KubeletVersions: map[string]uint16{ + "v1.7.3": 1, + }, + KubeadmVersion: "v1.8.1", + DNSVersion: "1.14.4", + }, + After: upgrade.ClusterState{ + KubeVersion: "v1.7.5", + KubeadmVersion: "v1.8.1", + DNSVersion: "1.14.4", + }, + }, + { + Description: "stable version", + Before: upgrade.ClusterState{ + KubeVersion: "v1.7.3", + KubeletVersions: map[string]uint16{ + "v1.7.3": 1, + }, + KubeadmVersion: "v1.8.1", + DNSVersion: "1.14.4", + }, + After: upgrade.ClusterState{ + KubeVersion: "v1.8.2", + KubeadmVersion: "v1.8.2", + DNSVersion: "1.14.4", + }, + }, + }, + expectedBytes: []byte(`Components that must be upgraded manually after you've upgraded the control plane with 'kubeadm upgrade apply': +COMPONENT CURRENT AVAILABLE +Kubelet 1 x v1.7.3 v1.7.5 + +Upgrade to the latest version in the v1.7 series: + +COMPONENT CURRENT AVAILABLE +API Server v1.7.3 v1.7.5 +Controller Manager v1.7.3 v1.7.5 +Scheduler v1.7.3 v1.7.5 +Kube Proxy v1.7.3 v1.7.5 +Kube DNS 1.14.4 1.14.4 + +You can now apply the upgrade by executing the following command: + + kubeadm upgrade apply v1.7.5 + +_____________________________________________________________________ + +Components that must be upgraded manually after you've upgraded the control plane with 'kubeadm upgrade apply': +COMPONENT CURRENT AVAILABLE +Kubelet 1 x v1.7.3 v1.8.2 + +Upgrade to the latest stable version: + +COMPONENT CURRENT AVAILABLE +API Server v1.7.3 v1.8.2 +Controller Manager v1.7.3 v1.8.2 +Scheduler v1.7.3 v1.8.2 +Kube Proxy v1.7.3 v1.8.2 +Kube DNS 1.14.4 1.14.4 + +You can now apply the upgrade by executing the following command: + + kubeadm upgrade apply v1.8.2 + +Note: Before you do can perform this upgrade, you have to update kubeadm to v1.8.2 + +_____________________________________________________________________ + +`), + }, + { + upgrades: []upgrade.Upgrade{ + { + Description: "experimental version", + Before: upgrade.ClusterState{ + KubeVersion: "v1.7.5", + KubeletVersions: map[string]uint16{ + "v1.7.5": 1, + }, + KubeadmVersion: "v1.7.5", + DNSVersion: "1.14.4", + }, + After: upgrade.ClusterState{ + KubeVersion: "v1.8.0-beta.1", + KubeadmVersion: "v1.8.0-beta.1", + DNSVersion: "1.14.4", + }, + }, + }, + expectedBytes: []byte(`Components that must be upgraded manually after you've upgraded the control plane with 'kubeadm upgrade apply': +COMPONENT CURRENT AVAILABLE +Kubelet 1 x v1.7.5 v1.8.0-beta.1 + +Upgrade to the latest experimental version: + +COMPONENT CURRENT AVAILABLE +API Server v1.7.5 v1.8.0-beta.1 +Controller Manager v1.7.5 v1.8.0-beta.1 +Scheduler v1.7.5 v1.8.0-beta.1 +Kube Proxy v1.7.5 v1.8.0-beta.1 +Kube DNS 1.14.4 1.14.4 + +You can now apply the upgrade by executing the following command: + + kubeadm upgrade apply v1.8.0-beta.1 + +Note: Before you do can perform this upgrade, you have to update kubeadm to v1.8.0-beta.1 + +_____________________________________________________________________ + +`), + }, + { + upgrades: []upgrade.Upgrade{ + { + Description: "release candidate version", + Before: upgrade.ClusterState{ + KubeVersion: "v1.7.5", + KubeletVersions: map[string]uint16{ + "v1.7.5": 1, + }, + KubeadmVersion: "v1.7.5", + DNSVersion: "1.14.4", + }, + After: upgrade.ClusterState{ + KubeVersion: "v1.8.0-rc.1", + KubeadmVersion: "v1.8.0-rc.1", + DNSVersion: "1.14.4", + }, + }, + }, + expectedBytes: []byte(`Components that must be upgraded manually after you've upgraded the control plane with 'kubeadm upgrade apply': +COMPONENT CURRENT AVAILABLE +Kubelet 1 x v1.7.5 v1.8.0-rc.1 + +Upgrade to the latest release candidate version: + +COMPONENT CURRENT AVAILABLE +API Server v1.7.5 v1.8.0-rc.1 +Controller Manager v1.7.5 v1.8.0-rc.1 +Scheduler v1.7.5 v1.8.0-rc.1 +Kube Proxy v1.7.5 v1.8.0-rc.1 +Kube DNS 1.14.4 1.14.4 + +You can now apply the upgrade by executing the following command: + + kubeadm upgrade apply v1.8.0-rc.1 + +Note: Before you do can perform this upgrade, you have to update kubeadm to v1.8.0-rc.1 + +_____________________________________________________________________ + +`), + }, + } + for _, rt := range tests { + rt.buf = bytes.NewBufferString("") + printAvailableUpgrades(rt.upgrades, rt.buf) + actualBytes := rt.buf.Bytes() + if !bytes.Equal(actualBytes, rt.expectedBytes) { + t.Errorf( + "failed PrintAvailableUpgrades:\n\texpected: %q\n\t actual: %q", + string(rt.expectedBytes), + string(actualBytes), + ) + } + } +} diff --git a/cmd/kubeadm/app/cmd/upgrade/upgrade.go b/cmd/kubeadm/app/cmd/upgrade/upgrade.go new file mode 100644 index 0000000000000..9f9b14f08ff13 --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/upgrade.go @@ -0,0 +1,64 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "io" + + "github.com/spf13/cobra" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" +) + +// cmdUpgradeFlags holds the values for the common flags in `kubeadm upgrade` +type cmdUpgradeFlags struct { + kubeConfigPath string + cfgPath string + allowExperimentalUpgrades bool + allowRCUpgrades bool + printConfig bool + skipPreFlight bool +} + +// NewCmdUpgrade returns the cobra command for `kubeadm upgrade` +func NewCmdUpgrade(out io.Writer) *cobra.Command { + flags := &cmdUpgradeFlags{ + kubeConfigPath: "/etc/kubernetes/admin.conf", + cfgPath: "", + allowExperimentalUpgrades: false, + allowRCUpgrades: false, + printConfig: false, + skipPreFlight: false, + } + + cmd := &cobra.Command{ + Use: "upgrade", + Short: "Upgrade your cluster smoothly to a newer version with this command.", + RunE: cmdutil.SubCmdRunE("upgrade"), + } + + cmd.PersistentFlags().StringVar(&flags.kubeConfigPath, "kubeconfig", flags.kubeConfigPath, "The KubeConfig file to use for talking to the cluster.") + cmd.PersistentFlags().StringVar(&flags.cfgPath, "config", flags.cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental).") + cmd.PersistentFlags().BoolVar(&flags.allowExperimentalUpgrades, "allow-experimental-upgrades", flags.allowExperimentalUpgrades, "Show unstable versions of Kubernetes as an upgrade alternative and allow upgrading to an alpha/beta/release candidate versions of Kubernetes.") + cmd.PersistentFlags().BoolVar(&flags.allowRCUpgrades, "allow-release-candidate-upgrades", flags.allowRCUpgrades, "Show release candidate versions of Kubernetes as an upgrade alternative and allow upgrading to a release candidate versions of Kubernetes.") + cmd.PersistentFlags().BoolVar(&flags.printConfig, "print-config", flags.printConfig, "Whether the configuration file that will be used in the upgrade should be printed or not.") + cmd.PersistentFlags().BoolVar(&flags.skipPreFlight, "skip-preflight-checks", flags.skipPreFlight, "Skip preflight checks normally run before modifying the system") + + cmd.AddCommand(NewCmdApply(flags)) + cmd.AddCommand(NewCmdPlan(flags)) + + return cmd +} diff --git a/cmd/kubeadm/app/cmd/util/BUILD b/cmd/kubeadm/app/cmd/util/BUILD new file mode 100644 index 0000000000000..cb123d65d2a60 --- /dev/null +++ b/cmd/kubeadm/app/cmd/util/BUILD @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["cmdutil.go"], + visibility = ["//visibility:public"], + deps = ["//vendor/github.com/spf13/cobra:go_default_library"], +) + +go_test( + name = "go_default_test", + srcs = ["cmdutil_test.go"], + library = ":go_default_library", +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kubeadm/app/cmd/util/cmdutil.go b/cmd/kubeadm/app/cmd/util/cmdutil.go new file mode 100644 index 0000000000000..3c0e7d65b1043 --- /dev/null +++ b/cmd/kubeadm/app/cmd/util/cmdutil.go @@ -0,0 +1,57 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// SubCmdRunE returns a function that handles a case where a subcommand must be specified +// Without this callback, if a user runs just the command without a subcommand, +// or with an invalid subcommand, cobra will print usage information, but still exit cleanly. +// We want to return an error code in these cases so that the +// user knows that their command was invalid. +func SubCmdRunE(name string) func(*cobra.Command, []string) error { + return func(_ *cobra.Command, args []string) error { + if len(args) < 1 { + return fmt.Errorf("missing subcommand; %q is not meant to be run on its own", name) + } + + return fmt.Errorf("invalid subcommand: %q", args[0]) + } +} + +// ValidateExactArgNumber validates that the required top-level arguments are specified +func ValidateExactArgNumber(args []string, supportedArgs []string) error { + validArgs := 0 + // Disregard possible "" arguments; they are invalid + for _, arg := range args { + if len(arg) > 0 { + validArgs++ + } + } + + if validArgs < len(supportedArgs) { + return fmt.Errorf("missing one or more required arguments. Required arguments: %v", supportedArgs) + } + if validArgs > len(supportedArgs) { + return fmt.Errorf("too many arguments, only %d argument(s) supported: %v", validArgs, supportedArgs) + } + return nil +} diff --git a/cmd/kubeadm/app/cmd/phases/phase_test.go b/cmd/kubeadm/app/cmd/util/cmdutil_test.go similarity index 92% rename from cmd/kubeadm/app/cmd/phases/phase_test.go rename to cmd/kubeadm/app/cmd/util/cmdutil_test.go index 67a5283337fc6..ef4d81ce009a0 100644 --- a/cmd/kubeadm/app/cmd/phases/phase_test.go +++ b/cmd/kubeadm/app/cmd/util/cmdutil_test.go @@ -52,10 +52,10 @@ func TestValidateExactArgNumber(t *testing.T) { }, } for _, rt := range tests { - actual := validateExactArgNumber(rt.args, rt.supportedArgs) + actual := ValidateExactArgNumber(rt.args, rt.supportedArgs) if (actual != nil) != rt.expectedErr { t.Errorf( - "failed validateExactArgNumber:\n\texpected error: %t\n\t actual error: %t", + "failed ValidateExactArgNumber:\n\texpected error: %t\n\t actual error: %t", rt.expectedErr, (actual != nil), ) diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 8a5a98c0ab7e7..6c93720085dd0 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -113,9 +113,18 @@ const ( // SelfHostingPrefix describes the prefix workloads that are self-hosted by kubeadm has SelfHostingPrefix = "self-hosted-" + // KubeCertificatesVolumeName specifies the name for the Volume that is used for injecting certificates to control plane components (can be both a hostPath volume or a projected, all-in-one volume) + KubeCertificatesVolumeName = "k8s-certs" + + // KubeConfigVolumeName specifies the name for the Volume that is used for injecting the kubeconfig to talk securely to the api server for a control plane component if applicable + KubeConfigVolumeName = "kubeconfig" + // NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in // TODO: This should be changed in the v1.8 dev cycle to a node-BT-specific group instead of the generic Bootstrap Token group that is used now NodeBootstrapTokenAuthGroup = "system:bootstrappers" + + // DefaultCIImageRepository points to image registry where CI uploads images from ci-cross build job + DefaultCIImageRepository = "gcr.io/kubernetes-ci-images" ) var ( @@ -147,6 +156,10 @@ var ( // MinimumCSRAutoApprovalClusterRolesVersion defines whether kubeadm can rely on the built-in CSR approval ClusterRole or not (note, the binding is always created by kubeadm!) // TODO: Remove this when the v1.9 cycle starts and we bump the minimum supported version to v1.8.0 MinimumCSRAutoApprovalClusterRolesVersion = version.MustParseSemantic("v1.8.0-alpha.3") + + // UseEnableBootstrapTokenAuthFlagVersion defines the first version where the API server supports the --enable-bootstrap-token-auth flag instead of the old and deprecated flag. + // TODO: Remove this when the v1.9 cycle starts and we bump the minimum supported version to v1.8.0 + UseEnableBootstrapTokenAuthFlagVersion = version.MustParseSemantic("v1.8.0-beta.0") ) // GetStaticPodDirectory returns the location on the disk where the Static Pod should be present diff --git a/cmd/kubeadm/app/cmd/features/BUILD b/cmd/kubeadm/app/features/BUILD similarity index 71% rename from cmd/kubeadm/app/cmd/features/BUILD rename to cmd/kubeadm/app/features/BUILD index 2341efdef2d38..1dcb86304462f 100644 --- a/cmd/kubeadm/app/cmd/features/BUILD +++ b/cmd/kubeadm/app/features/BUILD @@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( @@ -23,3 +24,10 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["features_test.go"], + library = ":go_default_library", + deps = ["//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library"], +) diff --git a/cmd/kubeadm/app/cmd/features/features.go b/cmd/kubeadm/app/features/features.go similarity index 59% rename from cmd/kubeadm/app/cmd/features/features.go rename to cmd/kubeadm/app/features/features.go index 62fa02574c6ce..767d2128230b2 100644 --- a/cmd/kubeadm/app/cmd/features/features.go +++ b/cmd/kubeadm/app/features/features.go @@ -17,6 +17,11 @@ limitations under the License. package features import ( + "fmt" + "sort" + "strconv" + "strings" + utilfeature "k8s.io/apiserver/pkg/util/feature" ) @@ -61,3 +66,48 @@ var InitFeatureGates = FeatureList{ SelfHosting: {Default: false, PreRelease: utilfeature.Beta}, StoreCertsInSecrets: {Default: false, PreRelease: utilfeature.Alpha}, } + +// KnownFeatures returns a slice of strings describing the FeatureList features. +func KnownFeatures(f *FeatureList) []string { + var known []string + for k, v := range *f { + pre := "" + if v.PreRelease != utilfeature.GA { + pre = fmt.Sprintf("%s - ", v.PreRelease) + } + known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default)) + } + sort.Strings(known) + return known +} + +// NewFeatureGate parse a string of the form "key1=value1,key2=value2,..." into a +// map[string]bool of known keys or returns an error. +func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { + featureGate := map[string]bool{} + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + + arr := strings.SplitN(s, "=", 2) + if len(arr) != 2 { + return nil, fmt.Errorf("missing bool value for feature-gate key:%s", s) + } + + k := strings.TrimSpace(arr[0]) + v := strings.TrimSpace(arr[1]) + + if !Supports(*f, k) { + return nil, fmt.Errorf("unrecognized feature-gate key: %s", k) + } + + boolValue, err := strconv.ParseBool(v) + if err != nil { + return nil, fmt.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k) + } + featureGate[k] = boolValue + } + + return featureGate, nil +} diff --git a/cmd/kubeadm/app/features/features_test.go b/cmd/kubeadm/app/features/features_test.go new file mode 100644 index 0000000000000..486f10f84fc73 --- /dev/null +++ b/cmd/kubeadm/app/features/features_test.go @@ -0,0 +1,119 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "reflect" + "testing" + + utilfeature "k8s.io/apiserver/pkg/util/feature" +) + +func TestKnownFeatures(t *testing.T) { + var someFeatures = FeatureList{ + "feature2": {Default: true, PreRelease: utilfeature.Alpha}, + "feature1": {Default: false, PreRelease: utilfeature.Beta}, + "feature3": {Default: false, PreRelease: utilfeature.GA}, + } + + r := KnownFeatures(&someFeatures) + + if len(r) != 3 { + t.Errorf("KnownFeatures returned %d values, expected 3", len(r)) + } + + // check the first value is feature1 (the list should be sorted); prerelease and default should be present + f1 := "feature1=true|false (BETA - default=false)" + if r[0] != f1 { + t.Errorf("KnownFeatures returned %s values, expected %s", r[0], f1) + } + // check the second value is feature2; prerelease and default should be present + f2 := "feature2=true|false (ALPHA - default=true)" + if r[1] != f2 { + t.Errorf("KnownFeatures returned %s values, expected %s", r[1], f2) + } + // check the second value is feature3; prerelease should not shown fo GA features; default should be present + f3 := "feature3=true|false (default=false)" + if r[2] != f3 { + t.Errorf("KnownFeatures returned %s values, expected %s", r[2], f3) + } +} + +func TestNewFeatureGate(t *testing.T) { + var someFeatures = FeatureList{ + "feature1": {Default: false, PreRelease: utilfeature.Beta}, + "feature2": {Default: true, PreRelease: utilfeature.Alpha}, + } + + var tests = []struct { + value string + expectedError bool + expectedFeaturesGate map[string]bool + }{ + { //invalid value (missing =) + value: "invalidValue", + expectedError: true, + }, + { //invalid value (missing =) + value: "feature1=true,invalidValue", + expectedError: true, + }, + { //invalid value (not a boolean) + value: "feature1=notABoolean", + expectedError: true, + }, + { //invalid value (not a boolean) + value: "feature1=true,feature2=notABoolean", + expectedError: true, + }, + { //unrecognized feature-gate key + value: "unknownFeature=false", + expectedError: true, + }, + { //unrecognized feature-gate key + value: "feature1=true,unknownFeature=false", + expectedError: true, + }, + { //one feature + value: "feature1=true", + expectedError: false, + expectedFeaturesGate: map[string]bool{"feature1": true}, + }, + { //two features + value: "feature1=true,feature2=false", + expectedError: false, + expectedFeaturesGate: map[string]bool{"feature1": true, "feature2": false}, + }, + } + + for _, test := range tests { + + r, err := NewFeatureGate(&someFeatures, test.value) + + if !test.expectedError && err != nil { + t.Errorf("NewFeatureGate failed when not expected: %v", err) + continue + } else if test.expectedError && err == nil { + t.Error("NewFeatureGate didn't failed when expected") + continue + } + + if !reflect.DeepEqual(r, test.expectedFeaturesGate) { + t.Errorf("NewFeatureGate returned a unexpected value") + } + } +} diff --git a/cmd/kubeadm/app/phases/addons/dns/BUILD b/cmd/kubeadm/app/phases/addons/dns/BUILD index 1d5c808cfab89..22622cc9fa4a4 100644 --- a/cmd/kubeadm/app/phases/addons/dns/BUILD +++ b/cmd/kubeadm/app/phases/addons/dns/BUILD @@ -10,12 +10,16 @@ load( go_test( name = "go_default_test", - srcs = ["dns_test.go"], + srcs = [ + "dns_test.go", + "versions_test.go", + ], library = ":go_default_library", tags = ["automanaged"], deps = [ "//cmd/kubeadm/app/util:go_default_library", "//pkg/api:go_default_library", + "//pkg/util/version:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library", @@ -28,6 +32,7 @@ go_library( srcs = [ "dns.go", "manifests.go", + "versions.go", ], tags = ["automanaged"], deps = [ @@ -36,6 +41,7 @@ go_library( "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//pkg/api:go_default_library", + "//pkg/util/version:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/cmd/kubeadm/app/phases/addons/dns/dns.go b/cmd/kubeadm/app/phases/addons/dns/dns.go index 51e4dae815fe0..135647d6ac2be 100644 --- a/cmd/kubeadm/app/phases/addons/dns/dns.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns.go @@ -32,6 +32,7 @@ import ( kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/util/version" ) const ( @@ -40,17 +41,20 @@ const ( ) // EnsureDNSAddon creates the kube-dns addon -func EnsureDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { +func EnsureDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, k8sVersion *version.Version) error { if err := CreateServiceAccount(client); err != nil { return err } - dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment, struct{ ImageRepository, Arch, Version, DNSDomain, MasterTaintKey string }{ + // Get the YAML manifest conditionally based on the k8s version + kubeDNSDeploymentBytes := GetKubeDNSManifest(k8sVersion) + dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(kubeDNSDeploymentBytes, struct{ ImageRepository, Arch, Version, DNSDomain, MasterTaintKey string }{ ImageRepository: cfg.ImageRepository, Arch: runtime.GOARCH, - Version: KubeDNSVersion, - DNSDomain: cfg.Networking.DNSDomain, - MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster, + // Get the kube-dns version conditionally based on the k8s version + Version: GetKubeDNSVersion(k8sVersion), + DNSDomain: cfg.Networking.DNSDomain, + MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster, }) if err != nil { return fmt.Errorf("error when parsing kube-dns deployment template: %v", err) diff --git a/cmd/kubeadm/app/phases/addons/dns/dns_test.go b/cmd/kubeadm/app/phases/addons/dns/dns_test.go index e65a9d983c101..22b1d444dedca 100644 --- a/cmd/kubeadm/app/phases/addons/dns/dns_test.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns_test.go @@ -90,7 +90,7 @@ func TestCompileManifests(t *testing.T) { expected bool }{ { - manifest: KubeDNSDeployment, + manifest: v170AndAboveKubeDNSDeployment, data: struct{ ImageRepository, Arch, Version, DNSDomain, MasterTaintKey string }{ ImageRepository: "foo", Arch: "foo", diff --git a/cmd/kubeadm/app/phases/addons/dns/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go index 866bd47464757..4aedb03b9f071 100644 --- a/cmd/kubeadm/app/phases/addons/dns/manifests.go +++ b/cmd/kubeadm/app/phases/addons/dns/manifests.go @@ -17,11 +17,8 @@ limitations under the License. package dns const ( - // KubeDNSVersion is the version of kube-dns to run - KubeDNSVersion = "1.14.4" - - // KubeDNSDeployment is the kube-dns Deployemnt manifest - KubeDNSDeployment = ` + // v170AndAboveKubeDNSDeployment is the kube-dns Deployment manifest for the kube-dns manifest for v1.7+ + v170AndAboveKubeDNSDeployment = ` apiVersion: extensions/v1beta1 kind: Deployment metadata: diff --git a/cmd/kubeadm/app/phases/addons/dns/versions.go b/cmd/kubeadm/app/phases/addons/dns/versions.go new file mode 100644 index 0000000000000..bd03155573a6e --- /dev/null +++ b/cmd/kubeadm/app/phases/addons/dns/versions.go @@ -0,0 +1,40 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dns + +import ( + "k8s.io/kubernetes/pkg/util/version" +) + +const ( + kubeDNSv170AndAboveVersion = "1.14.4" +) + +// GetKubeDNSVersion returns the right kube-dns version for a specific k8s version +func GetKubeDNSVersion(kubeVersion *version.Version) string { + // v1.7.0+ uses 1.14.4, just return that here + // In the future when the kube-dns version is bumped at HEAD; add conditional logic to return the right versions + // Also, the version might be bumped for different k8s releases on the same branch + return kubeDNSv170AndAboveVersion +} + +// GetKubeDNSManifest returns the right kube-dns YAML manifest for a specific k8s version +func GetKubeDNSManifest(kubeVersion *version.Version) string { + // v1.7.0+ has only one known YAML manifest spec, just return that here + // In the future when the kube-dns version is bumped at HEAD; add conditional logic to return the right manifest + return v170AndAboveKubeDNSDeployment +} diff --git a/cmd/kubeadm/app/phases/addons/dns/versions_test.go b/cmd/kubeadm/app/phases/addons/dns/versions_test.go new file mode 100644 index 0000000000000..1b87fb45108e2 --- /dev/null +++ b/cmd/kubeadm/app/phases/addons/dns/versions_test.go @@ -0,0 +1,70 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dns + +import ( + "testing" + + "k8s.io/kubernetes/pkg/util/version" +) + +func TestGetKubeDNSVersion(t *testing.T) { + var tests = []struct { + k8sVersion, expected string + }{ + { + k8sVersion: "v1.7.0", + expected: "1.14.4", + }, + { + k8sVersion: "v1.7.1", + expected: "1.14.4", + }, + { + k8sVersion: "v1.7.2", + expected: "1.14.4", + }, + { + k8sVersion: "v1.7.3", + expected: "1.14.4", + }, + { + k8sVersion: "v1.8.0-alpha.2", + expected: "1.14.4", + }, + { + k8sVersion: "v1.8.0", + expected: "1.14.4", + }, + } + for _, rt := range tests { + + k8sVersion, err := version.ParseSemantic(rt.k8sVersion) + if err != nil { + t.Fatalf("couldn't parse kubernetes version %q: %v", rt.k8sVersion, err) + } + + actualDNSVersion := GetKubeDNSVersion(k8sVersion) + if actualDNSVersion != rt.expected { + t.Errorf( + "failed GetKubeDNSVersion:\n\texpected: %s\n\t actual: %s", + rt.expected, + actualDNSVersion, + ) + } + } +} diff --git a/cmd/kubeadm/app/phases/addons/proxy/manifests.go b/cmd/kubeadm/app/phases/addons/proxy/manifests.go index dfca6bd5e96ef..b9903192d14d4 100644 --- a/cmd/kubeadm/app/phases/addons/proxy/manifests.go +++ b/cmd/kubeadm/app/phases/addons/proxy/manifests.go @@ -81,7 +81,6 @@ spec: volumeMounts: - mountPath: /var/lib/kube-proxy name: kube-proxy - # TODO: Make this a file hostpath mount - mountPath: /run/xtables.lock name: xtables-lock readOnly: false @@ -100,5 +99,6 @@ spec: - name: xtables-lock hostPath: path: /run/xtables.lock + type: FileOrCreate ` ) diff --git a/cmd/kubeadm/app/phases/addons/proxy/proxy.go b/cmd/kubeadm/app/phases/addons/proxy/proxy.go index 51d53a37d4db9..98c5dfe60d69a 100644 --- a/cmd/kubeadm/app/phases/addons/proxy/proxy.go +++ b/cmd/kubeadm/app/phases/addons/proxy/proxy.go @@ -49,16 +49,21 @@ func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte return fmt.Errorf("error when creating kube-proxy service account: %v", err) } + // Generate Master Enpoint kubeconfig file + masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg) + if err != nil { + return err + } + proxyConfigMapBytes, err := kubeadmutil.ParseTemplate(KubeProxyConfigMap, struct{ MasterEndpoint string }{ // Fetch this value from the kubeconfig file - MasterEndpoint: fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddress, cfg.API.BindPort), - }) + MasterEndpoint: masterEndpoint}) if err != nil { return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err) } proxyDaemonSetBytes, err := kubeadmutil.ParseTemplate(KubeProxyDaemonSet, struct{ ImageRepository, Arch, Version, ImageOverride, ClusterCIDR, MasterTaintKey, CloudTaintKey string }{ - ImageRepository: cfg.ImageRepository, + ImageRepository: cfg.GetControlPlaneImageRepository(), Arch: runtime.GOARCH, Version: kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion), ImageOverride: cfg.UnifiedControlPlaneImage, diff --git a/cmd/kubeadm/app/phases/bootstraptoken/node/token.go b/cmd/kubeadm/app/phases/bootstraptoken/node/token.go index e4c29912e4af7..4b59d81fcc057 100644 --- a/cmd/kubeadm/app/phases/bootstraptoken/node/token.go +++ b/cmd/kubeadm/app/phases/bootstraptoken/node/token.go @@ -18,6 +18,7 @@ package node import ( "fmt" + "strings" "time" "k8s.io/api/core/v1" @@ -33,12 +34,12 @@ const tokenCreateRetries = 5 // TODO(mattmoyer): Move CreateNewToken, UpdateOrCreateToken and encodeTokenSecretData out of this package to client-go for a generic abstraction and client for a Bootstrap Token // CreateNewToken tries to create a token and fails if one with the same ID already exists -func CreateNewToken(client clientset.Interface, token string, tokenDuration time.Duration, usages []string, description string) error { - return UpdateOrCreateToken(client, token, true, tokenDuration, usages, description) +func CreateNewToken(client clientset.Interface, token string, tokenDuration time.Duration, usages []string, extraGroups []string, description string) error { + return UpdateOrCreateToken(client, token, true, tokenDuration, usages, extraGroups, description) } // UpdateOrCreateToken attempts to update a token with the given ID, or create if it does not already exist. -func UpdateOrCreateToken(client clientset.Interface, token string, failIfExists bool, tokenDuration time.Duration, usages []string, description string) error { +func UpdateOrCreateToken(client clientset.Interface, token string, failIfExists bool, tokenDuration time.Duration, usages []string, extraGroups []string, description string) error { tokenID, tokenSecret, err := tokenutil.ParseToken(token) if err != nil { return err @@ -52,7 +53,7 @@ func UpdateOrCreateToken(client clientset.Interface, token string, failIfExists return fmt.Errorf("a token with id %q already exists", tokenID) } // Secret with this ID already exists, update it: - secret.Data = encodeTokenSecretData(tokenID, tokenSecret, tokenDuration, usages, description) + secret.Data = encodeTokenSecretData(tokenID, tokenSecret, tokenDuration, usages, extraGroups, description) if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Update(secret); err == nil { return nil } @@ -67,7 +68,7 @@ func UpdateOrCreateToken(client clientset.Interface, token string, failIfExists Name: secretName, }, Type: v1.SecretType(bootstrapapi.SecretTypeBootstrapToken), - Data: encodeTokenSecretData(tokenID, tokenSecret, tokenDuration, usages, description), + Data: encodeTokenSecretData(tokenID, tokenSecret, tokenDuration, usages, extraGroups, description), } if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret); err == nil { return nil @@ -85,12 +86,16 @@ func UpdateOrCreateToken(client clientset.Interface, token string, failIfExists } // encodeTokenSecretData takes the token discovery object and an optional duration and returns the .Data for the Secret -func encodeTokenSecretData(tokenID, tokenSecret string, duration time.Duration, usages []string, description string) map[string][]byte { +func encodeTokenSecretData(tokenID, tokenSecret string, duration time.Duration, usages []string, extraGroups []string, description string) map[string][]byte { data := map[string][]byte{ bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), } + if len(extraGroups) > 0 { + data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(strings.Join(extraGroups, ",")) + } + if duration > 0 { // Get the current time, add the specified duration, and format it accordingly durationString := time.Now().Add(duration).Format(time.RFC3339) diff --git a/cmd/kubeadm/app/phases/bootstraptoken/node/token_test.go b/cmd/kubeadm/app/phases/bootstraptoken/node/token_test.go index 48a6f80e9827a..7af575e01d161 100644 --- a/cmd/kubeadm/app/phases/bootstraptoken/node/token_test.go +++ b/cmd/kubeadm/app/phases/bootstraptoken/node/token_test.go @@ -33,7 +33,7 @@ func TestEncodeTokenSecretData(t *testing.T) { {token: &kubeadmapi.TokenDiscovery{ID: "foo", Secret: "bar"}, t: time.Second}, // should use default } for _, rt := range tests { - actual := encodeTokenSecretData(rt.token.ID, rt.token.Secret, rt.t, []string{}, "") + actual := encodeTokenSecretData(rt.token.ID, rt.token.Secret, rt.t, []string{}, []string{}, "") if !bytes.Equal(actual["token-id"], []byte(rt.token.ID)) { t.Errorf( "failed EncodeTokenSecretData:\n\texpected: %s\n\t actual: %s", diff --git a/cmd/kubeadm/app/phases/certs/BUILD b/cmd/kubeadm/app/phases/certs/BUILD index f144a02481a6a..16e44355db911 100644 --- a/cmd/kubeadm/app/phases/certs/BUILD +++ b/cmd/kubeadm/app/phases/certs/BUILD @@ -13,6 +13,9 @@ go_test( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", + "//cmd/kubeadm/test:go_default_library", + "//cmd/kubeadm/test/certs:go_default_library", ], ) @@ -27,7 +30,7 @@ go_library( "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", "//pkg/registry/core/service/ipallocator:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", "//vendor/k8s.io/client-go/util/cert:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go index 0fc6b5b27fa5c..a6b060a0da2f3 100644 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ b/cmd/kubeadm/app/phases/certs/certs.go @@ -22,14 +22,164 @@ import ( "fmt" "net" - "k8s.io/apimachinery/pkg/util/validation" certutil "k8s.io/client-go/util/cert" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" + "k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/validation" ) +// CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane. +// If the PKI assets already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned. +func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error { + + certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{ + CreateCACertAndKeyfiles, + CreateAPIServerCertAndKeyFiles, + CreateAPIServerKubeletClientCertAndKeyFiles, + CreateServiceAccountKeyAndPublicKeyFiles, + CreateFrontProxyCACertAndKeyFiles, + CreateFrontProxyClientCertAndKeyFiles, + } + + for _, action := range certActions { + err := action(cfg) + if err != nil { + return err + } + } + + fmt.Printf("[certificates] Valid certificates and keys now exist in %q\n", cfg.CertificatesDir) + + return nil +} + +// CreateCACertAndKeyfiles create a new self signed CA certificate and key files. +// If the CA certificate and key files already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned. +func CreateCACertAndKeyfiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := NewCACertAndKey() + if err != nil { + return err + } + + return writeCertificateAuthorithyFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.CACertAndKeyBaseName, + caCert, + caKey, + ) +} + +// CreateAPIServerCertAndKeyFiles create a new certificate and key files for the apiserver. +// If the apiserver certificate and key files already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned. +// It assumes the cluster CA certificate and key files should exists into the CertificatesDir +func CreateAPIServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + apiCert, apiKey, err := NewAPIServerCertAndKey(cfg, caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.APIServerCertAndKeyBaseName, + caCert, + apiCert, + apiKey, + ) +} + +// CreateAPIServerKubeletClientCertAndKeyFiles create a new CA certificate for kubelets calling apiserver +// If the apiserver-kubelet-client certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned. +// It assumes the cluster CA certificate and key files should exists into the CertificatesDir +func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + caCert, caKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) + if err != nil { + return err + } + + apiClientCert, apiClientKey, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, + caCert, + apiClientCert, + apiClientKey, + ) +} + +// CreateServiceAccountKeyAndPublicKeyFiles create a new public/private key files for signing service account users. +// If the sa public/private key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned. +func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + saSigningKey, err := NewServiceAccountSigningKey() + if err != nil { + return err + } + + return writeKeyFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.ServiceAccountKeyBaseName, + saSigningKey, + ) +} + +// CreateFrontProxyCACertAndKeyFiles create a self signed front proxy CA certificate and key files. +// Front proxy CA and client certs are used to secure a front proxy authenticator which is used to assert identity +// without the client cert; This is a separte CA, so that front proxy identities cannot hit the API and normal client certs cannot be used +// as front proxies. +// If the front proxy CA certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned. +func CreateFrontProxyCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + frontProxyCACert, frontProxyCAKey, err := NewFrontProxyCACertAndKey() + if err != nil { + return err + } + + return writeCertificateAuthorithyFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.FrontProxyCACertAndKeyBaseName, + frontProxyCACert, + frontProxyCAKey, + ) +} + +// CreateFrontProxyClientCertAndKeyFiles create a new certificate for proxy server client. +// If the front-proxy-client certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned. +// It assumes the front proxy CAA certificate and key files should exists into the CertificatesDir +func CreateFrontProxyClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error { + + frontProxyCACert, frontProxyCAKey, err := loadCertificateAuthorithy(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName) + if err != nil { + return err + } + + frontProxyClientCert, frontProxyClientKey, err := NewFrontProxyClientCertAndKey(frontProxyCACert, frontProxyCAKey) + if err != nil { + return err + } + + return writeCertificateFilesIfNotExist( + cfg.CertificatesDir, + kubeadmconstants.FrontProxyClientCertAndKeyBaseName, + frontProxyCACert, + frontProxyClientCert, + frontProxyClientKey, + ) +} + // NewCACertAndKey will generate a self signed CA. func NewCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) { @@ -91,10 +241,6 @@ func NewServiceAccountSigningKey() (*rsa.PrivateKey, error) { } // NewFrontProxyCACertAndKey generate a self signed front proxy CA. -// Front proxy CA and client certs are used to secure a front proxy authenticator which is used to assert identity -// without the client cert. -// This is a separte CA, so that front proxy identities cannot hit the API and normal client certs cannot be used -// as front proxies. func NewFrontProxyCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) { frontProxyCACert, frontProxyCAKey, err := pkiutil.NewCertificateAuthority() @@ -120,6 +266,138 @@ func NewFrontProxyClientCertAndKey(frontProxyCACert *x509.Certificate, frontProx return frontProxyClientCert, frontProxyClientKey, nil } +// loadCertificateAuthorithy loads certificate authorithy +func loadCertificateAuthorithy(pkiDir string, baseName string) (*x509.Certificate, *rsa.PrivateKey, error) { + // Checks if certificate authorithy exists in the PKI directory + if !pkiutil.CertOrKeyExist(pkiDir, baseName) { + return nil, nil, fmt.Errorf("couldn't load %s certificate authorithy from %s", baseName, pkiDir) + } + + // Try to load certificate authorithy .crt and .key from the PKI directory + caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName) + if err != nil { + return nil, nil, fmt.Errorf("failure loading %s certificate authorithy: %v", baseName, err) + } + + // Make sure the loaded CA cert actually is a CA + if !caCert.IsCA { + return nil, nil, fmt.Errorf("%s certificate is not a certificate authorithy", baseName) + } + + return caCert, caKey, nil +} + +// writeCertificateAuthorithyFilesIfNotExist write a new certificate Authorithy to the given path. +// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the +// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date, +// otherwise this function returns an error. +func writeCertificateAuthorithyFilesIfNotExist(pkiDir string, baseName string, caCert *x509.Certificate, caKey *rsa.PrivateKey) error { + + // If cert or key exists, we should try to load them + if pkiutil.CertOrKeyExist(pkiDir, baseName) { + + // Try to load .crt and .key from the PKI directory + caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName) + if err != nil { + return fmt.Errorf("failure loading %s certificate: %v", baseName, err) + } + + // Check if the existing cert is a CA + if !caCert.IsCA { + return fmt.Errorf("certificate %s is not a CA", baseName) + } + + // kubeadm doesn't validate the existing certificate Authorithy more than this; + // Basically, if we find a certificate file with the same path; and it is a CA + // kubeadm thinks those files are equal and doesn't bother writing a new file + fmt.Printf("[certificates] Using the existing %s certificate and key.\n", baseName) + } else { + + // Write .crt and .key files to disk + if err := pkiutil.WriteCertAndKey(pkiDir, baseName, caCert, caKey); err != nil { + return fmt.Errorf("failure while saving %s certificate and key: %v", baseName, err) + } + + fmt.Printf("[certificates] Generated %s certificate and key.\n", baseName) + } + return nil +} + +// writeCertificateFilesIfNotExist write a new certificate to the given path. +// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the +// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date, +// otherwise this function returns an error. +func writeCertificateFilesIfNotExist(pkiDir string, baseName string, signingCert *x509.Certificate, cert *x509.Certificate, key *rsa.PrivateKey) error { + + // Checks if the signed certificate exists in the PKI directory + if pkiutil.CertOrKeyExist(pkiDir, baseName) { + // Try to load signed certificate .crt and .key from the PKI directory + signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName) + if err != nil { + return fmt.Errorf("failure loading %s certificate: %v", baseName, err) + } + + // Check if the existing cert is signed by the given CA + if err := signedCert.CheckSignatureFrom(signingCert); err != nil { + return fmt.Errorf("certificate %s is not signed by corresponding CA", baseName) + } + + // kubeadm doesn't validate the existing certificate more than this; + // Basically, if we find a certificate file with the same path; and it is signed by + // the expected certificate authorithy, kubeadm thinks those files are equal and + // doesn't bother writing a new file + fmt.Printf("[certificates] Using the existing %s certificate and key.\n", baseName) + } else { + + // Write .crt and .key files to disk + if err := pkiutil.WriteCertAndKey(pkiDir, baseName, cert, key); err != nil { + return fmt.Errorf("failure while saving %s certificate and key: %v", baseName, err) + } + + fmt.Printf("[certificates] Generated %s certificate and key.\n", baseName) + if pkiutil.HasServerAuth(cert) { + fmt.Printf("[certificates] %s serving cert is signed for DNS names %v and IPs %v\n", baseName, cert.DNSNames, cert.IPAddresses) + } + } + + return nil +} + +// writeKeyFilesIfNotExist write a new key to the given path. +// If there already is a key file at the given path; kubeadm tries to load it and check if the values in the +// existing and the expected key equals. If they do; kubeadm will just skip writing the file as it's up-to-date, +// otherwise this function returns an error. +func writeKeyFilesIfNotExist(pkiDir string, baseName string, key *rsa.PrivateKey) error { + + // Checks if the key exists in the PKI directory + if pkiutil.CertOrKeyExist(pkiDir, baseName) { + + // Try to load .key from the PKI directory + _, err := pkiutil.TryLoadKeyFromDisk(pkiDir, baseName) + if err != nil { + return fmt.Errorf("%s key existed but it could not be loaded properly: %v", baseName, err) + } + + // kubeadm doesn't validate the existing certificate key more than this; + // Basically, if we find a key file with the same path kubeadm thinks those files + // are equal and doesn't bother writing a new file + fmt.Printf("[certificates] Using the existing %s key.\n", baseName) + } else { + + // Write .key and .pub files to disk + if err := pkiutil.WriteKey(pkiDir, baseName, key); err != nil { + return fmt.Errorf("failure while saving %s key: %v", baseName, err) + } + + if err := pkiutil.WritePublicKey(pkiDir, baseName, &key.PublicKey); err != nil { + return fmt.Errorf("failure while saving %s public key: %v", baseName, err) + } + fmt.Printf("[certificates] Generated %s key and public key.\n", baseName) + } + + return nil +} + // getAltNames builds an AltNames object for to be used when generating apiserver certificate func getAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) { diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go index 7a058373fe6d8..5840e6485243e 100644 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ b/cmd/kubeadm/app/phases/certs/certs_test.go @@ -17,21 +17,298 @@ limitations under the License. package certs import ( + "crypto/rsa" "crypto/x509" "net" + "os" "testing" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" + + testutil "k8s.io/kubernetes/cmd/kubeadm/test" + certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs" ) +func TestWriteCertificateAuthorithyFilesIfNotExist(t *testing.T) { + + setupCert, setupKey, _ := NewCACertAndKey() + caCert, caKey, _ := NewCACertAndKey() + + var tests = []struct { + setupFunc func(pkiDir string) error + expectedError bool + expectedCa *x509.Certificate + }{ + { // ca cert does not exists > ca written + expectedCa: caCert, + }, + { // ca cert exists, is ca > existing ca used + setupFunc: func(pkiDir string) error { + return writeCertificateAuthorithyFilesIfNotExist(pkiDir, "dummy", setupCert, setupKey) + }, + expectedCa: setupCert, + }, + { // some file exists, but it is not a valid ca cert > err + setupFunc: func(pkiDir string) error { + testutil.SetupEmptyFiles(t, pkiDir, "dummy.crt") + return nil + }, + expectedError: true, + }, + { // cert exists, but it is not a ca > err + setupFunc: func(pkiDir string) error { + cert, key, _ := NewFrontProxyClientCertAndKey(setupCert, setupKey) + return writeCertificateFilesIfNotExist(pkiDir, "dummy", setupCert, cert, key) + }, + expectedError: true, + }, + } + + for _, test := range tests { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // executes setup func (if necessary) + if test.setupFunc != nil { + if err := test.setupFunc(tmpdir); err != nil { + t.Errorf("error executing setupFunc: %v", err) + continue + } + } + + // executes create func + err := writeCertificateAuthorithyFilesIfNotExist(tmpdir, "dummy", caCert, caKey) + + if !test.expectedError && err != nil { + t.Errorf("error writeCertificateAuthorithyFilesIfNotExist failed when not expected to fail: %v", err) + continue + } else if test.expectedError && err == nil { + t.Error("error writeCertificateAuthorithyFilesIfNotExist didn't failed when expected") + continue + } else if test.expectedError { + continue + } + + // asserts expected files are there + testutil.AssertFileExists(t, tmpdir, "dummy.key", "dummy.crt") + + // check created cert + resultingCaCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(tmpdir, "dummy") + if err != nil { + t.Errorf("failure reading created cert: %v", err) + continue + } + if !resultingCaCert.Equal(test.expectedCa) { + t.Error("created ca cert does not match expected ca cert") + } + } +} + +func TestWriteCertificateFilesIfNotExist(t *testing.T) { + + caCert, caKey, _ := NewFrontProxyCACertAndKey() + setupCert, setupKey, _ := NewFrontProxyClientCertAndKey(caCert, caKey) + cert, key, _ := NewFrontProxyClientCertAndKey(caCert, caKey) + + var tests = []struct { + setupFunc func(pkiDir string) error + expectedError bool + expectedCert *x509.Certificate + }{ + { // cert does not exists > cert written + expectedCert: cert, + }, + { // cert exists, is signed by the same ca > existing cert used + setupFunc: func(pkiDir string) error { + return writeCertificateFilesIfNotExist(pkiDir, "dummy", caCert, setupCert, setupKey) + }, + expectedCert: setupCert, + }, + { // some file exists, but it is not a valid cert > err + setupFunc: func(pkiDir string) error { + testutil.SetupEmptyFiles(t, pkiDir, "dummy.crt") + return nil + }, + expectedError: true, + }, + { // cert exists, is signed by another ca > err + setupFunc: func(pkiDir string) error { + anotherCaCert, anotherCaKey, _ := NewFrontProxyCACertAndKey() + anotherCert, anotherKey, _ := NewFrontProxyClientCertAndKey(anotherCaCert, anotherCaKey) + + return writeCertificateFilesIfNotExist(pkiDir, "dummy", anotherCaCert, anotherCert, anotherKey) + }, + expectedError: true, + }, + } + + for _, test := range tests { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // executes setup func (if necessary) + if test.setupFunc != nil { + if err := test.setupFunc(tmpdir); err != nil { + t.Errorf("error executing setupFunc: %v", err) + continue + } + } + + // executes create func + err := writeCertificateFilesIfNotExist(tmpdir, "dummy", caCert, cert, key) + + if !test.expectedError && err != nil { + t.Errorf("error writeCertificateFilesIfNotExist failed when not expected to fail: %v", err) + continue + } else if test.expectedError && err == nil { + t.Error("error writeCertificateFilesIfNotExist didn't failed when expected") + continue + } else if test.expectedError { + continue + } + + // asserts expected files are there + testutil.AssertFileExists(t, tmpdir, "dummy.key", "dummy.crt") + + // check created cert + resultingCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(tmpdir, "dummy") + if err != nil { + t.Errorf("failure reading created cert: %v", err) + continue + } + if !resultingCert.Equal(test.expectedCert) { + t.Error("created cert does not match expected cert") + } + } +} + +func TestWriteKeyFilesIfNotExist(t *testing.T) { + + setupKey, _ := NewServiceAccountSigningKey() + key, _ := NewServiceAccountSigningKey() + + var tests = []struct { + setupFunc func(pkiDir string) error + expectedError bool + expectedKey *rsa.PrivateKey + }{ + { // key does not exists > key written + expectedKey: key, + }, + { // key exists > existing key used + setupFunc: func(pkiDir string) error { + return writeKeyFilesIfNotExist(pkiDir, "dummy", setupKey) + }, + expectedKey: setupKey, + }, + { // some file exists, but it is not a valid key > err + setupFunc: func(pkiDir string) error { + testutil.SetupEmptyFiles(t, pkiDir, "dummy.key") + return nil + }, + expectedError: true, + }, + } + + for _, test := range tests { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // executes setup func (if necessary) + if test.setupFunc != nil { + if err := test.setupFunc(tmpdir); err != nil { + t.Errorf("error executing setupFunc: %v", err) + continue + } + } + + // executes create func + err := writeKeyFilesIfNotExist(tmpdir, "dummy", key) + + if !test.expectedError && err != nil { + t.Errorf("error writeKeyFilesIfNotExist failed when not expected to fail: %v", err) + continue + } else if test.expectedError && err == nil { + t.Error("error writeKeyFilesIfNotExist didn't failed when expected") + continue + } else if test.expectedError { + continue + } + + // asserts expected files are there + testutil.AssertFileExists(t, tmpdir, "dummy.key", "dummy.pub") + + // check created key + resultingKey, err := pkiutil.TryLoadKeyFromDisk(tmpdir, "dummy") + if err != nil { + t.Errorf("failure reading created key: %v", err) + continue + } + + //TODO: check if there is a better method to compare keys + if resultingKey.D == key.D { + t.Error("created key does not match expected key") + } + } +} + +func TestGetAltNames(t *testing.T) { + hostname := "valid-hostname" + advertiseIP := "1.2.3.4" + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: advertiseIP}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: hostname, + } + + altNames, err := getAltNames(cfg) + if err != nil { + t.Fatalf("failed calling getAltNames: %v", err) + } + + expectedDNSNames := []string{hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local"} + for _, DNSName := range expectedDNSNames { + found := false + for _, val := range altNames.DNSNames { + if val == DNSName { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain DNSName %s", DNSName) + } + } + + expectedIPAddresses := []string{"10.96.0.1", advertiseIP} + for _, IPAddress := range expectedIPAddresses { + found := false + for _, val := range altNames.IPs { + if val.Equal(net.ParseIP(IPAddress)) { + found = true + break + } + } + + if !found { + t.Errorf("altNames does not contain IPAddress %s", IPAddress) + } + } +} + func TestNewCACertAndKey(t *testing.T) { caCert, _, err := NewCACertAndKey() if err != nil { t.Fatalf("failed call NewCACertAndKey: %v", err) } - assertIsCa(t, caCert) + certstestutil.AssertCertificateIsCa(t, caCert) } func TestNewAPIServerCertAndKey(t *testing.T) { @@ -51,15 +328,10 @@ func TestNewAPIServerCertAndKey(t *testing.T) { t.Fatalf("failed creation of cert and key: %v", err) } - assertIsSignedByCa(t, apiServerCert, caCert) - assertHasServerAuth(t, apiServerCert) - - for _, DNSName := range []string{hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local"} { - assertHasDNSNames(t, apiServerCert, DNSName) - } - for _, IPAddress := range []string{"10.96.0.1", addr} { - assertHasIPAddresses(t, apiServerCert, net.ParseIP(IPAddress)) - } + certstestutil.AssertCertificateIsSignedByCa(t, apiServerCert, caCert) + certstestutil.AssertCertificateHasServerAuthUsage(t, apiServerCert) + certstestutil.AssertCertificateHasDNSNames(t, apiServerCert, hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local") + certstestutil.AssertCertificateHasIPAddresses(t, apiServerCert, net.ParseIP("10.96.0.1"), net.ParseIP(addr)) } } @@ -71,9 +343,9 @@ func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) { t.Fatalf("failed creation of cert and key: %v", err) } - assertIsSignedByCa(t, apiClientCert, caCert) - assertHasClientAuth(t, apiClientCert) - assertHasOrganization(t, apiClientCert, constants.MastersGroup) + certstestutil.AssertCertificateIsSignedByCa(t, apiClientCert, caCert) + certstestutil.AssertCertificateHasClientAuthUsage(t, apiClientCert) + certstestutil.AssertCertificateHasOrganizations(t, apiClientCert, kubeadmconstants.MastersGroup) } func TestNewNewServiceAccountSigningKey(t *testing.T) { @@ -94,7 +366,7 @@ func TestNewFrontProxyCACertAndKey(t *testing.T) { t.Fatalf("failed creation of cert and key: %v", err) } - assertIsCa(t, frontProxyCACert) + certstestutil.AssertCertificateIsCa(t, frontProxyCACert) } func TestNewFrontProxyClientCertAndKey(t *testing.T) { @@ -105,63 +377,84 @@ func TestNewFrontProxyClientCertAndKey(t *testing.T) { t.Fatalf("failed creation of cert and key: %v", err) } - assertIsSignedByCa(t, frontProxyClientCert, frontProxyCACert) - assertHasClientAuth(t, frontProxyClientCert) + certstestutil.AssertCertificateIsSignedByCa(t, frontProxyClientCert, frontProxyCACert) + certstestutil.AssertCertificateHasClientAuthUsage(t, frontProxyClientCert) } -func assertIsCa(t *testing.T, cert *x509.Certificate) { - if !cert.IsCA { - t.Error("cert is not a valida CA") - } -} +func TestCreateCertificateFilesMethods(t *testing.T) { -func assertIsSignedByCa(t *testing.T, cert *x509.Certificate, ca *x509.Certificate) { - if err := cert.CheckSignatureFrom(ca); err != nil { - t.Error("cert is not signed by ca") + var tests = []struct { + setupFunc func(cfg *kubeadmapi.MasterConfiguration) error + createFunc func(cfg *kubeadmapi.MasterConfiguration) error + expectedFiles []string + }{ + { + createFunc: CreatePKIAssets, + expectedFiles: []string{ + kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, + kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName, + kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName, + kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName, + kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName, + kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName, + }, + }, + { + createFunc: CreateCACertAndKeyfiles, + expectedFiles: []string{kubeadmconstants.CACertName, kubeadmconstants.CAKeyName}, + }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateAPIServerCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName}, + }, + { + setupFunc: CreateCACertAndKeyfiles, + createFunc: CreateAPIServerKubeletClientCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName}, + }, + { + createFunc: CreateServiceAccountKeyAndPublicKeyFiles, + expectedFiles: []string{kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName}, + }, + { + createFunc: CreateFrontProxyCACertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName}, + }, + { + setupFunc: CreateFrontProxyCACertAndKeyFiles, + createFunc: CreateFrontProxyClientCertAndKeyFiles, + expectedFiles: []string{kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName}, + }, } -} -func assertHasClientAuth(t *testing.T, cert *x509.Certificate) { - for i := range cert.ExtKeyUsage { - if cert.ExtKeyUsage[i] == x509.ExtKeyUsageClientAuth { - return - } - } - t.Error("cert is not a ClientAuth") -} + for _, test := range tests { + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) -func assertHasServerAuth(t *testing.T, cert *x509.Certificate) { - for i := range cert.ExtKeyUsage { - if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth { - return + cfg := &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, + NodeName: "valid-hostname", + CertificatesDir: tmpdir, } - } - t.Error("cert is not a ServerAuth") -} -func assertHasOrganization(t *testing.T, cert *x509.Certificate, OU string) { - for i := range cert.Subject.Organization { - if cert.Subject.Organization[i] == OU { - return + // executes setup func (if necessary) + if test.setupFunc != nil { + if err := test.setupFunc(cfg); err != nil { + t.Errorf("error executing setupFunc: %v", err) + continue + } } - } - t.Errorf("cert does not contain OU %s", OU) -} -func assertHasDNSNames(t *testing.T, cert *x509.Certificate, DNSName string) { - for i := range cert.DNSNames { - if cert.DNSNames[i] == DNSName { - return + // executes create func + if err := test.createFunc(cfg); err != nil { + t.Errorf("error executing createFunc: %v", err) + continue } - } - t.Errorf("cert does not contain DNSName %s", DNSName) -} -func assertHasIPAddresses(t *testing.T, cert *x509.Certificate, IPAddress net.IP) { - for i := range cert.IPAddresses { - if cert.IPAddresses[i].Equal(IPAddress) { - return - } + // asserts expected files are there + testutil.AssertFileExists(t, tmpdir, test.expectedFiles...) } - t.Errorf("cert does not contain IPAddress %s", IPAddress) } diff --git a/cmd/kubeadm/app/phases/controlplane/BUILD b/cmd/kubeadm/app/phases/controlplane/BUILD index 71b71eebcc89d..c88f404feff6a 100644 --- a/cmd/kubeadm/app/phases/controlplane/BUILD +++ b/cmd/kubeadm/app/phases/controlplane/BUILD @@ -33,6 +33,7 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/images:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/staticpod:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", "//pkg/util/version:go_default_library", diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index ed073cedd5f82..19a4a5de55aaa 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -28,6 +28,7 @@ import ( kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" "k8s.io/kubernetes/pkg/util/version" @@ -71,7 +72,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version. staticPodSpecs := map[string]v1.Pod{ kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeAPIServer, - Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.ImageRepository, cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), + Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), Command: getAPIServerCommand(cfg, k8sVersion), VolumeMounts: mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer), LivenessProbe: staticpodutil.ComponentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS), @@ -80,7 +81,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version. }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)), kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeControllerManager, - Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.ImageRepository, cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), + Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), Command: getControllerManagerCommand(cfg, k8sVersion), VolumeMounts: mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager), LivenessProbe: staticpodutil.ComponentProbe(10252, "/healthz", v1.URISchemeHTTP), @@ -89,7 +90,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version. }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeScheduler, - Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.ImageRepository, cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), + Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), Command: getSchedulerCommand(cfg), VolumeMounts: mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler), LivenessProbe: staticpodutil.ComponentProbe(10251, "/healthz", v1.URISchemeHTTP), @@ -125,6 +126,8 @@ func createStaticPodFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguratio if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil { return fmt.Errorf("failed to create static pod manifest file for %q: %v", componentName, err) } + + fmt.Printf("[controlplane] Wrote Static Pod manifest for component %s to %q\n", componentName, kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir)) } return nil @@ -133,20 +136,19 @@ func createStaticPodFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguratio // getAPIServerCommand builds the right API server command from the given config object and version func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string { defaultArguments := map[string]string{ - "advertise-address": cfg.API.AdvertiseAddress, - "insecure-port": "0", - "admission-control": defaultv17AdmissionControl, - "service-cluster-ip-range": cfg.Networking.ServiceSubnet, - "service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName), - "client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), - "tls-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerCertName), - "tls-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName), - "kubelet-client-certificate": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName), - "kubelet-client-key": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName), - "secure-port": fmt.Sprintf("%d", cfg.API.BindPort), - "allow-privileged": "true", - "experimental-bootstrap-token-auth": "true", - "kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", + "advertise-address": cfg.API.AdvertiseAddress, + "insecure-port": "0", + "admission-control": defaultv17AdmissionControl, + "service-cluster-ip-range": cfg.Networking.ServiceSubnet, + "service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName), + "client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), + "tls-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerCertName), + "tls-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName), + "kubelet-client-certificate": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName), + "kubelet-client-key": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName), + "secure-port": fmt.Sprintf("%d", cfg.API.BindPort), + "allow-privileged": "true", + "kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", // add options to configure the front proxy. Without the generated client cert, this will never be useable // so add it unconditionally with recommended values "requestheader-username-headers": "X-Remote-User", @@ -159,7 +161,15 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio } command := []string{"kube-apiserver"} - command = append(command, staticpodutil.GetExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...) + + // Note: Mutating defaultArguments dynamically must happen before the BuildArgumentListFromMap call below + if k8sVersion.AtLeast(kubeadmconstants.UseEnableBootstrapTokenAuthFlagVersion) { + defaultArguments["enable-bootstrap-token-auth"] = "true" + } else { + defaultArguments["experimental-bootstrap-token-auth"] = "true" + } + + command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.APIServerExtraArgs)...) command = append(command, getAuthzParameters(cfg.AuthorizationModes)...) // Check if the user decided to use an external etcd cluster @@ -206,7 +216,7 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion } command := []string{"kube-controller-manager"} - command = append(command, staticpodutil.GetExtraParameters(cfg.ControllerManagerExtraArgs, defaultArguments)...) + command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.ControllerManagerExtraArgs)...) if cfg.CloudProvider != "" { command = append(command, "--cloud-provider="+cfg.CloudProvider) @@ -234,7 +244,7 @@ func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration) []string { } command := []string{"kube-scheduler"} - command = append(command, staticpodutil.GetExtraParameters(cfg.SchedulerExtraArgs, defaultArguments)...) + command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.SchedulerExtraArgs)...) return command } diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index 23dcc03bf808e..488e5a4df1279 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -171,6 +171,40 @@ func TestGetAPIServerCommand(t *testing.T) { "--etcd-servers=http://127.0.0.1:2379", }, }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, + Networking: kubeadmapi.Networking{ServiceSubnet: "bar"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.8.0-beta.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + "--secure-port=123", + "--allow-privileged=true", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--enable-bootstrap-token-auth=true", + "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", + "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=Node,RBAC", + "--advertise-address=1.2.3.4", + "--etcd-servers=http://127.0.0.1:2379", + }, + }, { cfg: &kubeadmapi.MasterConfiguration{ API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, diff --git a/cmd/kubeadm/app/phases/controlplane/volumes.go b/cmd/kubeadm/app/phases/controlplane/volumes.go index db60823788be1..a9c69d01b913e 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes.go @@ -30,11 +30,11 @@ import ( ) const ( - k8sCertsVolumeName = "k8s-certs" - caCertsVolumeName = "ca-certs" - caCertsVolumePath = "/etc/ssl/certs" - caCertsPkiVolumeName = "ca-certs-etc-pki" - kubeConfigVolumeName = "kubeconfig" + caCertsVolumeName = "ca-certs" + caCertsVolumePath = "/etc/ssl/certs" + caCertsPkiVolumeName = "ca-certs-etc-pki" + flexvolumeDirVolumeName = "flexvolume-dir" + flexvolumeDirVolumePath = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" ) // caCertsPkiVolumePath specifies the path that can be conditionally mounted into the apiserver and controller-manager containers @@ -44,14 +44,16 @@ var caCertsPkiVolumePath = "/etc/pki" // getHostPathVolumesForTheControlPlane gets the required hostPath volumes and mounts for the control plane func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.MasterConfiguration) controlPlaneHostPathMounts { + hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate + hostPathFileOrCreate := v1.HostPathFileOrCreate mounts := newControlPlaneHostPathMounts() // HostPath volumes for the API Server // Read-only mount for the certificates directory // TODO: Always mount the K8s Certificates directory to a static path inside of the container - mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, k8sCertsVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true) + mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate) // Read-only mount for the ca certs (/etc/ssl/certs) directory - mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true) + mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate) // If external etcd is specified, mount the directories needed for accessing the CA/serving certs and the private key if len(cfg.Etcd.Endpoints) != 0 { @@ -62,23 +64,26 @@ func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.MasterConfiguration) c // HostPath volumes for the controller manager // Read-only mount for the certificates directory // TODO: Always mount the K8s Certificates directory to a static path inside of the container - mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, k8sCertsVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true) + mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate) // Read-only mount for the ca certs (/etc/ssl/certs) directory - mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true) + mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate) // Read-only mount for the controller manager kubeconfig file controllerManagerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName) - mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeConfigVolumeName, controllerManagerKubeConfigFile, controllerManagerKubeConfigFile, true) + mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeConfigVolumeName, controllerManagerKubeConfigFile, controllerManagerKubeConfigFile, true, &hostPathFileOrCreate) + // Mount for the flexvolume directory (/usr/libexec/kubernetes/kubelet-plugins/volume/exec) directory + // Flexvolume dir must NOT be readonly as it is used for third-party plugins to integrate with their storage backends via unix domain socket. + mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, flexvolumeDirVolumeName, flexvolumeDirVolumePath, flexvolumeDirVolumePath, false, &hostPathDirectoryOrCreate) // HostPath volumes for the scheduler // Read-only mount for the scheduler kubeconfig file schedulerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName) - mounts.NewHostPathMount(kubeadmconstants.KubeScheduler, kubeConfigVolumeName, schedulerKubeConfigFile, schedulerKubeConfigFile, true) + mounts.NewHostPathMount(kubeadmconstants.KubeScheduler, kubeadmconstants.KubeConfigVolumeName, schedulerKubeConfigFile, schedulerKubeConfigFile, true, &hostPathFileOrCreate) // On some systems were we host-mount /etc/ssl/certs, it is also required to mount /etc/pki. This is needed // due to symlinks pointing from files in /etc/ssl/certs into /etc/pki/ if isPkiVolumeMountNeeded() { - mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true) - mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true) + mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true, &hostPathDirectoryOrCreate) + mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsPkiVolumeName, caCertsPkiVolumePath, caCertsPkiVolumePath, true, &hostPathDirectoryOrCreate) } return mounts @@ -97,8 +102,8 @@ func newControlPlaneHostPathMounts() controlPlaneHostPathMounts { } } -func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool) { - c.volumes[component] = append(c.volumes[component], staticpodutil.NewVolume(mountName, hostPath)) +func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool, hostPathType *v1.HostPathType) { + c.volumes[component] = append(c.volumes[component], staticpodutil.NewVolume(mountName, hostPath, hostPathType)) c.volumeMounts[component] = append(c.volumeMounts[component], staticpodutil.NewVolumeMount(mountName, containerPath, readOnly)) } @@ -145,9 +150,10 @@ func getEtcdCertVolumes(etcdCfg kubeadmapi.Etcd) ([]v1.Volume, []v1.VolumeMount) volumes := []v1.Volume{} volumeMounts := []v1.VolumeMount{} + pathType := v1.HostPathDirectoryOrCreate for i, certDir := range certDirs.List() { name := fmt.Sprintf("etcd-certs-%d", i) - volumes = append(volumes, staticpodutil.NewVolume(name, certDir)) + volumes = append(volumes, staticpodutil.NewVolume(name, certDir, &pathType)) volumeMounts = append(volumeMounts, staticpodutil.NewVolumeMount(name, certDir, true)) } return volumes, volumeMounts diff --git a/cmd/kubeadm/app/phases/controlplane/volumes_test.go b/cmd/kubeadm/app/phases/controlplane/volumes_test.go index d711e03bad12d..af784c0e64b30 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes_test.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes_test.go @@ -29,6 +29,7 @@ import ( ) func TestGetEtcdCertVolumes(t *testing.T) { + hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate var tests = []struct { ca, cert, key string vol []v1.Volume @@ -67,7 +68,10 @@ func TestGetEtcdCertVolumes(t *testing.T) { { Name: "etcd-certs-0", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/etcd"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/certs/etcd", + Type: &hostPathDirectoryOrCreate, + }, }, }, }, @@ -88,13 +92,19 @@ func TestGetEtcdCertVolumes(t *testing.T) { { Name: "etcd-certs-0", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/certs/etcd", + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "etcd-certs-1", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/etcd"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/certs/etcd", + Type: &hostPathDirectoryOrCreate, + }, }, }, }, @@ -120,19 +130,28 @@ func TestGetEtcdCertVolumes(t *testing.T) { { Name: "etcd-certs-0", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/certs/etcd", + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "etcd-certs-1", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/etcd"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/certs/etcd", + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "etcd-certs-2", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/private"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/certs/private", + Type: &hostPathDirectoryOrCreate, + }, }, }, }, @@ -163,7 +182,10 @@ func TestGetEtcdCertVolumes(t *testing.T) { { Name: "etcd-certs-0", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/certs/etcd", + Type: &hostPathDirectoryOrCreate, + }, }, }, }, @@ -184,7 +206,10 @@ func TestGetEtcdCertVolumes(t *testing.T) { { Name: "etcd-certs-0", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/certs/etcd", + Type: &hostPathDirectoryOrCreate, + }, }, }, }, @@ -222,6 +247,8 @@ func TestGetEtcdCertVolumes(t *testing.T) { } func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { + hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate + hostPathFileOrCreate := v1.HostPathFileOrCreate var tests = []struct { cfg *kubeadmapi.MasterConfiguration vol map[string][]v1.Volume @@ -238,13 +265,19 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { { Name: "k8s-certs", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: testCertsDir}, + HostPath: &v1.HostPathVolumeSource{ + Path: testCertsDir, + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "ca-certs", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/ssl/certs"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/ssl/certs", + Type: &hostPathDirectoryOrCreate, + }, }, }, }, @@ -252,19 +285,37 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { { Name: "k8s-certs", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: testCertsDir}, + HostPath: &v1.HostPathVolumeSource{ + Path: testCertsDir, + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "ca-certs", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/ssl/certs"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/ssl/certs", + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "kubeconfig", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/controller-manager.conf"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/controller-manager.conf", + Type: &hostPathFileOrCreate, + }, + }, + }, + { + Name: "flexvolume-dir", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec", + Type: &hostPathDirectoryOrCreate, + }, }, }, }, @@ -272,7 +323,10 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { { Name: "kubeconfig", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/scheduler.conf"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/scheduler.conf", + Type: &hostPathFileOrCreate, + }, }, }, }, @@ -306,6 +360,11 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { MountPath: "/etc/kubernetes/controller-manager.conf", ReadOnly: true, }, + { + Name: "flexvolume-dir", + MountPath: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec", + ReadOnly: false, + }, }, kubeadmconstants.KubeScheduler: { { @@ -332,25 +391,37 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { { Name: "k8s-certs", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: testCertsDir}, + HostPath: &v1.HostPathVolumeSource{ + Path: testCertsDir, + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "ca-certs", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/ssl/certs"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/ssl/certs", + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "etcd-certs-0", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/certs/etcd"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/certs/etcd", + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "etcd-certs-1", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/var/lib/certs/etcd"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/certs/etcd", + Type: &hostPathDirectoryOrCreate, + }, }, }, }, @@ -358,19 +429,37 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { { Name: "k8s-certs", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: testCertsDir}, + HostPath: &v1.HostPathVolumeSource{ + Path: testCertsDir, + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "ca-certs", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/ssl/certs"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/ssl/certs", + Type: &hostPathDirectoryOrCreate, + }, }, }, { Name: "kubeconfig", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/controller-manager.conf"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/controller-manager.conf", + Type: &hostPathFileOrCreate, + }, + }, + }, + { + Name: "flexvolume-dir", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec", + Type: &hostPathDirectoryOrCreate, + }, }, }, }, @@ -378,7 +467,10 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { { Name: "kubeconfig", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/scheduler.conf"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/scheduler.conf", + Type: &hostPathFileOrCreate, + }, }, }, }, @@ -422,6 +514,11 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { MountPath: "/etc/kubernetes/controller-manager.conf", ReadOnly: true, }, + { + Name: "flexvolume-dir", + MountPath: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec", + ReadOnly: false, + }, }, kubeadmconstants.KubeScheduler: { { diff --git a/cmd/kubeadm/app/phases/etcd/BUILD b/cmd/kubeadm/app/phases/etcd/BUILD index 91d159ac60ffa..ba43cd2aff7ac 100644 --- a/cmd/kubeadm/app/phases/etcd/BUILD +++ b/cmd/kubeadm/app/phases/etcd/BUILD @@ -28,6 +28,7 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/images:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/staticpod:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", ], diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 17f2af83814a8..e947794fccb5e 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -17,10 +17,13 @@ limitations under the License. package etcd import ( + "fmt" + "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) @@ -35,12 +38,18 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.Ma spec := GetEtcdPodSpec(cfg) // writes etcd StaticPod to disk - return staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec) + if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil { + return err + } + + fmt.Printf("[etcd] Wrote Static Pod manifest for a local etcd instance to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir)) + return nil } // GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current MasterConfiguration // NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod mainfests. func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { + pathType := v1.HostPathDirectoryOrCreate return staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.Etcd, Command: getEtcdCommand(cfg), @@ -48,7 +57,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)}, LivenessProbe: staticpodutil.ComponentProbe(2379, "/health", v1.URISchemeHTTP), - }, []v1.Volume{staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir)}) + }, []v1.Volume{staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType)}) } // getEtcdCommand builds the right etcd command from the given config object @@ -60,6 +69,6 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { } command := []string{"etcd"} - command = append(command, staticpodutil.GetExtraParameters(cfg.Etcd.ExtraArgs, defaultArguments)...) + command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.ExtraArgs)...) return command } diff --git a/cmd/kubeadm/app/phases/kubeconfig/BUILD b/cmd/kubeadm/app/phases/kubeconfig/BUILD index 2ead488e93e70..d62577dd5b389 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/BUILD +++ b/cmd/kubeadm/app/phases/kubeconfig/BUILD @@ -16,6 +16,7 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", @@ -44,6 +45,7 @@ go_test( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/test:go_default_library", "//cmd/kubeadm/test/certs:go_default_library", "//cmd/kubeadm/test/kubeconfig:go_default_library", diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go index 667283b9258b8..1d1474417baeb 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go @@ -32,6 +32,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" ) @@ -134,10 +135,15 @@ func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeCo return nil, fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err) } + masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg) + if err != nil { + return nil, err + } + var kubeConfigSpec = map[string]*kubeConfigSpec{ kubeadmconstants.AdminKubeConfigFileName: { CACert: caCert, - APIServer: cfg.GetMasterEndpoint(), + APIServer: masterEndpoint, ClientName: "kubernetes-admin", ClientCertAuth: &clientCertAuth{ CAKey: caKey, @@ -146,7 +152,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeCo }, kubeadmconstants.KubeletKubeConfigFileName: { CACert: caCert, - APIServer: cfg.GetMasterEndpoint(), + APIServer: masterEndpoint, ClientName: fmt.Sprintf("system:node:%s", cfg.NodeName), ClientCertAuth: &clientCertAuth{ CAKey: caKey, @@ -155,7 +161,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeCo }, kubeadmconstants.ControllerManagerKubeConfigFileName: { CACert: caCert, - APIServer: cfg.GetMasterEndpoint(), + APIServer: masterEndpoint, ClientName: kubeadmconstants.ControllerManagerUser, ClientCertAuth: &clientCertAuth{ CAKey: caKey, @@ -163,7 +169,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeCo }, kubeadmconstants.SchedulerKubeConfigFileName: { CACert: caCert, - APIServer: cfg.GetMasterEndpoint(), + APIServer: masterEndpoint, ClientName: kubeadmconstants.SchedulerUser, ClientCertAuth: &clientCertAuth{ CAKey: caKey, @@ -266,9 +272,14 @@ func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.MasterConfigur return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err) } + masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg) + if err != nil { + return err + } + spec := &kubeConfigSpec{ ClientName: clientName, - APIServer: cfg.GetMasterEndpoint(), + APIServer: masterEndpoint, CACert: caCert, ClientCertAuth: &clientCertAuth{ CAKey: caKey, @@ -287,9 +298,14 @@ func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.MasterConfiguration return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err) } + masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg) + if err != nil { + return err + } + spec := &kubeConfigSpec{ ClientName: clientName, - APIServer: cfg.GetMasterEndpoint(), + APIServer: masterEndpoint, CACert: caCert, TokenAuth: &tokenAuth{ Token: token, diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go index c2e376c248ed9..5290b2fb598a8 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go @@ -34,6 +34,7 @@ import ( pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" testutil "k8s.io/kubernetes/cmd/kubeadm/test" certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs" kubeconfigtestutil "k8s.io/kubernetes/cmd/kubeadm/test/kubeconfig" @@ -117,8 +118,12 @@ func TestGetKubeConfigSpecs(t *testing.T) { } // Asserts MasterConfiguration values injected into spec - if spec.APIServer != cfg.GetMasterEndpoint() { - t.Errorf("getKubeConfigSpecs didn't injected cfg.APIServer address into spec for %s", assertion.kubeConfigFile) + masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg) + if err != nil { + t.Error(err) + } + if spec.APIServer != masterEndpoint { + t.Errorf("getKubeConfigSpecs didn't injected cfg.APIServer endpoint into spec for %s", assertion.kubeConfigFile) } // Asserts CA certs and CA keys loaded into specs diff --git a/cmd/kubeadm/app/phases/markmaster/markmaster.go b/cmd/kubeadm/app/phases/markmaster/markmaster.go index 79e817ffc0ba1..9917f08a92b9a 100644 --- a/cmd/kubeadm/app/phases/markmaster/markmaster.go +++ b/cmd/kubeadm/app/phases/markmaster/markmaster.go @@ -34,6 +34,8 @@ import ( // MarkMaster taints the master and sets the master label func MarkMaster(client clientset.Interface, masterName string) error { + fmt.Printf("[markmaster] Will mark node %s as master by adding a label and a taint\n", masterName) + // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.MarkMasterTimeout, func() (bool, error) { // First get the node object diff --git a/cmd/kubeadm/app/phases/selfhosting/BUILD b/cmd/kubeadm/app/phases/selfhosting/BUILD index 4f541f6c0fec3..0860458c1af45 100644 --- a/cmd/kubeadm/app/phases/selfhosting/BUILD +++ b/cmd/kubeadm/app/phases/selfhosting/BUILD @@ -15,8 +15,6 @@ go_test( ], library = ":go_default_library", deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/cmd/features:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", @@ -32,8 +30,9 @@ go_library( ], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/cmd/features:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//pkg/api:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go index 3bb36afca67d3..65575bb27b4b2 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go @@ -17,43 +17,55 @@ limitations under the License. package selfhosting import ( + "path/filepath" + "k8s.io/api/core/v1" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) -// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting -func mutatePodSpec(cfg *kubeadmapi.MasterConfiguration, name string, podSpec *v1.PodSpec) { - mutators := map[string][]func(*kubeadmapi.MasterConfiguration, *v1.PodSpec){ +const ( + // selfHostedKubeConfigDir sets the directory where kubeconfig files for the scheduler and controller-manager should be mounted + // Due to how the projected volume mount works (can only be a full directory, not mount individual files), we must change this from + // the default as mounts cannot be nested (/etc/kubernetes would override /etc/kubernetes/pki) + selfHostedKubeConfigDir = "/etc/kubernetes/kubeconfig" +) + +// PodSpecMutatorFunc is a function capable of mutating a PodSpec +type PodSpecMutatorFunc func(*v1.PodSpec) + +// getDefaultMutators gets the mutator functions that alwasy should be used +func getDefaultMutators() map[string][]PodSpecMutatorFunc { + return map[string][]PodSpecMutatorFunc{ kubeadmconstants.KubeAPIServer: { addNodeSelectorToPodSpec, setMasterTolerationOnPodSpec, setRightDNSPolicyOnPodSpec, - setVolumesOnKubeAPIServerPodSpec, }, kubeadmconstants.KubeControllerManager: { addNodeSelectorToPodSpec, setMasterTolerationOnPodSpec, setRightDNSPolicyOnPodSpec, - setVolumesOnKubeControllerManagerPodSpec, }, kubeadmconstants.KubeScheduler: { addNodeSelectorToPodSpec, setMasterTolerationOnPodSpec, setRightDNSPolicyOnPodSpec, - setVolumesOnKubeSchedulerPodSpec, }, } +} +// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting +func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpec *v1.PodSpec) { // Get the mutator functions for the component in question, then loop through and execute them mutatorsForComponent := mutators[name] for _, mutateFunc := range mutatorsForComponent { - mutateFunc(cfg, podSpec) + mutateFunc(podSpec) } } // addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the master label -func addNodeSelectorToPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { +func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) { if podSpec.NodeSelector == nil { podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""} return @@ -63,7 +75,7 @@ func addNodeSelectorToPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.P } // setMasterTolerationOnPodSpec makes the Pod tolerate the master taint -func setMasterTolerationOnPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { +func setMasterTolerationOnPodSpec(podSpec *v1.PodSpec) { if podSpec.Tolerations == nil { podSpec.Tolerations = []v1.Toleration{kubeadmconstants.MasterToleration} return @@ -73,38 +85,68 @@ func setMasterTolerationOnPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec * } // setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary -func setRightDNSPolicyOnPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { +func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) { podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet } -// setVolumesOnKubeAPIServerPodSpec makes sure the self-hosted api server has the required files -func setVolumesOnKubeAPIServerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { - setK8sVolume(apiServerVolume, cfg, podSpec) - for _, c := range podSpec.Containers { - c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount()) +// setSelfHostedVolumesForAPIServer makes sure the self-hosted api server has the right volume source coming from a self-hosted cluster +func setSelfHostedVolumesForAPIServer(podSpec *v1.PodSpec) { + for i, v := range podSpec.Volumes { + // If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted + if v.Name == kubeadmconstants.KubeCertificatesVolumeName { + podSpec.Volumes[i].VolumeSource = apiServerCertificatesVolumeSource() + } } } -// setVolumesOnKubeControllerManagerPodSpec makes sure the self-hosted controller manager has the required files -func setVolumesOnKubeControllerManagerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { - setK8sVolume(controllerManagerVolume, cfg, podSpec) - for _, c := range podSpec.Containers { - c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount()) +// setSelfHostedVolumesForControllerManager makes sure the self-hosted controller manager has the right volume source coming from a self-hosted cluster +func setSelfHostedVolumesForControllerManager(podSpec *v1.PodSpec) { + for i, v := range podSpec.Volumes { + // If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted + if v.Name == kubeadmconstants.KubeCertificatesVolumeName { + podSpec.Volumes[i].VolumeSource = controllerManagerCertificatesVolumeSource() + } else if v.Name == kubeadmconstants.KubeConfigVolumeName { + podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName) + } } -} -// setVolumesOnKubeSchedulerPodSpec makes sure the self-hosted scheduler has the required files -func setVolumesOnKubeSchedulerPodSpec(cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { - setK8sVolume(schedulerVolume, cfg, podSpec) - for _, c := range podSpec.Containers { - c.VolumeMounts = append(c.VolumeMounts, k8sSelfHostedVolumeMount()) + // Change directory for the kubeconfig directory to selfHostedKubeConfigDir + for i, vm := range podSpec.Containers[0].VolumeMounts { + if vm.Name == kubeadmconstants.KubeConfigVolumeName { + podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir + } } + + // Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki) + // This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes + // don't support that. + podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string { + argMap["kubeconfig"] = filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.ControllerManagerKubeConfigFileName) + return argMap + }) } -func setK8sVolume(cb func(cfg *kubeadmapi.MasterConfiguration) v1.Volume, cfg *kubeadmapi.MasterConfiguration, podSpec *v1.PodSpec) { +// setSelfHostedVolumesForScheduler makes sure the self-hosted scheduler has the right volume source coming from a self-hosted cluster +func setSelfHostedVolumesForScheduler(podSpec *v1.PodSpec) { for i, v := range podSpec.Volumes { - if v.Name == "k8s" { - podSpec.Volumes[i] = cb(cfg) + // If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted + if v.Name == kubeadmconstants.KubeConfigVolumeName { + podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName) } } + + // Change directory for the kubeconfig directory to selfHostedKubeConfigDir + for i, vm := range podSpec.Containers[0].VolumeMounts { + if vm.Name == kubeadmconstants.KubeConfigVolumeName { + podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir + } + } + + // Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki) + // This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes + // don't support that. + podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string { + argMap["kubeconfig"] = filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.SchedulerKubeConfigFileName) + return argMap + }) } diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go index 9b2ce07b38b6b..b5e2483c94ebe 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go @@ -18,10 +18,10 @@ package selfhosting import ( "reflect" + "sort" "testing" "k8s.io/api/core/v1" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) @@ -72,9 +72,8 @@ func TestMutatePodSpec(t *testing.T) { }, } - cfg := &kubeadmapi.MasterConfiguration{} for _, rt := range tests { - mutatePodSpec(cfg, rt.component, rt.podSpec) + mutatePodSpec(getDefaultMutators(), rt.component, rt.podSpec) if !reflect.DeepEqual(*rt.podSpec, rt.expected) { t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) @@ -110,9 +109,8 @@ func TestAddNodeSelectorToPodSpec(t *testing.T) { }, } - cfg := &kubeadmapi.MasterConfiguration{} for _, rt := range tests { - addNodeSelectorToPodSpec(cfg, rt.podSpec) + addNodeSelectorToPodSpec(rt.podSpec) if !reflect.DeepEqual(*rt.podSpec, rt.expected) { t.Errorf("failed addNodeSelectorToPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) @@ -148,9 +146,8 @@ func TestSetMasterTolerationOnPodSpec(t *testing.T) { }, } - cfg := &kubeadmapi.MasterConfiguration{} for _, rt := range tests { - setMasterTolerationOnPodSpec(cfg, rt.podSpec) + setMasterTolerationOnPodSpec(rt.podSpec) if !reflect.DeepEqual(*rt.podSpec, rt.expected) { t.Errorf("failed setMasterTolerationOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) @@ -179,12 +176,293 @@ func TestSetRightDNSPolicyOnPodSpec(t *testing.T) { }, } - cfg := &kubeadmapi.MasterConfiguration{} for _, rt := range tests { - setRightDNSPolicyOnPodSpec(cfg, rt.podSpec) + setRightDNSPolicyOnPodSpec(rt.podSpec) if !reflect.DeepEqual(*rt.podSpec, rt.expected) { t.Errorf("failed setRightDNSPolicyOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) } } } + +func TestSetSelfHostedVolumesForAPIServer(t *testing.T) { + hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate + var tests = []struct { + podSpec *v1.PodSpec + expected v1.PodSpec + }{ + { + podSpec: &v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "ca-certs", + MountPath: "/etc/ssl/certs", + }, + { + Name: "k8s-certs", + MountPath: "/etc/kubernetes/pki", + }, + }, + Command: []string{ + "--foo=bar", + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "ca-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/ssl/certs", + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + { + Name: "k8s-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/pki", + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + }, + }, + expected: v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "ca-certs", + MountPath: "/etc/ssl/certs", + }, + { + Name: "k8s-certs", + MountPath: "/etc/kubernetes/pki", + }, + }, + Command: []string{ + "--foo=bar", + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "ca-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/ssl/certs", + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + { + Name: "k8s-certs", + VolumeSource: apiServerCertificatesVolumeSource(), + }, + }, + }, + }, + } + + for _, rt := range tests { + setSelfHostedVolumesForAPIServer(rt.podSpec) + sort.Strings(rt.podSpec.Containers[0].Command) + sort.Strings(rt.expected.Containers[0].Command) + + if !reflect.DeepEqual(*rt.podSpec, rt.expected) { + t.Errorf("failed setSelfHostedVolumesForAPIServer:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) + } + } +} + +func TestSetSelfHostedVolumesForControllerManager(t *testing.T) { + hostPathFileOrCreate := v1.HostPathFileOrCreate + hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate + var tests = []struct { + podSpec *v1.PodSpec + expected v1.PodSpec + }{ + { + podSpec: &v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "ca-certs", + MountPath: "/etc/ssl/certs", + }, + { + Name: "k8s-certs", + MountPath: "/etc/kubernetes/pki", + }, + { + Name: "kubeconfig", + MountPath: "/etc/kubernetes/controller-manager.conf", + }, + }, + Command: []string{ + "--kubeconfig=/etc/kubernetes/controller-manager.conf", + "--foo=bar", + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "ca-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/ssl/certs", + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + { + Name: "k8s-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/pki", + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + { + Name: "kubeconfig", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/controller-manager.conf", + Type: &hostPathFileOrCreate, + }, + }, + }, + }, + }, + expected: v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "ca-certs", + MountPath: "/etc/ssl/certs", + }, + { + Name: "k8s-certs", + MountPath: "/etc/kubernetes/pki", + }, + { + Name: "kubeconfig", + MountPath: "/etc/kubernetes/kubeconfig", + }, + }, + Command: []string{ + "--kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf", + "--foo=bar", + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "ca-certs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/ssl/certs", + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + { + Name: "k8s-certs", + VolumeSource: controllerManagerCertificatesVolumeSource(), + }, + { + Name: "kubeconfig", + VolumeSource: kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName), + }, + }, + }, + }, + } + + for _, rt := range tests { + setSelfHostedVolumesForControllerManager(rt.podSpec) + sort.Strings(rt.podSpec.Containers[0].Command) + sort.Strings(rt.expected.Containers[0].Command) + + if !reflect.DeepEqual(*rt.podSpec, rt.expected) { + t.Errorf("failed setSelfHostedVolumesForControllerManager:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) + } + } +} + +func TestSetSelfHostedVolumesForScheduler(t *testing.T) { + hostPathFileOrCreate := v1.HostPathFileOrCreate + var tests = []struct { + podSpec *v1.PodSpec + expected v1.PodSpec + }{ + { + podSpec: &v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "kubeconfig", + MountPath: "/etc/kubernetes/scheduler.conf", + }, + }, + Command: []string{ + "--kubeconfig=/etc/kubernetes/scheduler.conf", + "--foo=bar", + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "kubeconfig", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/kubernetes/scheduler.conf", + Type: &hostPathFileOrCreate, + }, + }, + }, + }, + }, + expected: v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + { + Name: "kubeconfig", + MountPath: "/etc/kubernetes/kubeconfig", + }, + }, + Command: []string{ + "--kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf", + "--foo=bar", + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "kubeconfig", + VolumeSource: kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName), + }, + }, + }, + }, + } + + for _, rt := range tests { + setSelfHostedVolumesForScheduler(rt.podSpec) + sort.Strings(rt.podSpec.Containers[0].Command) + sort.Strings(rt.expected.Containers[0].Command) + + if !reflect.DeepEqual(*rt.podSpec, rt.expected) { + t.Errorf("failed setSelfHostedVolumesForScheduler:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) + } + } +} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go index 454d6d928c517..acae768abd929 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go @@ -28,8 +28,8 @@ import ( kuberuntime "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/features" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/api" ) @@ -37,6 +37,9 @@ import ( const ( // selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out selfHostingWaitTimeout = 2 * time.Minute + + // selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets + selfHostingFailureThreshold uint8 = 5 ) // CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one @@ -51,20 +54,33 @@ const ( // 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue // Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes // 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop -func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { +func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, waiter apiclient.Waiter) error { + + // Adjust the timeout slightly to something self-hosting specific + waiter.SetTimeout(selfHostingWaitTimeout) + + // Here the map of different mutators to use for the control plane's podspec is stored + mutators := getDefaultMutators() + + // Some extra work to be done if we should store the control plane certificates in Secrets + if features.Enabled(cfg.FeatureGates, features.StoreCertsInSecrets) { - if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) { - if err := createTLSSecrets(cfg, client); err != nil { + // Upload the certificates and kubeconfig files from disk to the cluster as Secrets + if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil { return err } - if err := createOpaqueSecrets(cfg, client); err != nil { + if err := uploadKubeConfigSecrets(client, kubeConfigDir); err != nil { return err } + // Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them + mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer) + mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager) + mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler) } for _, componentName := range kubeadmconstants.MasterComponents { start := time.Now() - manifestPath := kubeadmconstants.GetStaticPodFilepath(componentName, kubeadmconstants.GetStaticPodDirectory()) + manifestPath := kubeadmconstants.GetStaticPodFilepath(componentName, manifestsDir) // Since we want this function to be idempotent; just continue and try the next component if this file doesn't exist if _, err := os.Stat(manifestPath); err != nil { @@ -79,15 +95,17 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client cl } // Build a DaemonSet object from the loaded PodSpec - ds := buildDaemonSet(cfg, componentName, podSpec) + ds := buildDaemonSet(componentName, podSpec, mutators) - // Create the DaemonSet in the API Server - if err := apiclient.CreateOrUpdateDaemonSet(client, ds); err != nil { + // Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out + if err := apiclient.TryRunCommand(func() error { + return apiclient.CreateOrUpdateDaemonSet(client, ds) + }, selfHostingFailureThreshold); err != nil { return err } // Wait for the self-hosted component to come up - if err := apiclient.WaitForPodsWithLabel(client, selfHostingWaitTimeout, os.Stdout, buildSelfHostedWorkloadLabelQuery(componentName)); err != nil { + if err := waiter.WaitForPodsWithLabel(buildSelfHostedWorkloadLabelQuery(componentName)); err != nil { return err } @@ -100,12 +118,12 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client cl // remove the Static Pod (or the mirror Pod respectively). This implicitely also tests that the API server endpoint is healthy, // because this blocks until the API server returns a 404 Not Found when getting the Static Pod staticPodName := fmt.Sprintf("%s-%s", componentName, cfg.NodeName) - if err := apiclient.WaitForStaticPodToDisappear(client, selfHostingWaitTimeout, staticPodName); err != nil { + if err := waiter.WaitForPodToDisappear(staticPodName); err != nil { return err } // Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint (although we know it could return a GET answer for a Pod above) - if err := apiclient.WaitForAPI(client, selfHostingWaitTimeout); err != nil { + if err := waiter.WaitForAPI(); err != nil { return err } @@ -115,10 +133,10 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client cl } // buildDaemonSet is responsible for mutating the PodSpec and return a DaemonSet which is suitable for the self-hosting purporse -func buildDaemonSet(cfg *kubeadmapi.MasterConfiguration, name string, podSpec *v1.PodSpec) *extensions.DaemonSet { +func buildDaemonSet(name string, podSpec *v1.PodSpec, mutators map[string][]PodSpecMutatorFunc) *extensions.DaemonSet { // Mutate the PodSpec so it's suitable for self-hosting - mutatePodSpec(cfg, name, podSpec) + mutatePodSpec(mutators, name, podSpec) // Return a DaemonSet based on that Spec return &extensions.DaemonSet{ diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go index 0a1afc69b7f55..3ac5a6e6ad7e6 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go @@ -21,88 +21,13 @@ import ( "fmt" "io/ioutil" "os" - "strings" "testing" "github.com/ghodss/yaml" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/features" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) const ( - apiProjectedSecret = `- name: k8s - projected: - sources: - - secret: - items: - - key: tls.crt - path: ca.crt - - key: tls.key - path: ca.key - name: ca - - secret: - items: - - key: tls.crt - path: apiserver.crt - - key: tls.key - path: apiserver.key - name: apiserver - - secret: - items: - - key: tls.crt - path: apiserver-kubelet-client.crt - - key: tls.key - path: apiserver-kubelet-client.key - name: apiserver-kubelet-client - - secret: - items: - - key: tls.crt - path: sa.pub - - key: tls.key - path: sa.key - name: sa - - secret: - items: - - key: tls.crt - path: front-proxy-ca.crt - name: front-proxy-ca - - secret: - items: - - key: tls.crt - path: front-proxy-client.crt - - key: tls.key - path: front-proxy-client.key - name: front-proxy-client` - - controllerManagerProjectedSecret = `- name: k8s - projected: - sources: - - secret: - name: controller-manager.conf - - secret: - items: - - key: tls.crt - path: ca.crt - - key: tls.key - path: ca.key - name: ca - - secret: - items: - - key: tls.key - path: sa.key - name: sa` - - schedulerProjectedSecret = `- name: k8s - projected: - sources: - - secret: - name: scheduler.conf` - - hostPathVol = `- hostPath: - path: /etc/kubernetes - name: k8s` - testAPIServerPod = ` apiVersion: v1 kind: Pod @@ -110,39 +35,36 @@ metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" creationTimestamp: null - labels: - component: kube-apiserver - tier: control-plane name: kube-apiserver namespace: kube-system spec: containers: - command: - kube-apiserver - - --client-ca-file=/etc/kubernetes/pki/ca.crt - - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key - - --allow-privileged=true - - --service-cluster-ip-range=10.96.0.0/12 - --service-account-key-file=/etc/kubernetes/pki/sa.pub + - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key + - --secure-port=6443 + - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --requestheader-group-headers=X-Remote-Group + - --service-cluster-ip-range=10.96.0.0/12 - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt - - --secure-port=6443 + - --advertise-address=192.168.1.115 + - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - --insecure-port=0 - - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota - - --requestheader-extra-headers-prefix=X-Remote-Extra- - - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt - --experimental-bootstrap-token-auth=true - - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-allowed-names=front-proxy-client - - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt + - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota + - --allow-privileged=true + - --client-ca-file=/etc/kubernetes/pki/ca.crt + - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --requestheader-username-headers=X-Remote-User - --authorization-mode=Node,RBAC - - --advertise-address=192.168.200.101 - --etcd-servers=http://127.0.0.1:2379 - image: gcr.io/google_containers/kube-apiserver-amd64:v1.7.0 + image: gcr.io/google_containers/kube-apiserver-amd64:v1.7.4 livenessProbe: failureThreshold: 8 httpGet: @@ -157,22 +79,26 @@ spec: requests: cpu: 250m volumeMounts: - - mountPath: /etc/kubernetes - name: k8s + - mountPath: /etc/kubernetes/pki + name: k8s-certs readOnly: true - mountPath: /etc/ssl/certs - name: certs + name: ca-certs + readOnly: true - mountPath: /etc/pki - name: pki + name: ca-certs-etc-pki + readOnly: true hostNetwork: true volumes: - %s + - hostPath: + path: /etc/kubernetes/pki + name: k8s-certs - hostPath: path: /etc/ssl/certs - name: certs + name: ca-certs - hostPath: path: /etc/pki - name: pki + name: ca-certs-etc-pki status: {} ` @@ -192,30 +118,30 @@ spec: containers: - command: - kube-apiserver - - --client-ca-file=/etc/kubernetes/pki/ca.crt - - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key - - --allow-privileged=true - - --service-cluster-ip-range=10.96.0.0/12 - --service-account-key-file=/etc/kubernetes/pki/sa.pub + - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key + - --secure-port=6443 + - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --requestheader-group-headers=X-Remote-Group + - --service-cluster-ip-range=10.96.0.0/12 - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt - - --secure-port=6443 + - --advertise-address=192.168.1.115 + - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - --insecure-port=0 - - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota - - --requestheader-extra-headers-prefix=X-Remote-Extra- - - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt - --experimental-bootstrap-token-auth=true - - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-allowed-names=front-proxy-client - - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt + - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota + - --allow-privileged=true + - --client-ca-file=/etc/kubernetes/pki/ca.crt + - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --requestheader-username-headers=X-Remote-User - --authorization-mode=Node,RBAC - - --advertise-address=192.168.200.101 - --etcd-servers=http://127.0.0.1:2379 - image: gcr.io/google_containers/kube-apiserver-amd64:v1.7.0 + image: gcr.io/google_containers/kube-apiserver-amd64:v1.7.4 livenessProbe: failureThreshold: 8 httpGet: @@ -230,13 +156,15 @@ spec: requests: cpu: 250m volumeMounts: - - mountPath: /etc/kubernetes - name: k8s + - mountPath: /etc/kubernetes/pki + name: k8s-certs readOnly: true - mountPath: /etc/ssl/certs - name: certs + name: ca-certs + readOnly: true - mountPath: /etc/pki - name: pki + name: ca-certs-etc-pki + readOnly: true dnsPolicy: ClusterFirstWithHostNet hostNetwork: true nodeSelector: @@ -245,13 +173,15 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - %s + - hostPath: + path: /etc/kubernetes/pki + name: k8s-certs - hostPath: path: /etc/ssl/certs - name: certs + name: ca-certs - hostPath: path: /etc/pki - name: pki + name: ca-certs-etc-pki updateStrategy: {} status: currentNumberScheduled: 0 @@ -267,25 +197,22 @@ metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" creationTimestamp: null - labels: - component: kube-controller-manager - tier: control-plane name: kube-controller-manager namespace: kube-system spec: containers: - command: - kube-controller-manager - - --service-account-private-key-file=/etc/kubernetes/pki/sa.key - - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - --leader-elect=true - - --kubeconfig=/etc/kubernetes/controller-manager.conf - --controllers=*,bootstrapsigner,tokencleaner + - --kubeconfig=/etc/kubernetes/controller-manager.conf - --root-ca-file=/etc/kubernetes/pki/ca.crt + - --service-account-private-key-file=/etc/kubernetes/pki/sa.key + - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt + - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - --address=127.0.0.1 - --use-service-account-credentials=true - image: gcr.io/google_containers/kube-controller-manager-amd64:v1.7.0 + image: gcr.io/google_containers/kube-controller-manager-amd64:v1.7.4 livenessProbe: failureThreshold: 8 httpGet: @@ -300,22 +227,33 @@ spec: requests: cpu: 200m volumeMounts: - - mountPath: /etc/kubernetes - name: k8s + - mountPath: /etc/kubernetes/pki + name: k8s-certs readOnly: true - mountPath: /etc/ssl/certs - name: certs + name: ca-certs + readOnly: true + - mountPath: /etc/kubernetes/controller-manager.conf + name: kubeconfig + readOnly: true - mountPath: /etc/pki - name: pki + name: ca-certs-etc-pki + readOnly: true hostNetwork: true volumes: - %s + - hostPath: + path: /etc/kubernetes/pki + name: k8s-certs - hostPath: path: /etc/ssl/certs - name: certs + name: ca-certs + - hostPath: + path: /etc/kubernetes/controller-manager.conf + type: FileOrCreate + name: kubeconfig - hostPath: path: /etc/pki - name: pki + name: ca-certs-etc-pki status: {} ` @@ -335,16 +273,16 @@ spec: containers: - command: - kube-controller-manager - - --service-account-private-key-file=/etc/kubernetes/pki/sa.key - - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - --leader-elect=true - - --kubeconfig=/etc/kubernetes/controller-manager.conf - --controllers=*,bootstrapsigner,tokencleaner + - --kubeconfig=/etc/kubernetes/controller-manager.conf - --root-ca-file=/etc/kubernetes/pki/ca.crt + - --service-account-private-key-file=/etc/kubernetes/pki/sa.key + - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt + - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - --address=127.0.0.1 - --use-service-account-credentials=true - image: gcr.io/google_containers/kube-controller-manager-amd64:v1.7.0 + image: gcr.io/google_containers/kube-controller-manager-amd64:v1.7.4 livenessProbe: failureThreshold: 8 httpGet: @@ -359,13 +297,18 @@ spec: requests: cpu: 200m volumeMounts: - - mountPath: /etc/kubernetes - name: k8s + - mountPath: /etc/kubernetes/pki + name: k8s-certs readOnly: true - mountPath: /etc/ssl/certs - name: certs + name: ca-certs + readOnly: true + - mountPath: /etc/kubernetes/controller-manager.conf + name: kubeconfig + readOnly: true - mountPath: /etc/pki - name: pki + name: ca-certs-etc-pki + readOnly: true dnsPolicy: ClusterFirstWithHostNet hostNetwork: true nodeSelector: @@ -374,13 +317,19 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - %s + - hostPath: + path: /etc/kubernetes/pki + name: k8s-certs - hostPath: path: /etc/ssl/certs - name: certs + name: ca-certs + - hostPath: + path: /etc/kubernetes/controller-manager.conf + type: FileOrCreate + name: kubeconfig - hostPath: path: /etc/pki - name: pki + name: ca-certs-etc-pki updateStrategy: {} status: currentNumberScheduled: 0 @@ -396,19 +345,16 @@ metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" creationTimestamp: null - labels: - component: kube-scheduler - tier: control-plane name: kube-scheduler namespace: kube-system spec: containers: - command: - kube-scheduler - - --address=127.0.0.1 - --leader-elect=true - --kubeconfig=/etc/kubernetes/scheduler.conf - image: gcr.io/google_containers/kube-scheduler-amd64:v1.7.0 + - --address=127.0.0.1 + image: gcr.io/google_containers/kube-scheduler-amd64:v1.7.4 livenessProbe: failureThreshold: 8 httpGet: @@ -423,12 +369,15 @@ spec: requests: cpu: 100m volumeMounts: - - mountPath: /etc/kubernetes - name: k8s + - mountPath: /etc/kubernetes/scheduler.conf + name: kubeconfig readOnly: true hostNetwork: true volumes: - %s + - hostPath: + path: /etc/kubernetes/scheduler.conf + type: FileOrCreate + name: kubeconfig status: {} ` @@ -448,10 +397,10 @@ spec: containers: - command: - kube-scheduler - - --address=127.0.0.1 - --leader-elect=true - --kubeconfig=/etc/kubernetes/scheduler.conf - image: gcr.io/google_containers/kube-scheduler-amd64:v1.7.0 + - --address=127.0.0.1 + image: gcr.io/google_containers/kube-scheduler-amd64:v1.7.4 livenessProbe: failureThreshold: 8 httpGet: @@ -466,8 +415,8 @@ spec: requests: cpu: 100m volumeMounts: - - mountPath: /etc/kubernetes - name: k8s + - mountPath: /etc/kubernetes/scheduler.conf + name: kubeconfig readOnly: true dnsPolicy: ClusterFirstWithHostNet hostNetwork: true @@ -477,7 +426,10 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - %s + - hostPath: + path: /etc/kubernetes/scheduler.conf + type: FileOrCreate + name: kubeconfig updateStrategy: {} status: currentNumberScheduled: 0 @@ -487,67 +439,26 @@ status: ` ) -var ( - testAPIServerSecretsPod = fmt.Sprintf(testAPIServerPod, apiProjectedSecret) - testAPIServerSecretsDS = fmt.Sprintf(testAPIServerDaemonSet, indentString(apiProjectedSecret, 4)) - testAPIServerHostPathPod = fmt.Sprintf(testAPIServerPod, hostPathVol) - testAPIServerHostPathDS = fmt.Sprintf(testAPIServerDaemonSet, indentString(hostPathVol, 4)) - - testSchedulerSecretsPod = fmt.Sprintf(testSchedulerPod, schedulerProjectedSecret) - testSchedulerSecretsDS = fmt.Sprintf(testSchedulerDaemonSet, indentString(schedulerProjectedSecret, 4)) - testSchedulerHostPathPod = fmt.Sprintf(testSchedulerPod, hostPathVol) - testSchedulerHostPathDS = fmt.Sprintf(testSchedulerDaemonSet, indentString(hostPathVol, 4)) - - testControllerManagerSecretsPod = fmt.Sprintf(testControllerManagerPod, controllerManagerProjectedSecret) - testControllerManagerSecretsDS = fmt.Sprintf(testControllerManagerDaemonSet, indentString(controllerManagerProjectedSecret, 4)) - testControllerManagerHostPathPod = fmt.Sprintf(testControllerManagerPod, hostPathVol) - testControllerManagerHostPathDS = fmt.Sprintf(testControllerManagerDaemonSet, indentString(hostPathVol, 4)) -) - func TestBuildDaemonSet(t *testing.T) { var tests = []struct { - component string - podBytes []byte - dsBytes []byte - selfHostedSecrets bool + component string + podBytes []byte + dsBytes []byte }{ - // vols as secrets - { - component: kubeadmconstants.KubeAPIServer, - podBytes: []byte(testAPIServerSecretsPod), - dsBytes: []byte(testAPIServerSecretsDS), - selfHostedSecrets: true, - }, - { - component: kubeadmconstants.KubeControllerManager, - podBytes: []byte(testControllerManagerSecretsPod), - dsBytes: []byte(testControllerManagerSecretsDS), - selfHostedSecrets: true, - }, - { - component: kubeadmconstants.KubeScheduler, - podBytes: []byte(testSchedulerSecretsPod), - dsBytes: []byte(testSchedulerSecretsDS), - selfHostedSecrets: true, - }, - // hostPath vols { - component: kubeadmconstants.KubeAPIServer, - podBytes: []byte(testAPIServerHostPathPod), - dsBytes: []byte(testAPIServerHostPathDS), - selfHostedSecrets: false, + component: constants.KubeAPIServer, + podBytes: []byte(testAPIServerPod), + dsBytes: []byte(testAPIServerDaemonSet), }, { - component: kubeadmconstants.KubeControllerManager, - podBytes: []byte(testControllerManagerHostPathPod), - dsBytes: []byte(testControllerManagerHostPathDS), - selfHostedSecrets: false, + component: constants.KubeControllerManager, + podBytes: []byte(testControllerManagerPod), + dsBytes: []byte(testControllerManagerDaemonSet), }, { - component: kubeadmconstants.KubeScheduler, - podBytes: []byte(testSchedulerHostPathPod), - dsBytes: []byte(testSchedulerHostPathDS), - selfHostedSecrets: false, + component: constants.KubeScheduler, + podBytes: []byte(testSchedulerPod), + dsBytes: []byte(testSchedulerDaemonSet), }, } @@ -557,21 +468,17 @@ func TestBuildDaemonSet(t *testing.T) { podSpec, err := loadPodSpecFromFile(tempFile) if err != nil { - t.Fatalf("couldn't load the specified Pod: %v", err) + t.Fatalf("couldn't load the specified Pod") } - cfg := &kubeadmapi.MasterConfiguration{ - FeatureFlags: map[string]bool{string(features.StoreCertsInSecrets): rt.selfHostedSecrets}, - } - - ds := buildDaemonSet(cfg, rt.component, podSpec) + ds := buildDaemonSet(rt.component, podSpec, getDefaultMutators()) dsBytes, err := yaml.Marshal(ds) if err != nil { t.Fatalf("failed to marshal daemonset to YAML: %v", err) } if !bytes.Equal(dsBytes, rt.dsBytes) { - t.Errorf("failed TestBuildDaemonSet for name=%s (secrets=%t):\nexpected:\n%s\nsaw:\n%s", rt.component, rt.selfHostedSecrets, rt.dsBytes, dsBytes) + t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes) } } } @@ -651,18 +558,3 @@ func createTempFileWithContent(content []byte) (string, error) { } return tempFile.Name(), nil } - -func indentString(input string, count int) string { - output := "" - lines := strings.Split(input, "\n") - for i, line := range lines { - if i > 0 { - output += strings.Repeat(" ", count) - } - output += line - if i < len(lines)-1 { - output += "\n" - } - } - return output -} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go index 4c5c7f37db90d..627fb01f043f6 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go @@ -19,19 +19,13 @@ package selfhosting import ( "fmt" "io/ioutil" - "path" + "path/filepath" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/features" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -const ( - volumeName = "k8s" - volumeMountName = "k8s" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) type tlsKeyPair struct { @@ -40,234 +34,164 @@ type tlsKeyPair struct { key string } -func k8sSelfHostedVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ - Name: volumeMountName, - MountPath: kubeadmconstants.KubernetesDir, - ReadOnly: true, - } -} - -func apiServerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { - var volumeSource v1.VolumeSource - if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) { - volumeSource = v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.CACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.CACertName), - }, - { - Key: v1.TLSPrivateKeyKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.CAKeyName), - }, - }, +func apiServerCertificatesVolumeSource() v1.VolumeSource { + return v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{ + { + Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: kubeadmconstants.CACertAndKeyBaseName, }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.APIServerCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.APIServerCertName), - }, - { - Key: v1.TLSPrivateKeyKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.APIServerKeyName), - }, + Items: []v1.KeyToPath{ + { + Key: v1.TLSCertKey, + Path: kubeadmconstants.CACertName, }, }, }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, + }, + { + Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: kubeadmconstants.APIServerCertAndKeyBaseName, + }, + Items: []v1.KeyToPath{ + { + Key: v1.TLSCertKey, + Path: kubeadmconstants.APIServerCertName, }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.APIServerKubeletClientCertName), - }, - { - Key: v1.TLSPrivateKeyKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.APIServerKubeletClientKeyName), - }, + { + Key: v1.TLSPrivateKeyKey, + Path: kubeadmconstants.APIServerKeyName, }, }, }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.ServiceAccountKeyBaseName, + }, + { + Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, + }, + Items: []v1.KeyToPath{ + { + Key: v1.TLSCertKey, + Path: kubeadmconstants.APIServerKubeletClientCertName, }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.ServiceAccountPublicKeyName), - }, - { - Key: v1.TLSPrivateKeyKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.ServiceAccountPrivateKeyName), - }, + { + Key: v1.TLSPrivateKeyKey, + Path: kubeadmconstants.APIServerKubeletClientKeyName, }, }, }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.FrontProxyCACertName), - }, + }, + { + Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: kubeadmconstants.ServiceAccountKeyBaseName, + }, + Items: []v1.KeyToPath{ + { + Key: v1.TLSCertKey, + Path: kubeadmconstants.ServiceAccountPublicKeyName, }, }, }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.FrontProxyClientCertName), - }, - { - Key: v1.TLSPrivateKeyKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.FrontProxyClientKeyName), - }, + }, + { + Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName, + }, + Items: []v1.KeyToPath{ + { + Key: v1.TLSCertKey, + Path: kubeadmconstants.FrontProxyCACertName, }, }, }, }, - }, - } - } else { - volumeSource = v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: kubeadmconstants.KubernetesDir, - }, - } - } - return v1.Volume{ - Name: volumeName, - VolumeSource: volumeSource, - } -} - -func schedulerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { - var volumeSource v1.VolumeSource - if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) { - volumeSource = v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.SchedulerKubeConfigFileName, + { + Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName, + }, + Items: []v1.KeyToPath{ + { + Key: v1.TLSCertKey, + Path: kubeadmconstants.FrontProxyClientCertName, + }, + { + Key: v1.TLSPrivateKeyKey, + Path: kubeadmconstants.FrontProxyClientKeyName, }, }, }, }, }, - } - } else { - volumeSource = v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: kubeadmconstants.KubernetesDir, - }, - } - } - return v1.Volume{ - Name: volumeName, - VolumeSource: volumeSource, + }, } } -func controllerManagerVolume(cfg *kubeadmapi.MasterConfiguration) v1.Volume { - var volumeSource v1.VolumeSource - if features.Enabled(cfg.FeatureFlags, features.StoreCertsInSecrets) { - volumeSource = v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.ControllerManagerKubeConfigFileName, - }, +func controllerManagerCertificatesVolumeSource() v1.VolumeSource { + return v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{ + { + Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: kubeadmconstants.CACertAndKeyBaseName, }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.CACertAndKeyBaseName, + Items: []v1.KeyToPath{ + { + Key: v1.TLSCertKey, + Path: kubeadmconstants.CACertName, }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.CACertName), - }, - { - Key: v1.TLSPrivateKeyKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.CAKeyName), - }, + { + Key: v1.TLSPrivateKeyKey, + Path: kubeadmconstants.CAKeyName, }, }, }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.ServiceAccountKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSPrivateKeyKey, - Path: path.Join(path.Base(cfg.CertificatesDir), kubeadmconstants.ServiceAccountPrivateKeyName), - }, + }, + { + Secret: &v1.SecretProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: kubeadmconstants.ServiceAccountKeyBaseName, + }, + Items: []v1.KeyToPath{ + { + Key: v1.TLSPrivateKeyKey, + Path: kubeadmconstants.ServiceAccountPrivateKeyName, }, }, }, }, }, - } - } else { - volumeSource = v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: kubeadmconstants.KubernetesDir, - }, - } + }, } - return v1.Volume{ - Name: volumeName, - VolumeSource: volumeSource, +} + +func kubeConfigVolumeSource(kubeconfigSecretName string) v1.VolumeSource { + return v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: kubeconfigSecretName, + }, } } -func createTLSSecrets(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { +func uploadTLSSecrets(client clientset.Interface, certDir string) error { for _, tlsKeyPair := range getTLSKeyPairs() { secret, err := createTLSSecretFromFiles( tlsKeyPair.name, - path.Join(cfg.CertificatesDir, tlsKeyPair.cert), - path.Join(cfg.CertificatesDir, tlsKeyPair.key), + filepath.Join(certDir, tlsKeyPair.cert), + filepath.Join(certDir, tlsKeyPair.key), ) if err != nil { return err } - if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret); err != nil { + if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil { return err } fmt.Printf("[self-hosted] Created TLS secret %q from %s and %s\n", tlsKeyPair.name, tlsKeyPair.cert, tlsKeyPair.key) @@ -276,24 +200,22 @@ func createTLSSecrets(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte return nil } -func createOpaqueSecrets(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { +func uploadKubeConfigSecrets(client clientset.Interface, kubeConfigDir string) error { files := []string{ kubeadmconstants.SchedulerKubeConfigFileName, kubeadmconstants.ControllerManagerKubeConfigFileName, } for _, file := range files { - secret, err := createOpaqueSecretFromFile( - file, - path.Join(kubeadmconstants.KubernetesDir, file), - ) + kubeConfigPath := filepath.Join(kubeConfigDir, file) + secret, err := createOpaqueSecretFromFile(file, kubeConfigPath) if err != nil { return err } - if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret); err != nil { + if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil { return err } - fmt.Printf("[self-hosted] Created secret %q\n", file) + fmt.Printf("[self-hosted] Created secret for kubeconfig file %q\n", file) } return nil @@ -335,7 +257,7 @@ func createOpaqueSecretFromFile(secretName, file string) (*v1.Secret, error) { }, Type: v1.SecretTypeOpaque, Data: map[string][]byte{ - path.Base(file): fileBytes, + filepath.Base(file): fileBytes, }, }, nil } diff --git a/cmd/kubeadm/app/phases/upgrade/BUILD b/cmd/kubeadm/app/phases/upgrade/BUILD new file mode 100644 index 0000000000000..ae4a84b4a547c --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/BUILD @@ -0,0 +1,36 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "compute.go", + "configuration.go", + "health.go", + "policy.go", + "postupgrade.go", + "staticpods.go", + "versiongetter.go", + ], + visibility = ["//visibility:public"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", + "//pkg/api:go_default_library", + "//pkg/util/version:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kubeadm/app/phases/upgrade/compute.go b/cmd/kubeadm/app/phases/upgrade/compute.go new file mode 100644 index 0000000000000..e81eb93b9d3b2 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/compute.go @@ -0,0 +1,63 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "fmt" +) + +// Upgrade defines an upgrade possibility to upgrade from a current version to a new one +type Upgrade struct { + Description string + Before ClusterState + After ClusterState +} + +// CanUpgradeKubelets returns whether an upgrade of any kubelet in the cluster is possible +func (u *Upgrade) CanUpgradeKubelets() bool { + // If there are multiple different versions now, an upgrade is possible (even if only for a subset of the nodes) + if len(u.Before.KubeletVersions) > 1 { + return true + } + // Don't report something available for upgrade if we don't know the current state + if len(u.Before.KubeletVersions) == 0 { + return false + } + + // if the same version number existed both before and after, we don't have to upgrade it + _, sameVersionFound := u.Before.KubeletVersions[u.After.KubeVersion] + return !sameVersionFound +} + +// ClusterState describes the state of certain versions for a cluster +type ClusterState struct { + // KubeVersion describes the version of the Kubernetes API Server, Controller Manager, Scheduler and Proxy. + KubeVersion string + // DNSVersion describes the version of the kube-dns images used and manifest version + DNSVersion string + // KubeadmVersion describes the version of the kubeadm CLI + KubeadmVersion string + // KubeletVersions is a map with a version number linked to the amount of kubelets running that version in the cluster + KubeletVersions map[string]uint16 +} + +// GetAvailableUpgrades fetches all versions from the specified VersionGetter and computes which +// kinds of upgrades can be performed +func GetAvailableUpgrades(_ VersionGetter, _, _ bool) ([]Upgrade, error) { + fmt.Println("[upgrade] Fetching available versions to upgrade to:") + return []Upgrade{}, nil +} diff --git a/cmd/kubeadm/app/phases/upgrade/configuration.go b/cmd/kubeadm/app/phases/upgrade/configuration.go new file mode 100644 index 0000000000000..a91e4140a8a06 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/configuration.go @@ -0,0 +1,36 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "fmt" + "io" + + clientset "k8s.io/client-go/kubernetes" + kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/pkg/api" +) + +// FetchConfiguration fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster +func FetchConfiguration(_ clientset.Interface, _ io.Writer, _ string) (*kubeadmapiext.MasterConfiguration, error) { + fmt.Println("[upgrade/config] Making sure the configuration is correct:") + + cfg := &kubeadmapiext.MasterConfiguration{} + api.Scheme.Default(cfg) + + return cfg, nil +} diff --git a/cmd/kubeadm/app/phases/upgrade/health.go b/cmd/kubeadm/app/phases/upgrade/health.go new file mode 100644 index 0000000000000..1beac03999660 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/health.go @@ -0,0 +1,36 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + clientset "k8s.io/client-go/kubernetes" +) + +// CheckClusterHealth makes sure: +// - the API /healthz endpoint is healthy +// - all Nodes are Ready +// - (if self-hosted) that there are DaemonSets with at least one Pod for all control plane components +// - (if static pod-hosted) that all required Static Pod manifests exist on disk +func CheckClusterHealth(_ clientset.Interface) error { + return nil +} + +// IsControlPlaneSelfHosted returns whether the control plane is self hosted or not +func IsControlPlaneSelfHosted(_ clientset.Interface) bool { + // No-op for now + return false +} diff --git a/cmd/kubeadm/app/phases/upgrade/policy.go b/cmd/kubeadm/app/phases/upgrade/policy.go new file mode 100644 index 0000000000000..6b06a4116e4f1 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/policy.go @@ -0,0 +1,33 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "k8s.io/kubernetes/pkg/util/version" +) + +// VersionSkewPolicyErrors describes version skew errors that might be seen during the validation process in EnforceVersionPolicies +type VersionSkewPolicyErrors struct { + Mandatory []error + Skippable []error +} + +// EnforceVersionPolicies enforces that the proposed new version is compatible with all the different version skew policies +func EnforceVersionPolicies(_ VersionGetter, _ string, _ *version.Version, _, _ bool) *VersionSkewPolicyErrors { + // No-op now and return no skew errors + return nil +} diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go new file mode 100644 index 0000000000000..3510caa57a27d --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -0,0 +1,30 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/pkg/util/version" +) + +// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do +// Note that the markmaster phase is left out, not needed, and no token is created as that doesn't belong to the upgrade +func PerformPostUpgradeTasks(_ clientset.Interface, _ *kubeadmapi.MasterConfiguration, _ *version.Version) error { + // No-op; don't do anything here yet + return nil +} diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go new file mode 100644 index 0000000000000..6970560f54137 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -0,0 +1,29 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/pkg/util/version" +) + +// PerformStaticPodControlPlaneUpgrade upgrades a static pod-hosted control plane +func PerformStaticPodControlPlaneUpgrade(_ clientset.Interface, _ *kubeadmapi.MasterConfiguration, _ *version.Version) error { + // No-op for now; doesn't do anything yet + return nil +} diff --git a/cmd/kubeadm/app/phases/upgrade/versiongetter.go b/cmd/kubeadm/app/phases/upgrade/versiongetter.go new file mode 100644 index 0000000000000..879ca11276f30 --- /dev/null +++ b/cmd/kubeadm/app/phases/upgrade/versiongetter.go @@ -0,0 +1,83 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "fmt" + "io" + + clientset "k8s.io/client-go/kubernetes" + versionutil "k8s.io/kubernetes/pkg/util/version" +) + +// VersionGetter defines an interface for fetching different versions. +// Easy to implement a fake variant of this interface for unit testing +type VersionGetter interface { + // ClusterVersion should return the version of the cluster i.e. the API Server version + ClusterVersion() (string, *versionutil.Version, error) + // KubeadmVersion should return the version of the kubeadm CLI + KubeadmVersion() (string, *versionutil.Version, error) + // VersionFromCILabel should resolve CI labels like `latest`, `stable`, `stable-1.8`, etc. to real versions + VersionFromCILabel(string, string) (string, *versionutil.Version, error) + // KubeletVersions should return a map with a version and a number that describes how many kubelets there are for that version + KubeletVersions() (map[string]uint16, error) +} + +// KubeVersionGetter handles the version-fetching mechanism from external sources +type KubeVersionGetter struct { + client clientset.Interface + w io.Writer +} + +// Make sure KubeVersionGetter implements the VersionGetter interface +var _ VersionGetter = &KubeVersionGetter{} + +// NewKubeVersionGetter returns a new instance of KubeVersionGetter +func NewKubeVersionGetter(client clientset.Interface, writer io.Writer) *KubeVersionGetter { + return &KubeVersionGetter{ + client: client, + w: writer, + } +} + +// ClusterVersion gets API server version +func (g *KubeVersionGetter) ClusterVersion() (string, *versionutil.Version, error) { + fmt.Fprintf(g.w, "[upgrade/versions] Cluster version: ") + fmt.Fprintln(g.w, "v1.7.0") + + return "v1.7.0", versionutil.MustParseSemantic("v1.7.0"), nil +} + +// KubeadmVersion gets kubeadm version +func (g *KubeVersionGetter) KubeadmVersion() (string, *versionutil.Version, error) { + fmt.Fprintf(g.w, "[upgrade/versions] kubeadm version: %s\n", "v1.8.0") + + return "v1.8.0", versionutil.MustParseSemantic("v1.8.0"), nil +} + +// VersionFromCILabel resolves different labels like "stable" to action semver versions using the Kubernetes CI uploads to GCS +func (g *KubeVersionGetter) VersionFromCILabel(_, _ string) (string, *versionutil.Version, error) { + return "v1.8.1", versionutil.MustParseSemantic("v1.8.0"), nil +} + +// KubeletVersions gets the versions of the kubelets in the cluster +func (g *KubeVersionGetter) KubeletVersions() (map[string]uint16, error) { + // This tells kubeadm that there are two nodes in the cluster; both on the v1.7.1 version currently + return map[string]uint16{ + "v1.7.1": 2, + }, nil +} diff --git a/cmd/kubeadm/app/preflight/BUILD b/cmd/kubeadm/app/preflight/BUILD index 28212bc28e4da..b2ae8681ae0fb 100644 --- a/cmd/kubeadm/app/preflight/BUILD +++ b/cmd/kubeadm/app/preflight/BUILD @@ -17,6 +17,8 @@ go_library( "//pkg/api/validation:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", "//pkg/util/initsystem:go_default_library", + "//pkg/util/version:go_default_library", + "//pkg/version:go_default_library", "//plugin/cmd/kube-scheduler/app/options:go_default_library", "//test/e2e_node/system:go_default_library", "//vendor/github.com/PuerkitoBio/purell:go_default_library", diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index 9d2ddee7e3eab..1e766db4dbd7f 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -28,6 +28,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "crypto/tls" @@ -46,6 +47,8 @@ import ( "k8s.io/kubernetes/pkg/api/validation" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" "k8s.io/kubernetes/pkg/util/initsystem" + versionutil "k8s.io/kubernetes/pkg/util/version" + kubeadmversion "k8s.io/kubernetes/pkg/version" schoptions "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options" "k8s.io/kubernetes/test/e2e_node/system" ) @@ -394,6 +397,40 @@ func (sysver SystemVerificationCheck) Check() (warnings, errors []error) { return warns, nil } +type KubernetesVersionCheck struct { + KubeadmVersion string + KubernetesVersion string +} + +func (kubever KubernetesVersionCheck) Check() (warnings, errors []error) { + + // Skip this check for "super-custom builds", where apimachinery/the overall codebase version is not set. + if strings.HasPrefix(kubever.KubeadmVersion, "v0.0.0") { + return nil, nil + } + + kadmVersion, err := versionutil.ParseSemantic(kubever.KubeadmVersion) + if err != nil { + return nil, []error{fmt.Errorf("couldn't parse kubeadm version %q: %v", kubever.KubeadmVersion, err)} + } + + k8sVersion, err := versionutil.ParseSemantic(kubever.KubernetesVersion) + if err != nil { + return nil, []error{fmt.Errorf("couldn't parse kubernetes version %q: %v", kubever.KubernetesVersion, err)} + } + + // Checks if k8sVersion greater or equal than the first unsupported versions by current version of kubeadm, + // that is major.minor+1 (all patch and pre-releases versions included) + // NB. in semver patches number is a numeric, while prerelease is a string where numeric identifiers always have lower precedence than non-numeric identifiers. + // thus setting the value to x.y.0-0 we are defining the very first patch - prereleases within x.y minor release. + firstUnsupportedVersion := versionutil.MustParseSemantic(fmt.Sprintf("%d.%d.%s", kadmVersion.Major(), kadmVersion.Minor()+1, "0-0")) + if k8sVersion.AtLeast(firstUnsupportedVersion) { + return []error{fmt.Errorf("kubernetes version is greater than kubeadm version. Please consider to upgrade kubeadm. kubernetes version: %s. Kubeadm version: %d.%d.x", k8sVersion, kadmVersion.Components()[0], kadmVersion.Components()[1])}, nil + } + + return nil, nil +} + type etcdVersionResponse struct { Etcdserver string `json:"etcdserver"` Etcdcluster string `json:"etcdcluster"` @@ -534,6 +571,7 @@ func RunInitMasterChecks(cfg *kubeadmapi.MasterConfiguration) error { } checks := []Checker{ + KubernetesVersionCheck{KubernetesVersion: cfg.KubernetesVersion, KubeadmVersion: kubeadmversion.Get().GitVersion}, SystemVerificationCheck{}, IsRootCheck{}, HostnameCheck{nodeName: cfg.NodeName}, diff --git a/cmd/kubeadm/app/preflight/checks_test.go b/cmd/kubeadm/app/preflight/checks_test.go index f790f354291ab..2db21a775fdf2 100644 --- a/cmd/kubeadm/app/preflight/checks_test.go +++ b/cmd/kubeadm/app/preflight/checks_test.go @@ -26,7 +26,6 @@ import ( "os" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) @@ -180,7 +179,7 @@ func TestRunInitMasterChecks(t *testing.T) { }{ { cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadm.API{AdvertiseAddress: "foo"}, + API: kubeadmapi.API{AdvertiseAddress: "foo"}, }, expected: false, }, @@ -347,3 +346,66 @@ func TestConfigCertAndKey(t *testing.T) { ) } } + +func TestKubernetesVersionCheck(t *testing.T) { + var tests = []struct { + check KubernetesVersionCheck + expectWarnings bool + }{ + { + check: KubernetesVersionCheck{ + KubeadmVersion: "v1.6.6", //Same version + KubernetesVersion: "v1.6.6", + }, + expectWarnings: false, + }, + { + check: KubernetesVersionCheck{ + KubeadmVersion: "v1.6.6", //KubernetesVersion version older than KubeadmVersion + KubernetesVersion: "v1.5.5", + }, + expectWarnings: false, + }, + { + check: KubernetesVersionCheck{ + KubeadmVersion: "v1.6.6", //KubernetesVersion newer than KubeadmVersion, within the same minor release (new patch) + KubernetesVersion: "v1.6.7", + }, + expectWarnings: false, + }, + { + check: KubernetesVersionCheck{ + KubeadmVersion: "v1.6.6", //KubernetesVersion newer than KubeadmVersion, in a different minor/in pre-release + KubernetesVersion: "v1.7.0-alpha.0", + }, + expectWarnings: true, + }, + { + check: KubernetesVersionCheck{ + KubeadmVersion: "v1.6.6", //KubernetesVersion newer than KubeadmVersion, in a different minor/stable + KubernetesVersion: "v1.7.0", + }, + expectWarnings: true, + }, + { + check: KubernetesVersionCheck{ + KubeadmVersion: "v0.0.0", //"super-custom" builds - Skip this check + KubernetesVersion: "v1.7.0", + }, + expectWarnings: false, + }, + } + + for _, rt := range tests { + warning, _ := rt.check.Check() + if (warning != nil) != rt.expectWarnings { + t.Errorf( + "failed KubernetesVersionCheck:\n\texpected: %t\n\t actual: %t (KubeadmVersion:%s, KubernetesVersion: %s)", + rt.expectWarnings, + (warning != nil), + rt.check.KubeadmVersion, + rt.check.KubernetesVersion, + ) + } + } +} diff --git a/cmd/kubeadm/app/util/BUILD b/cmd/kubeadm/app/util/BUILD index e06553c7893aa..62caae99a8980 100644 --- a/cmd/kubeadm/app/util/BUILD +++ b/cmd/kubeadm/app/util/BUILD @@ -9,11 +9,14 @@ load( go_library( name = "go_default_library", srcs = [ + "arguments.go", + "endpoint.go", "error.go", "template.go", "version.go", ], deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", ], @@ -22,12 +25,17 @@ go_library( go_test( name = "go_default_test", srcs = [ + "arguments_test.go", + "endpoint_test.go", "error_test.go", "template_test.go", "version_test.go", ], library = ":go_default_library", - deps = ["//cmd/kubeadm/app/preflight:go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/preflight:go_default_library", + ], ) filegroup( @@ -43,6 +51,7 @@ filegroup( ":package-srcs", "//cmd/kubeadm/app/util/apiclient:all-srcs", "//cmd/kubeadm/app/util/config:all-srcs", + "//cmd/kubeadm/app/util/dryrun:all-srcs", "//cmd/kubeadm/app/util/kubeconfig:all-srcs", "//cmd/kubeadm/app/util/pubkeypin:all-srcs", "//cmd/kubeadm/app/util/staticpod:all-srcs", diff --git a/cmd/kubeadm/app/util/apiclient/BUILD b/cmd/kubeadm/app/util/apiclient/BUILD index 02286772b76b4..1537d4d22f1da 100644 --- a/cmd/kubeadm/app/util/apiclient/BUILD +++ b/cmd/kubeadm/app/util/apiclient/BUILD @@ -3,23 +3,37 @@ package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( name = "go_default_library", srcs = [ + "clientbacked_dryrun.go", + "dryrunclient.go", "idempotency.go", + "init_dryrun.go", "wait.go", ], deps = [ "//cmd/kubeadm/app/constants:go_default_library", + "//pkg/registry/core/service/ipallocator:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/api/rbac/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/testing:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", ], ) @@ -35,3 +49,19 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = [ + "dryrunclient_test.go", + "init_dryrun_test.go", + ], + library = ":go_default_library", + deps = [ + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/api/rbac/v1beta1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/client-go/testing:go_default_library", + ], +) diff --git a/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go b/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go new file mode 100644 index 0000000000000..6d9266276e95b --- /dev/null +++ b/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go @@ -0,0 +1,139 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiclient + +import ( + "encoding/json" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + core "k8s.io/client-go/testing" + "k8s.io/client-go/tools/clientcmd" +) + +// ClientBackedDryRunGetter implements the DryRunGetter interface for use in NewDryRunClient() and proxies all GET and LIST requests to the backing API server reachable via rest.Config +type ClientBackedDryRunGetter struct { + baseConfig *rest.Config + dynClientPool dynamic.ClientPool +} + +// InitDryRunGetter should implement the DryRunGetter interface +var _ DryRunGetter = &ClientBackedDryRunGetter{} + +// NewClientBackedDryRunGetter creates a new ClientBackedDryRunGetter instance based on the rest.Config object +func NewClientBackedDryRunGetter(config *rest.Config) *ClientBackedDryRunGetter { + return &ClientBackedDryRunGetter{ + baseConfig: config, + dynClientPool: dynamic.NewDynamicClientPool(config), + } +} + +// NewClientBackedDryRunGetterFromKubeconfig creates a new ClientBackedDryRunGetter instance from the given KubeConfig file +func NewClientBackedDryRunGetterFromKubeconfig(file string) (*ClientBackedDryRunGetter, error) { + config, err := clientcmd.LoadFromFile(file) + if err != nil { + return nil, fmt.Errorf("failed to load kubeconfig: %v", err) + } + clientConfig, err := clientcmd.NewDefaultClientConfig(*config, &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return nil, fmt.Errorf("failed to create API client configuration from kubeconfig: %v", err) + } + return NewClientBackedDryRunGetter(clientConfig), nil +} + +// HandleGetAction handles GET actions to the dryrun clientset this interface supports +func (clg *ClientBackedDryRunGetter) HandleGetAction(action core.GetAction) (bool, runtime.Object, error) { + rc, err := clg.actionToResourceClient(action) + if err != nil { + return true, nil, err + } + + unversionedObj, err := rc.Get(action.GetName(), metav1.GetOptions{}) + // If the unversioned object does not have .apiVersion; the inner object is probably nil + if len(unversionedObj.GetAPIVersion()) == 0 { + return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), action.GetName()) + } + newObj, err := decodeUnversionedIntoAPIObject(action, unversionedObj) + if err != nil { + fmt.Printf("error after decode: %v %v\n", unversionedObj, err) + return true, nil, err + } + return true, newObj, err +} + +// HandleListAction handles LIST actions to the dryrun clientset this interface supports +func (clg *ClientBackedDryRunGetter) HandleListAction(action core.ListAction) (bool, runtime.Object, error) { + rc, err := clg.actionToResourceClient(action) + if err != nil { + return true, nil, err + } + + listOpts := metav1.ListOptions{ + LabelSelector: action.GetListRestrictions().Labels.String(), + FieldSelector: action.GetListRestrictions().Fields.String(), + } + + unversionedList, err := rc.List(listOpts) + // If the runtime.Object here is nil, we should return successfully with no result + if unversionedList == nil { + return true, unversionedList, nil + } + newObj, err := decodeUnversionedIntoAPIObject(action, unversionedList) + if err != nil { + fmt.Printf("error after decode: %v %v\n", unversionedList, err) + return true, nil, err + } + return true, newObj, err +} + +// actionToResourceClient returns the ResourceInterface for the given action +// First; the function gets the right API group interface from the resource type. The API group struct behind the interface +// returned may be cached in the dynamic client pool. Then, an APIResource object is constructed so that it can be passed to +// dynamic.Interface's Resource() function, which will give us the final ResourceInterface to query +func (clg *ClientBackedDryRunGetter) actionToResourceClient(action core.Action) (dynamic.ResourceInterface, error) { + dynIface, err := clg.dynClientPool.ClientForGroupVersionResource(action.GetResource()) + if err != nil { + return nil, err + } + + apiResource := &metav1.APIResource{ + Name: action.GetResource().Resource, + Namespaced: action.GetNamespace() != "", + } + + return dynIface.Resource(apiResource, action.GetNamespace()), nil +} + +// decodeUnversionedIntoAPIObject converts the *unversioned.Unversioned object returned from the dynamic client +// to bytes; and then decodes it back _to an external api version (k8s.io/api vs k8s.io/kubernetes/pkg/api*)_ using the normal API machinery +func decodeUnversionedIntoAPIObject(action core.Action, unversionedObj runtime.Object) (runtime.Object, error) { + objBytes, err := json.Marshal(unversionedObj) + if err != nil { + return nil, err + } + newObj, err := kuberuntime.Decode(clientsetscheme.Codecs.UniversalDecoder(action.GetResource().GroupVersion()), objBytes) + if err != nil { + return nil, err + } + return newObj, nil +} diff --git a/cmd/kubeadm/app/util/apiclient/dryrunclient.go b/cmd/kubeadm/app/util/apiclient/dryrunclient.go new file mode 100644 index 0000000000000..e20ce89221c5f --- /dev/null +++ b/cmd/kubeadm/app/util/apiclient/dryrunclient.go @@ -0,0 +1,244 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiclient + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + clientset "k8s.io/client-go/kubernetes" + fakeclientset "k8s.io/client-go/kubernetes/fake" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + core "k8s.io/client-go/testing" +) + +// DryRunGetter is an interface that must be supplied to the NewDryRunClient function in order to contstruct a fully functional fake dryrun clientset +type DryRunGetter interface { + HandleGetAction(core.GetAction) (bool, runtime.Object, error) + HandleListAction(core.ListAction) (bool, runtime.Object, error) +} + +// MarshalFunc takes care of converting any object to a byte array for displaying the object to the user +type MarshalFunc func(runtime.Object, schema.GroupVersion) ([]byte, error) + +// DefaultMarshalFunc is the default MarshalFunc used; uses YAML to print objects to the user +func DefaultMarshalFunc(obj runtime.Object, gv schema.GroupVersion) ([]byte, error) { + mediaType := "application/yaml" + info, ok := runtime.SerializerInfoForMediaType(clientsetscheme.Codecs.SupportedMediaTypes(), mediaType) + if !ok { + return []byte{}, fmt.Errorf("unsupported media type %q", mediaType) + } + + encoder := clientsetscheme.Codecs.EncoderForVersion(info.Serializer, gv) + return runtime.Encode(encoder, obj) +} + +// DryRunClientOptions specifies options to pass to NewDryRunClientWithOpts in order to get a dryrun clientset +type DryRunClientOptions struct { + Writer io.Writer + Getter DryRunGetter + PrependReactors []core.Reactor + AppendReactors []core.Reactor + MarshalFunc MarshalFunc + PrintGETAndLIST bool +} + +// actionWithName is the generic interface for an action that has a name associated with it +// This just makes it easier to catch all actions that has a name; instead of hard-coding all request that has it associated +type actionWithName interface { + core.Action + GetName() string +} + +// actionWithObject is the generic interface for an action that has an object associated with it +// This just makes it easier to catch all actions that has an object; instead of hard-coding all request that has it associated +type actionWithObject interface { + core.Action + GetObject() runtime.Object +} + +// NewDryRunClient is a wrapper for NewDryRunClientWithOpts using some default values +func NewDryRunClient(drg DryRunGetter, w io.Writer) clientset.Interface { + return NewDryRunClientWithOpts(DryRunClientOptions{ + Writer: w, + Getter: drg, + PrependReactors: []core.Reactor{}, + AppendReactors: []core.Reactor{}, + MarshalFunc: DefaultMarshalFunc, + PrintGETAndLIST: false, + }) +} + +// NewDryRunClientWithOpts returns a clientset.Interface that can be used normally for talking to the Kubernetes API. +// This client doesn't apply changes to the backend. The client gets GET/LIST values from the DryRunGetter implementation. +// This client logs all I/O to the writer w in YAML format +func NewDryRunClientWithOpts(opts DryRunClientOptions) clientset.Interface { + // Build a chain of reactors to act like a normal clientset; but log everything's that happening and don't change any state + client := fakeclientset.NewSimpleClientset() + + // Build the chain of reactors. Order matters; first item here will be invoked first on match, then the second one will be evaluted, etc. + defaultReactorChain := []core.Reactor{ + // Log everything that happens. Default the object if it's about to be created/updated so that the logged object is representative. + &core.SimpleReactor{ + Verb: "*", + Resource: "*", + Reaction: func(action core.Action) (bool, runtime.Object, error) { + logDryRunAction(action, opts.Writer, opts.MarshalFunc) + + return false, nil, nil + }, + }, + // Let the DryRunGetter implementation take care of all GET requests. + // The DryRunGetter implementation may call a real API Server behind the scenes or just fake everything + &core.SimpleReactor{ + Verb: "get", + Resource: "*", + Reaction: func(action core.Action) (bool, runtime.Object, error) { + getAction, ok := action.(core.GetAction) + if !ok { + // something's wrong, we can't handle this event + return true, nil, fmt.Errorf("can't cast get reactor event action object to GetAction interface") + } + handled, obj, err := opts.Getter.HandleGetAction(getAction) + + if opts.PrintGETAndLIST { + // Print the marshalled object format with one tab indentation + objBytes, err := opts.MarshalFunc(obj, action.GetResource().GroupVersion()) + if err == nil { + fmt.Println("[dryrun] Returning faked GET response:") + PrintBytesWithLinePrefix(opts.Writer, objBytes, "\t") + } + } + + return handled, obj, err + }, + }, + // Let the DryRunGetter implementation take care of all GET requests. + // The DryRunGetter implementation may call a real API Server behind the scenes or just fake everything + &core.SimpleReactor{ + Verb: "list", + Resource: "*", + Reaction: func(action core.Action) (bool, runtime.Object, error) { + listAction, ok := action.(core.ListAction) + if !ok { + // something's wrong, we can't handle this event + return true, nil, fmt.Errorf("can't cast list reactor event action object to ListAction interface") + } + handled, objs, err := opts.Getter.HandleListAction(listAction) + + if opts.PrintGETAndLIST { + // Print the marshalled object format with one tab indentation + objBytes, err := opts.MarshalFunc(objs, action.GetResource().GroupVersion()) + if err == nil { + fmt.Println("[dryrun] Returning faked LIST response:") + PrintBytesWithLinePrefix(opts.Writer, objBytes, "\t") + } + } + + return handled, objs, err + }, + }, + // For the verbs that modify anything on the server; just return the object if present and exit successfully + &core.SimpleReactor{ + Verb: "create", + Resource: "*", + Reaction: successfulModificationReactorFunc, + }, + &core.SimpleReactor{ + Verb: "update", + Resource: "*", + Reaction: successfulModificationReactorFunc, + }, + &core.SimpleReactor{ + Verb: "delete", + Resource: "*", + Reaction: successfulModificationReactorFunc, + }, + &core.SimpleReactor{ + Verb: "delete-collection", + Resource: "*", + Reaction: successfulModificationReactorFunc, + }, + &core.SimpleReactor{ + Verb: "patch", + Resource: "*", + Reaction: successfulModificationReactorFunc, + }, + } + + // The chain of reactors will look like this: + // opts.PrependReactors | defaultReactorChain | opts.AppendReactors | client.Fake.ReactionChain (default reactors for the fake clientset) + fullReactorChain := append(opts.PrependReactors, defaultReactorChain...) + fullReactorChain = append(fullReactorChain, opts.AppendReactors...) + + // Prepend the reaction chain with our reactors. Important, these MUST be prepended; not appended due to how the fake clientset works by default + client.Fake.ReactionChain = append(fullReactorChain, client.Fake.ReactionChain...) + return client +} + +// successfulModificationReactorFunc is a no-op that just returns the POSTed/PUTed value if present; but does nothing to edit any backing data store. +func successfulModificationReactorFunc(action core.Action) (bool, runtime.Object, error) { + objAction, ok := action.(actionWithObject) + if ok { + return true, objAction.GetObject(), nil + } + return true, nil, nil +} + +// logDryRunAction logs the action that was recorded by the "catch-all" (*,*) reactor and tells the user what would have happened in an user-friendly way +func logDryRunAction(action core.Action, w io.Writer, marshalFunc MarshalFunc) { + + group := action.GetResource().Group + if len(group) == 0 { + group = "core" + } + fmt.Fprintf(w, "[dryrun] Would perform action %s on resource %q in API group \"%s/%s\"\n", strings.ToUpper(action.GetVerb()), action.GetResource().Resource, group, action.GetResource().Version) + + namedAction, ok := action.(actionWithName) + if ok { + fmt.Fprintf(w, "[dryrun] Resource name: %q\n", namedAction.GetName()) + } + + objAction, ok := action.(actionWithObject) + if ok && objAction.GetObject() != nil { + // Print the marshalled object with a tab indentation + objBytes, err := marshalFunc(objAction.GetObject(), action.GetResource().GroupVersion()) + if err == nil { + fmt.Println("[dryrun] Attached object:") + PrintBytesWithLinePrefix(w, objBytes, "\t") + } + } + + patchAction, ok := action.(core.PatchAction) + if ok { + // Replace all occurences of \" with a simple " when printing + fmt.Fprintf(w, "[dryrun] Attached patch:\n\t%s\n", strings.Replace(string(patchAction.GetPatch()), `\"`, `"`, -1)) + } +} + +// PrintBytesWithLinePrefix prints objBytes to writer w with linePrefix in the beginning of every line +func PrintBytesWithLinePrefix(w io.Writer, objBytes []byte, linePrefix string) { + scanner := bufio.NewScanner(bytes.NewReader(objBytes)) + for scanner.Scan() { + fmt.Fprintf(w, "%s%s\n", linePrefix, scanner.Text()) + } +} diff --git a/cmd/kubeadm/app/util/apiclient/dryrunclient_test.go b/cmd/kubeadm/app/util/apiclient/dryrunclient_test.go new file mode 100644 index 0000000000000..f3c4a7fec9d5a --- /dev/null +++ b/cmd/kubeadm/app/util/apiclient/dryrunclient_test.go @@ -0,0 +1,102 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiclient + +import ( + "bytes" + "testing" + + "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + core "k8s.io/client-go/testing" +) + +func TestLogDryRunAction(t *testing.T) { + var tests = []struct { + action core.Action + expectedBytes []byte + buf *bytes.Buffer + }{ + { + action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "default", "kubernetes"), + expectedBytes: []byte(`[dryrun] Would perform action GET on resource "services" in API group "core/v1" +[dryrun] Resource name: "kubernetes" +`), + }, + { + action: core.NewRootGetAction(schema.GroupVersionResource{Group: rbac.GroupName, Version: rbac.SchemeGroupVersion.Version, Resource: "clusterrolebindings"}, "system:node"), + expectedBytes: []byte(`[dryrun] Would perform action GET on resource "clusterrolebindings" in API group "rbac.authorization.k8s.io/v1beta1" +[dryrun] Resource name: "system:node" +`), + }, + { + action: core.NewListAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, schema.GroupVersionKind{Version: "v1", Kind: "Service"}, "default", metav1.ListOptions{}), + expectedBytes: []byte(`[dryrun] Would perform action LIST on resource "services" in API group "core/v1" +`), + }, + { + action: core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "default", &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: v1.ServiceSpec{ + ClusterIP: "1.1.1.1", + }, + }), + expectedBytes: []byte(`[dryrun] Would perform action CREATE on resource "services" in API group "core/v1" + apiVersion: v1 + kind: Service + metadata: + creationTimestamp: null + name: foo + spec: + clusterIP: 1.1.1.1 + status: + loadBalancer: {} +`), + }, + { + action: core.NewPatchAction(schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, "default", "my-node", []byte(`{"spec":{"taints":[{"key": "foo", "value": "bar"}]}}`)), + expectedBytes: []byte(`[dryrun] Would perform action PATCH on resource "nodes" in API group "core/v1" +[dryrun] Resource name: "my-node" +[dryrun] Attached patch: + {"spec":{"taints":[{"key": "foo", "value": "bar"}]}} +`), + }, + { + action: core.NewDeleteAction(schema.GroupVersionResource{Version: "v1", Resource: "pods"}, "default", "my-pod"), + expectedBytes: []byte(`[dryrun] Would perform action DELETE on resource "pods" in API group "core/v1" +[dryrun] Resource name: "my-pod" +`), + }, + } + for _, rt := range tests { + rt.buf = bytes.NewBufferString("") + logDryRunAction(rt.action, rt.buf, DefaultMarshalFunc) + actualBytes := rt.buf.Bytes() + + if !bytes.Equal(actualBytes, rt.expectedBytes) { + t.Errorf( + "failed LogDryRunAction:\n\texpected bytes: %q\n\t actual: %q", + rt.expectedBytes, + actualBytes, + ) + } + } +} diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go index ecae7d87c9566..ae5886bf4d39c 100644 --- a/cmd/kubeadm/app/util/apiclient/idempotency.go +++ b/cmd/kubeadm/app/util/apiclient/idempotency.go @@ -43,6 +43,20 @@ func CreateOrUpdateConfigMap(client clientset.Interface, cm *v1.ConfigMap) error return nil } +// CreateOrUpdateSecret creates a Secret if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func CreateOrUpdateSecret(client clientset.Interface, secret *v1.Secret) error { + if _, err := client.CoreV1().Secrets(secret.ObjectMeta.Namespace).Create(secret); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create secret: %v", err) + } + + if _, err := client.CoreV1().Secrets(secret.ObjectMeta.Namespace).Update(secret); err != nil { + return fmt.Errorf("unable to update secret: %v", err) + } + } + return nil +} + // CreateOrUpdateServiceAccount creates a ServiceAccount if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. func CreateOrUpdateServiceAccount(client clientset.Interface, sa *v1.ServiceAccount) error { if _, err := client.CoreV1().ServiceAccounts(sa.ObjectMeta.Namespace).Create(sa); err != nil { diff --git a/cmd/kubeadm/app/util/apiclient/init_dryrun.go b/cmd/kubeadm/app/util/apiclient/init_dryrun.go new file mode 100644 index 0000000000000..ab31a98665499 --- /dev/null +++ b/cmd/kubeadm/app/util/apiclient/init_dryrun.go @@ -0,0 +1,162 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiclient + +import ( + "fmt" + "net" + "strings" + + "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + core "k8s.io/client-go/testing" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" +) + +// InitDryRunGetter implements the DryRunGetter interface and can be used to GET/LIST values in the dryrun fake clientset +// Need to handle these routes in a special manner: +// - GET /default/services/kubernetes -- must return a valid Service +// - GET /clusterrolebindings/system:nodes -- can safely return a NotFound error +// - GET /kube-system/secrets/bootstrap-token-* -- can safely return a NotFound error +// - GET /nodes/ -- must return a valid Node +// - ...all other, unknown GETs/LISTs will be logged +type InitDryRunGetter struct { + masterName string + serviceSubnet string +} + +// InitDryRunGetter should implement the DryRunGetter interface +var _ DryRunGetter = &InitDryRunGetter{} + +// NewInitDryRunGetter creates a new instance of the InitDryRunGetter struct +func NewInitDryRunGetter(masterName string, serviceSubnet string) *InitDryRunGetter { + return &InitDryRunGetter{ + masterName: masterName, + serviceSubnet: serviceSubnet, + } +} + +// HandleGetAction handles GET actions to the dryrun clientset this interface supports +func (idr *InitDryRunGetter) HandleGetAction(action core.GetAction) (bool, runtime.Object, error) { + funcs := []func(core.GetAction) (bool, runtime.Object, error){ + idr.handleKubernetesService, + idr.handleGetNode, + idr.handleSystemNodesClusterRoleBinding, + idr.handleGetBootstrapToken, + } + for _, f := range funcs { + handled, obj, err := f(action) + if handled { + return handled, obj, err + } + } + + return false, nil, nil +} + +// HandleListAction handles GET actions to the dryrun clientset this interface supports. +// Currently there are no known LIST calls during kubeadm init this code has to take care of. +func (idr *InitDryRunGetter) HandleListAction(action core.ListAction) (bool, runtime.Object, error) { + return false, nil, nil +} + +// handleKubernetesService returns a faked Kubernetes service in order to be able to continue running kubeadm init. +// The kube-dns addon code GETs the kubernetes service in order to extract the service subnet +func (idr *InitDryRunGetter) handleKubernetesService(action core.GetAction) (bool, runtime.Object, error) { + if action.GetName() != "kubernetes" || action.GetNamespace() != metav1.NamespaceDefault || action.GetResource().Resource != "services" { + // We can't handle this event + return false, nil, nil + } + + _, svcSubnet, err := net.ParseCIDR(idr.serviceSubnet) + if err != nil { + return true, nil, fmt.Errorf("error parsing CIDR %q: %v", idr.serviceSubnet, err) + } + + internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(svcSubnet, 1) + if err != nil { + return true, nil, fmt.Errorf("unable to get first IP address from the given CIDR (%s): %v", svcSubnet.String(), err) + } + + // The only used field of this Service object is the ClusterIP, which kube-dns uses to calculate its own IP + return true, &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubernetes", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{ + "component": "apiserver", + "provider": "kubernetes", + }, + }, + Spec: v1.ServiceSpec{ + ClusterIP: internalAPIServerVirtualIP.String(), + Ports: []v1.ServicePort{ + { + Name: "https", + Port: 443, + TargetPort: intstr.FromInt(6443), + }, + }, + }, + }, nil +} + +// handleGetNode returns a fake node object for the purpose of moving kubeadm init forwards. +func (idr *InitDryRunGetter) handleGetNode(action core.GetAction) (bool, runtime.Object, error) { + if action.GetName() != idr.masterName || action.GetResource().Resource != "nodes" { + // We can't handle this event + return false, nil, nil + } + + return true, &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: idr.masterName, + Labels: map[string]string{ + "kubernetes.io/hostname": idr.masterName, + }, + }, + Spec: v1.NodeSpec{ + ExternalID: idr.masterName, + }, + }, nil +} + +// handleSystemNodesClusterRoleBinding handles the GET call to the system:nodes clusterrolebinding +func (idr *InitDryRunGetter) handleSystemNodesClusterRoleBinding(action core.GetAction) (bool, runtime.Object, error) { + if action.GetName() != constants.NodesClusterRoleBinding || action.GetResource().Resource != "clusterrolebindings" { + // We can't handle this event + return false, nil, nil + } + // We can safely return a NotFound error here as the code will just proceed normally and don't care about modifying this clusterrolebinding + // This can only happen on an upgrade; and in that case the ClientBackedDryRunGetter impl will be used + return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), "clusterrolebinding not found") +} + +// handleGetBootstrapToken handles the case where kubeadm init creates the default token; and the token code GETs the +// bootstrap token secret first in order to check if it already exists +func (idr *InitDryRunGetter) handleGetBootstrapToken(action core.GetAction) (bool, runtime.Object, error) { + if !strings.HasPrefix(action.GetName(), "bootstrap-token-") || action.GetNamespace() != metav1.NamespaceSystem || action.GetResource().Resource != "secrets" { + // We can't handle this event + return false, nil, nil + } + // We can safely return a NotFound error here as the code will just proceed normally and create the Bootstrap Token + return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), "secret not found") +} diff --git a/cmd/kubeadm/app/util/apiclient/init_dryrun_test.go b/cmd/kubeadm/app/util/apiclient/init_dryrun_test.go new file mode 100644 index 0000000000000..9e681b5218746 --- /dev/null +++ b/cmd/kubeadm/app/util/apiclient/init_dryrun_test.go @@ -0,0 +1,126 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiclient + +import ( + "bytes" + "encoding/json" + "testing" + + rbac "k8s.io/api/rbac/v1beta1" + "k8s.io/apimachinery/pkg/runtime/schema" + core "k8s.io/client-go/testing" +) + +func TestHandleGetAction(t *testing.T) { + masterName := "master-foo" + serviceSubnet := "10.96.0.1/12" + idr := NewInitDryRunGetter(masterName, serviceSubnet) + + var tests = []struct { + action core.GetActionImpl + expectedHandled bool + expectedObjectJSON []byte + expectedErr bool + }{ + { + action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "default", "kubernetes"), + expectedHandled: true, + expectedObjectJSON: []byte(`{"metadata":{"name":"kubernetes","namespace":"default","creationTimestamp":null,"labels":{"component":"apiserver","provider":"kubernetes"}},"spec":{"ports":[{"name":"https","port":443,"targetPort":6443}],"clusterIP":"10.96.0.1"},"status":{"loadBalancer":{}}}`), + expectedErr: false, + }, + { + action: core.NewRootGetAction(schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, masterName), + expectedHandled: true, + expectedObjectJSON: []byte(`{"metadata":{"name":"master-foo","creationTimestamp":null,"labels":{"kubernetes.io/hostname":"master-foo"}},"spec":{"externalID":"master-foo"},"status":{"daemonEndpoints":{"kubeletEndpoint":{"Port":0}},"nodeInfo":{"machineID":"","systemUUID":"","bootID":"","kernelVersion":"","osImage":"","containerRuntimeVersion":"","kubeletVersion":"","kubeProxyVersion":"","operatingSystem":"","architecture":""}}}`), + expectedErr: false, + }, + { + action: core.NewRootGetAction(schema.GroupVersionResource{Group: rbac.GroupName, Version: rbac.SchemeGroupVersion.Version, Resource: "clusterrolebindings"}, "system:node"), + expectedHandled: true, + expectedObjectJSON: []byte(``), + expectedErr: true, // we expect a NotFound error here + }, + { + action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, "kube-system", "bootstrap-token-abcdef"), + expectedHandled: true, + expectedObjectJSON: []byte(``), + expectedErr: true, // we expect a NotFound error here + }, + { // an ask for a kubernetes service in the _kube-system_ ns should not be answered + action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "kube-system", "kubernetes"), + expectedHandled: false, + expectedObjectJSON: []byte(``), + expectedErr: false, + }, + { // an ask for an other service than kubernetes should not be answered + action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "default", "my-other-service"), + expectedHandled: false, + expectedObjectJSON: []byte(``), + expectedErr: false, + }, + { // an ask for an other node than the master should not be answered + action: core.NewRootGetAction(schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, "other-node"), + expectedHandled: false, + expectedObjectJSON: []byte(``), + expectedErr: false, + }, + { // an ask for a secret in any other ns than kube-system should not be answered + action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, "default", "bootstrap-token-abcdef"), + expectedHandled: false, + expectedObjectJSON: []byte(``), + expectedErr: false, + }, + } + for _, rt := range tests { + handled, obj, actualErr := idr.HandleGetAction(rt.action) + objBytes := []byte(``) + if obj != nil { + var err error + objBytes, err = json.Marshal(obj) + if err != nil { + t.Fatalf("couldn't marshal returned object") + } + } + + if handled != rt.expectedHandled { + t.Errorf( + "failed HandleGetAction:\n\texpected handled: %t\n\t actual: %t %v", + rt.expectedHandled, + handled, + rt.action, + ) + } + + if !bytes.Equal(objBytes, rt.expectedObjectJSON) { + t.Errorf( + "failed HandleGetAction:\n\texpected object: %q\n\t actual: %q", + rt.expectedObjectJSON, + objBytes, + ) + } + + if (actualErr != nil) != rt.expectedErr { + t.Errorf( + "failed HandleGetAction:\n\texpected error: %t\n\t actual: %t %v", + rt.expectedErr, + (actualErr != nil), + rt.action, + ) + } + } +} diff --git a/cmd/kubeadm/app/util/apiclient/wait.go b/cmd/kubeadm/app/util/apiclient/wait.go index 93629a8caaf8b..59eb1599f7a2e 100644 --- a/cmd/kubeadm/app/util/apiclient/wait.go +++ b/cmd/kubeadm/app/util/apiclient/wait.go @@ -30,12 +30,40 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) +// Waiter is an interface for waiting for criterias in Kubernetes to happen +type Waiter interface { + // WaitForAPI waits for the API Server's /healthz endpoint to become "ok" + WaitForAPI() error + // WaitForPodsWithLabel waits for Pods in the kube-system namespace to become Ready + WaitForPodsWithLabel(kvLabel string) error + // WaitForPodToDisappear waits for the given Pod in the kube-system namespace to be deleted + WaitForPodToDisappear(staticPodName string) error + // SetTimeout adjusts the timeout to the specified duration + SetTimeout(timeout time.Duration) +} + +// KubeWaiter is an implementation of Waiter that is backed by a Kubernetes client +type KubeWaiter struct { + client clientset.Interface + timeout time.Duration + writer io.Writer +} + +// NewKubeWaiter returns a new Waiter object that talks to the given Kubernetes cluster +func NewKubeWaiter(client clientset.Interface, timeout time.Duration, writer io.Writer) Waiter { + return &KubeWaiter{ + client: client, + timeout: timeout, + writer: writer, + } +} + // WaitForAPI waits for the API Server's /healthz endpoint to report "ok" -func WaitForAPI(client clientset.Interface, timeout time.Duration) error { +func (w *KubeWaiter) WaitForAPI() error { start := time.Now() - return wait.PollImmediate(constants.APICallRetryInterval, timeout, func() (bool, error) { + return wait.PollImmediate(constants.APICallRetryInterval, w.timeout, func() (bool, error) { healthStatus := 0 - client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus) + w.client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus) if healthStatus != http.StatusOK { return false, nil } @@ -47,19 +75,19 @@ func WaitForAPI(client clientset.Interface, timeout time.Duration) error { // WaitForPodsWithLabel will lookup pods with the given label and wait until they are all // reporting status as running. -func WaitForPodsWithLabel(client clientset.Interface, timeout time.Duration, out io.Writer, labelKeyValPair string) error { +func (w *KubeWaiter) WaitForPodsWithLabel(kvLabel string) error { lastKnownPodNumber := -1 - return wait.PollImmediate(constants.APICallRetryInterval, timeout, func() (bool, error) { - listOpts := metav1.ListOptions{LabelSelector: labelKeyValPair} - pods, err := client.CoreV1().Pods(metav1.NamespaceSystem).List(listOpts) + return wait.PollImmediate(constants.APICallRetryInterval, w.timeout, func() (bool, error) { + listOpts := metav1.ListOptions{LabelSelector: kvLabel} + pods, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).List(listOpts) if err != nil { - fmt.Fprintf(out, "[apiclient] Error getting Pods with label selector %q [%v]\n", labelKeyValPair, err) + fmt.Fprintf(w.writer, "[apiclient] Error getting Pods with label selector %q [%v]\n", kvLabel, err) return false, nil } if lastKnownPodNumber != len(pods.Items) { - fmt.Fprintf(out, "[apiclient] Found %d Pods for label selector %s\n", len(pods.Items), labelKeyValPair) + fmt.Fprintf(w.writer, "[apiclient] Found %d Pods for label selector %s\n", len(pods.Items), kvLabel) lastKnownPodNumber = len(pods.Items) } @@ -77,10 +105,10 @@ func WaitForPodsWithLabel(client clientset.Interface, timeout time.Duration, out }) } -// WaitForStaticPodToDisappear blocks until it timeouts or gets a "NotFound" response from the API Server when getting the Static Pod in question -func WaitForStaticPodToDisappear(client clientset.Interface, timeout time.Duration, podName string) error { - return wait.PollImmediate(constants.APICallRetryInterval, timeout, func() (bool, error) { - _, err := client.CoreV1().Pods(metav1.NamespaceSystem).Get(podName, metav1.GetOptions{}) +// WaitForPodToDisappear blocks until it timeouts or gets a "NotFound" response from the API Server when getting the Static Pod in question +func (w *KubeWaiter) WaitForPodToDisappear(podName string) error { + return wait.PollImmediate(constants.APICallRetryInterval, w.timeout, func() (bool, error) { + _, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).Get(podName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { fmt.Printf("[apiclient] The Static Pod %q is now removed\n", podName) return true, nil @@ -88,3 +116,27 @@ func WaitForStaticPodToDisappear(client clientset.Interface, timeout time.Durati return false, nil }) } + +// SetTimeout adjusts the timeout to the specified duration +func (w *KubeWaiter) SetTimeout(timeout time.Duration) { + w.timeout = timeout +} + +// TryRunCommand runs a function a maximum of failureThreshold times, and retries on error. If failureThreshold is hit; the last error is returned +func TryRunCommand(f func() error, failureThreshold uint8) error { + var numFailures uint8 + return wait.PollImmediate(5*time.Second, 20*time.Minute, func() (bool, error) { + err := f() + if err != nil { + numFailures++ + // If we've reached the maximum amount of failures, error out + if numFailures == failureThreshold { + return false, err + } + // Retry + return false, nil + } + // The last f() call was a success! + return true, nil + }) +} diff --git a/cmd/kubeadm/app/util/arguments.go b/cmd/kubeadm/app/util/arguments.go new file mode 100644 index 0000000000000..323b2da944cce --- /dev/null +++ b/cmd/kubeadm/app/util/arguments.go @@ -0,0 +1,96 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "strings" +) + +// BuildArgumentListFromMap takes two string-string maps, one with the base arguments and one with optional override arguments +func BuildArgumentListFromMap(baseArguments map[string]string, overrideArguments map[string]string) []string { + var command []string + for k, v := range overrideArguments { + // values of "" are allowed as well + command = append(command, fmt.Sprintf("--%s=%s", k, v)) + } + for k, v := range baseArguments { + if _, overrideExists := overrideArguments[k]; !overrideExists { + command = append(command, fmt.Sprintf("--%s=%s", k, v)) + } + } + return command +} + +// ParseArgumentListToMap parses a CLI argument list in the form "--foo=bar" to a string-string map +func ParseArgumentListToMap(arguments []string) map[string]string { + resultingMap := map[string]string{} + for i, arg := range arguments { + key, val, err := parseArgument(arg) + + // Ignore if the first argument doesn't satisfy the criteria, it's most often the binary name + // Warn in all other cases, but don't error out. This can happen only if the user has edited the argument list by hand, so they might know what they are doing + if err != nil { + if i != 0 { + fmt.Printf("[kubeadm] WARNING: The component argument %q could not be parsed correctly. The argument must be of the form %q. Skipping...", arg, "--") + } + continue + } + + resultingMap[key] = val + } + return resultingMap +} + +// ReplaceArgument gets a command list; converts it to a map for easier modification, runs the provided function that +// returns a new modified map, and then converts the map back to a command string slice +func ReplaceArgument(command []string, argMutateFunc func(map[string]string) map[string]string) []string { + argMap := ParseArgumentListToMap(command) + + // Save the first command (the executable) if we're sure it's not an argument (i.e. no --) + var newCommand []string + if len(command) > 0 && !strings.HasPrefix(command[0], "--") { + newCommand = append(newCommand, command[0]) + } + newArgMap := argMutateFunc(argMap) + newCommand = append(newCommand, BuildArgumentListFromMap(newArgMap, map[string]string{})...) + return newCommand +} + +// parseArgument parses the argument "--foo=bar" to "foo" and "bar" +func parseArgument(arg string) (string, string, error) { + if !strings.HasPrefix(arg, "--") { + return "", "", fmt.Errorf("the argument should start with '--'") + } + if !strings.Contains(arg, "=") { + return "", "", fmt.Errorf("the argument should have a '=' between the flag and the value") + } + // Remove the starting -- + arg = strings.TrimPrefix(arg, "--") + // Split the string on =. Return only two substrings, since we want only key/value, but the value can include '=' as well + keyvalSlice := strings.SplitN(arg, "=", 2) + + // Make sure both a key and value is present + if len(keyvalSlice) != 2 { + return "", "", fmt.Errorf("the argument must have both a key and a value") + } + if len(keyvalSlice[0]) == 0 { + return "", "", fmt.Errorf("the argument must have a key") + } + + return keyvalSlice[0], keyvalSlice[1], nil +} diff --git a/cmd/kubeadm/app/util/arguments_test.go b/cmd/kubeadm/app/util/arguments_test.go new file mode 100644 index 0000000000000..f2c6373bf8151 --- /dev/null +++ b/cmd/kubeadm/app/util/arguments_test.go @@ -0,0 +1,341 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "reflect" + "sort" + "testing" +) + +func TestBuildArgumentListFromMap(t *testing.T) { + var tests = []struct { + base map[string]string + overrides map[string]string + expected []string + }{ + { // override an argument from the base + base: map[string]string{ + "admission-control": "NamespaceLifecycle", + "insecure-bind-address": "127.0.0.1", + "allow-privileged": "true", + }, + overrides: map[string]string{ + "admission-control": "NamespaceLifecycle,LimitRanger", + }, + expected: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + }, + }, + { // add an argument that is not in base + base: map[string]string{ + "insecure-bind-address": "127.0.0.1", + "allow-privileged": "true", + }, + overrides: map[string]string{ + "admission-control": "NamespaceLifecycle,LimitRanger", + }, + expected: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + }, + }, + { // allow empty strings in base + base: map[string]string{ + "insecure-bind-address": "127.0.0.1", + "allow-privileged": "true", + "something-that-allows-empty-string": "", + }, + overrides: map[string]string{ + "admission-control": "NamespaceLifecycle,LimitRanger", + }, + expected: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + "--something-that-allows-empty-string=", + }, + }, + { // allow empty strings in overrides + base: map[string]string{ + "insecure-bind-address": "127.0.0.1", + "allow-privileged": "true", + "something-that-allows-empty-string": "foo", + }, + overrides: map[string]string{ + "admission-control": "NamespaceLifecycle,LimitRanger", + "something-that-allows-empty-string": "", + }, + expected: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + "--something-that-allows-empty-string=", + }, + }, + } + + for _, rt := range tests { + actual := BuildArgumentListFromMap(rt.base, rt.overrides) + sort.Strings(actual) + sort.Strings(rt.expected) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf("failed BuildArgumentListFromMap:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) + } + } +} + +func TestParseArgumentListToMap(t *testing.T) { + var tests = []struct { + args []string + expectedMap map[string]string + }{ + { + // normal case + args: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + }, + expectedMap: map[string]string{ + "admission-control": "NamespaceLifecycle,LimitRanger", + "insecure-bind-address": "127.0.0.1", + "allow-privileged": "true", + }, + }, + { + // test that feature-gates is working + args: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + "--feature-gates=EnableFoo=true,EnableBar=false", + }, + expectedMap: map[string]string{ + "admission-control": "NamespaceLifecycle,LimitRanger", + "insecure-bind-address": "127.0.0.1", + "allow-privileged": "true", + "feature-gates": "EnableFoo=true,EnableBar=false", + }, + }, + { + // test that a binary can be the first arg + args: []string{ + "kube-apiserver", + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + "--feature-gates=EnableFoo=true,EnableBar=false", + }, + expectedMap: map[string]string{ + "admission-control": "NamespaceLifecycle,LimitRanger", + "insecure-bind-address": "127.0.0.1", + "allow-privileged": "true", + "feature-gates": "EnableFoo=true,EnableBar=false", + }, + }, + } + + for _, rt := range tests { + actualMap := ParseArgumentListToMap(rt.args) + if !reflect.DeepEqual(actualMap, rt.expectedMap) { + t.Errorf("failed ParseArgumentListToMap:\nexpected:\n%v\nsaw:\n%v", rt.expectedMap, actualMap) + } + } +} + +func TestReplaceArgument(t *testing.T) { + var tests = []struct { + args []string + mutateFunc func(map[string]string) map[string]string + expectedArgs []string + }{ + { + // normal case + args: []string{ + "kube-apiserver", + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + }, + mutateFunc: func(argMap map[string]string) map[string]string { + argMap["admission-control"] = "NamespaceLifecycle,LimitRanger,ResourceQuota" + return argMap + }, + expectedArgs: []string{ + "kube-apiserver", + "--admission-control=NamespaceLifecycle,LimitRanger,ResourceQuota", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + }, + }, + { + // normal case + args: []string{ + "kube-apiserver", + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + }, + mutateFunc: func(argMap map[string]string) map[string]string { + argMap["new-arg-here"] = "foo" + return argMap + }, + expectedArgs: []string{ + "kube-apiserver", + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + "--new-arg-here=foo", + }, + }, + } + + for _, rt := range tests { + actualArgs := ReplaceArgument(rt.args, rt.mutateFunc) + sort.Strings(actualArgs) + sort.Strings(rt.expectedArgs) + if !reflect.DeepEqual(actualArgs, rt.expectedArgs) { + t.Errorf("failed ReplaceArgument:\nexpected:\n%v\nsaw:\n%v", rt.expectedArgs, actualArgs) + } + } +} + +func TestRoundtrip(t *testing.T) { + var tests = []struct { + args []string + }{ + { + // normal case + args: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + }, + }, + { + // test that feature-gates is working + args: []string{ + "--admission-control=NamespaceLifecycle,LimitRanger", + "--insecure-bind-address=127.0.0.1", + "--allow-privileged=true", + "--feature-gates=EnableFoo=true,EnableBar=false", + }, + }, + } + + for _, rt := range tests { + // These two methods should be each other's opposite functions, test that by chaining the methods and see if you get the same result back + actual := BuildArgumentListFromMap(ParseArgumentListToMap(rt.args), map[string]string{}) + sort.Strings(actual) + sort.Strings(rt.args) + + if !reflect.DeepEqual(actual, rt.args) { + t.Errorf("failed TestRoundtrip:\nexpected:\n%v\nsaw:\n%v", rt.args, actual) + } + } +} + +func TestParseArgument(t *testing.T) { + var tests = []struct { + arg string + expectedKey string + expectedVal string + expectedErr bool + }{ + { + // cannot be empty + arg: "", + expectedErr: true, + }, + { + // must contain -- and = + arg: "a", + expectedErr: true, + }, + { + // must contain -- and = + arg: "a-z", + expectedErr: true, + }, + { + // must contain -- + arg: "a=b", + expectedErr: true, + }, + { + // must contain a key + arg: "--=b", + expectedErr: true, + }, + { + // can contain key but no value + arg: "--a=", + expectedKey: "a", + expectedVal: "", + expectedErr: false, + }, + { + // simple case + arg: "--a=b", + expectedKey: "a", + expectedVal: "b", + expectedErr: false, + }, + { + // keys/values with '-' should be supported + arg: "--very-long-flag-name=some-value", + expectedKey: "very-long-flag-name", + expectedVal: "some-value", + expectedErr: false, + }, + { + // numbers should be handled correctly + arg: "--some-number=0.2", + expectedKey: "some-number", + expectedVal: "0.2", + expectedErr: false, + }, + { + // lists should be handled correctly + arg: "--admission-control=foo,bar,baz", + expectedKey: "admission-control", + expectedVal: "foo,bar,baz", + expectedErr: false, + }, + { + // more than one '=' should be allowed + arg: "--feature-gates=EnableFoo=true,EnableBar=false", + expectedKey: "feature-gates", + expectedVal: "EnableFoo=true,EnableBar=false", + expectedErr: false, + }, + } + + for _, rt := range tests { + key, val, actual := parseArgument(rt.arg) + if (actual != nil) != rt.expectedErr { + t.Errorf("failed parseArgument:\nexpected error:\n%t\nsaw error:\n%v", rt.expectedErr, actual) + } + if (key != rt.expectedKey) || (val != rt.expectedVal) { + t.Errorf("failed parseArgument:\nexpected key: %s\nsaw key: %s\nexpected value: %s\nsaw value: %s", rt.expectedKey, key, rt.expectedVal, val) + } + } +} diff --git a/cmd/kubeadm/app/util/config/masterconfig.go b/cmd/kubeadm/app/util/config/masterconfig.go index 8cb6aaf018c09..7a2bb0d94bb07 100644 --- a/cmd/kubeadm/app/util/config/masterconfig.go +++ b/cmd/kubeadm/app/util/config/masterconfig.go @@ -44,6 +44,11 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error { } cfg.API.AdvertiseAddress = ip.String() + // Requested version is automatic CI build, thus use KubernetesCI Image Repository for core images + if kubeadmutil.KubernetesIsCIVersion(cfg.KubernetesVersion) { + cfg.CIImageRepository = kubeadmconstants.DefaultCIImageRepository + } + // Validate version argument ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion) if err != nil { diff --git a/cmd/kubeadm/app/util/dryrun/BUILD b/cmd/kubeadm/app/util/dryrun/BUILD new file mode 100644 index 0000000000000..b175152e7b68b --- /dev/null +++ b/cmd/kubeadm/app/util/dryrun/BUILD @@ -0,0 +1,26 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["dryrun.go"], + visibility = ["//visibility:public"], + deps = [ + "//cmd/kubeadm/app/util/apiclient:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kubeadm/app/util/dryrun/dryrun.go b/cmd/kubeadm/app/util/dryrun/dryrun.go new file mode 100644 index 0000000000000..dbba795d9ebfe --- /dev/null +++ b/cmd/kubeadm/app/util/dryrun/dryrun.go @@ -0,0 +1,100 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dryrun + +import ( + "fmt" + "io" + "io/ioutil" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" +) + +// FileToPrint represents a temporary file on disk that might want to be aliased when printing +// Useful for things like loading a file from /tmp/ but saying to the user "Would write file foo to /etc/kubernetes/..." +type FileToPrint struct { + RealPath string + PrintPath string +} + +// NewFileToPrint makes a new instance of FileToPrint with the specified arguments +func NewFileToPrint(realPath, printPath string) FileToPrint { + return FileToPrint{ + RealPath: realPath, + PrintPath: printPath, + } +} + +// PrintDryRunFiles prints the contents of the FileToPrints given to it to the writer w +func PrintDryRunFiles(files []FileToPrint, w io.Writer) error { + errs := []error{} + for _, file := range files { + if len(file.RealPath) == 0 { + continue + } + + fileBytes, err := ioutil.ReadFile(file.RealPath) + if err != nil { + errs = append(errs, err) + continue + } + + // Make it possible to fake the path of the file; i.e. you may want to tell the user + // "Here is what would be written to /etc/kubernetes/admin.conf", although you wrote it to /tmp/kubeadm-dryrun/admin.conf and are loading it from there + // Fall back to the "real" path if PrintPath is not set + outputFilePath := file.PrintPath + if len(outputFilePath) == 0 { + outputFilePath = file.RealPath + } + + fmt.Fprintf(w, "[dryrun] Would write file %q with content:\n", outputFilePath) + apiclient.PrintBytesWithLinePrefix(w, fileBytes, "\t") + } + return errors.NewAggregate(errs) +} + +// Waiter is an implementation of apiclient.Waiter that should be used for dry-running +type Waiter struct{} + +// NewWaiter returns a new Waiter object that talks to the given Kubernetes cluster +func NewWaiter() apiclient.Waiter { + return &Waiter{} +} + +// WaitForAPI just returns a dummy nil, to indicate that the program should just proceed +func (w *Waiter) WaitForAPI() error { + fmt.Println("[dryrun] Would wait for the API Server's /healthz endpoint to return 'ok'") + return nil +} + +// WaitForPodsWithLabel just returns a dummy nil, to indicate that the program should just proceed +func (w *Waiter) WaitForPodsWithLabel(kvLabel string) error { + fmt.Printf("[dryrun] Would wait for the Pods with the label %q in the %s namespace to become Running\n", kvLabel, metav1.NamespaceSystem) + return nil +} + +// WaitForPodToDisappear just returns a dummy nil, to indicate that the program should just proceed +func (w *Waiter) WaitForPodToDisappear(podName string) error { + fmt.Printf("[dryrun] Would wait for the %q Pod in the %s namespace to be deleted\n", podName, metav1.NamespaceSystem) + return nil +} + +// SetTimeout is a no-op; we don't wait in this implementation +func (w *Waiter) SetTimeout(_ time.Duration) {} diff --git a/cmd/kubeadm/app/util/endpoint.go b/cmd/kubeadm/app/util/endpoint.go new file mode 100644 index 0000000000000..d3e32dbeceaaa --- /dev/null +++ b/cmd/kubeadm/app/util/endpoint.go @@ -0,0 +1,51 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "net" + "strconv" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" +) + +// GetMasterEndpoint returns a properly formatted Master Endpoint +// or passes the error from GetMasterHostPort. +func GetMasterEndpoint(cfg *kubeadmapi.MasterConfiguration) (string, error) { + hostPort, err := GetMasterHostPort(cfg) + if err != nil { + return "", err + } + return fmt.Sprintf("https://%s", hostPort), nil +} + +// GetMasterHostPort returns a properly formatted Master IP/port pair or error +// if the IP address can not be parsed or port is outside the valid TCP range. +func GetMasterHostPort(cfg *kubeadmapi.MasterConfiguration) (string, error) { + masterIP := net.ParseIP(cfg.API.AdvertiseAddress) + if masterIP == nil { + return "", fmt.Errorf("error parsing address %s", cfg.API.AdvertiseAddress) + } + + if cfg.API.BindPort < 0 || cfg.API.BindPort > 65535 { + return "", fmt.Errorf("api server port must be between 0 and 65535") + } + + hostPort := net.JoinHostPort(masterIP.String(), strconv.Itoa(int(cfg.API.BindPort))) + return hostPort, nil +} diff --git a/cmd/kubeadm/app/util/endpoint_test.go b/cmd/kubeadm/app/util/endpoint_test.go new file mode 100644 index 0000000000000..f5bef19dbb5e4 --- /dev/null +++ b/cmd/kubeadm/app/util/endpoint_test.go @@ -0,0 +1,223 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" +) + +func TestGetMasterEndpoint(t *testing.T) { + var tests = []struct { + name string + cfg *kubeadmapi.MasterConfiguration + endpoint string + expected bool + }{ + { + name: "valid IPv4 endpoint", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: 1234, + }, + }, + endpoint: "https://1.2.3.4:1234", + expected: true, + }, + { + name: "valid IPv6 endpoint", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "2001:db8::1", + BindPort: 4321, + }, + }, + endpoint: "https://[2001:db8::1]:4321", + expected: true, + }, + { + name: "invalid IPv4 endpoint", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: 1234, + }, + }, + endpoint: "https://[1.2.3.4]:1234", + expected: false, + }, + { + name: "invalid IPv6 endpoint", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "2001:db8::1", + BindPort: 4321, + }, + }, + endpoint: "https://2001:db8::1:4321", + expected: false, + }, + { + name: "invalid IPv4 AdvertiseAddress", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1.2.34", + BindPort: 1234, + }, + }, + endpoint: "https://1.2.3.4:1234", + expected: false, + }, + { + name: "invalid IPv6 AdvertiseAddress", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "2001::db8::1", + BindPort: 4321, + }, + }, + endpoint: "https://[2001:db8::1]:4321", + expected: false, + }, + } + for _, rt := range tests { + actual, err := GetMasterEndpoint(rt.cfg) + if err != nil && rt.expected { + t.Error(err) + } + if actual != rt.endpoint && rt.expected { + t.Errorf( + "%s test case failed:\n\texpected: %s\n\t actual: %s", + rt.name, + rt.endpoint, + (actual), + ) + } + } +} + +func TestGetMasterHostPort(t *testing.T) { + var tests = []struct { + name string + cfg *kubeadmapi.MasterConfiguration + hostPort string + expected bool + }{ + { + name: "valid IPv4 master host and port", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: 1234, + }, + }, + hostPort: "1.2.3.4:1234", + expected: true, + }, + { + name: "valid IPv6 master host port", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "2001:db8::1", + BindPort: 4321, + }, + }, + hostPort: "[2001:db8::1]:4321", + expected: true, + }, + { + name: "invalid IPv4 address", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1.2.34", + BindPort: 1234, + }, + }, + hostPort: "1.2.3.4:1234", + expected: false, + }, + { + name: "invalid IPv6 address", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "2001::db8::1", + BindPort: 4321, + }, + }, + hostPort: "[2001:db8::1]:4321", + expected: false, + }, + { + name: "invalid TCP port number", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: 987654321, + }, + }, + hostPort: "1.2.3.4:987654321", + expected: false, + }, + { + name: "invalid negative TCP port number", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1.2.3.4", + BindPort: -987654321, + }, + }, + hostPort: "1.2.3.4:-987654321", + expected: false, + }, + { + name: "unspecified IPv4 TCP port", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1.2.3.4", + }, + }, + hostPort: "1.2.3.4:0", + expected: true, + }, + { + name: "unspecified IPv6 TCP port", + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadmapi.API{ + AdvertiseAddress: "1:2:3::4", + }, + }, + hostPort: "[1:2:3::4]:0", + expected: true, + }, + } + for _, rt := range tests { + actual, err := GetMasterHostPort(rt.cfg) + if err != nil && rt.expected { + t.Error(err) + } + if actual != rt.hostPort && rt.expected { + t.Errorf( + "%s test case failed:\n\texpected: %s\n\t actual: %s", + rt.name, + rt.hostPort, + (actual), + ) + } + } +} diff --git a/cmd/kubeadm/app/util/staticpod/BUILD b/cmd/kubeadm/app/util/staticpod/BUILD index 51c976f2d9fcc..c6d7379b2f237 100644 --- a/cmd/kubeadm/app/util/staticpod/BUILD +++ b/cmd/kubeadm/app/util/staticpod/BUILD @@ -15,6 +15,7 @@ go_test( tags = ["automanaged"], deps = [ "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", ], ) diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index bdeb09f657a31..9b40ea66642f7 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -42,6 +42,9 @@ func ComponentPod(container v1.Container, volumes []v1.Volume) v1.Pod { Name: container.Name, Namespace: metav1.NamespaceSystem, Annotations: map[string]string{kubetypes.CriticalPodAnnotationKey: ""}, + // The component and tier labels are useful for quickly identifying the control plane Pods when doing a .List() + // against Pods in the kube-system namespace. Can for example be used together with the WaitForPodsWithLabel function + Labels: map[string]string{"component": container.Name, "tier": "control-plane"}, }, Spec: v1.PodSpec{ Containers: []v1.Container{container}, @@ -79,11 +82,14 @@ func ComponentProbe(port int, path string, scheme v1.URIScheme) *v1.Probe { } // NewVolume creates a v1.Volume with a hostPath mount to the specified location -func NewVolume(name, path string) v1.Volume { +func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { return v1.Volume{ Name: name, VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: path}, + HostPath: &v1.HostPathVolumeSource{ + Path: path, + Type: pathType, + }, }, } } diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go index e0e844d8a9afb..9d7707f961ecd 100644 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ b/cmd/kubeadm/app/util/staticpod/utils_test.go @@ -22,6 +22,7 @@ import ( "testing" "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -79,32 +80,55 @@ func TestComponentProbe(t *testing.T) { func TestComponentPod(t *testing.T) { var tests = []struct { - n string + name string + expected v1.Pod }{ { - n: "foo", + name: "foo", + expected: v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "kube-system", + Annotations: map[string]string{"scheduler.alpha.kubernetes.io/critical-pod": ""}, + Labels: map[string]string{"component": "foo", "tier": "control-plane"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + }, + }, + HostNetwork: true, + Volumes: []v1.Volume{}, + }, + }, }, } for _, rt := range tests { - c := v1.Container{Name: rt.n} - v := []v1.Volume{} - actual := ComponentPod(c, v) - if actual.ObjectMeta.Name != rt.n { + c := v1.Container{Name: rt.name} + actual := ComponentPod(c, []v1.Volume{}) + if !reflect.DeepEqual(rt.expected, actual) { t.Errorf( - "failed componentPod:\n\texpected: %s\n\t actual: %s", - rt.n, - actual.ObjectMeta.Name, + "failed componentPod:\n\texpected: %v\n\t actual: %v", + rt.expected, + actual, ) } } } func TestNewVolume(t *testing.T) { + hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate var tests = []struct { name string path string expected v1.Volume + pathType *v1.HostPathType }{ { name: "foo", @@ -112,14 +136,18 @@ func TestNewVolume(t *testing.T) { expected: v1.Volume{ Name: "foo", VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/foo"}, + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/foo", + Type: &hostPathDirectoryOrCreate, + }, }, }, + pathType: &hostPathDirectoryOrCreate, }, } for _, rt := range tests { - actual := NewVolume(rt.name, rt.path) + actual := NewVolume(rt.name, rt.path, rt.pathType) if !reflect.DeepEqual(actual, rt.expected) { t.Errorf( "failed newVolume:\n\texpected: %v\n\t actual: %v", diff --git a/cmd/kubeadm/app/util/version.go b/cmd/kubeadm/app/util/version.go index ae442701aa05a..81d521c26169d 100644 --- a/cmd/kubeadm/app/util/version.go +++ b/cmd/kubeadm/app/util/version.go @@ -25,9 +25,10 @@ import ( ) var ( - kubeReleaseBucketURL = "https://storage.googleapis.com/kubernetes-release/release" + kubeReleaseBucketURL = "https://dl.k8s.io" kubeReleaseRegex = regexp.MustCompile(`^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)([-0-9a-zA-Z_\.+]*)?$`) kubeReleaseLabelRegex = regexp.MustCompile(`^[[:lower:]]+(-[-\w_\.]+)?$`) + kubeBucketPrefixes = regexp.MustCompile(`^((release|ci|ci-cross)/)?([-\w_\.+]+)$`) ) // KubernetesReleaseVersion is helper function that can fetch @@ -53,22 +54,20 @@ func KubernetesReleaseVersion(version string) (string, error) { return version, nil } return "v" + version, nil - } else if kubeReleaseLabelRegex.MatchString(version) { - url := fmt.Sprintf("%s/%s.txt", kubeReleaseBucketURL, version) - resp, err := http.Get(url) - if err != nil { - return "", fmt.Errorf("unable to get URL %q: %s", url, err.Error()) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("unable to fetch release information. URL: %q Status: %v", url, resp.Status) - } - body, err := ioutil.ReadAll(resp.Body) + } + + bucketURL, versionLabel, err := splitVersion(version) + if err != nil { + return "", err + } + if kubeReleaseLabelRegex.MatchString(versionLabel) { + url := fmt.Sprintf("%s/%s.txt", bucketURL, versionLabel) + body, err := fetchFromURL(url) if err != nil { - return "", fmt.Errorf("unable to read content of URL %q: %s", url, err.Error()) + return "", err } // Re-validate received version and return. - return KubernetesReleaseVersion(strings.Trim(string(body), " \t\n")) + return KubernetesReleaseVersion(body) } return "", fmt.Errorf("version %q doesn't match patterns for neither semantic version nor labels (stable, latest, ...)", version) } @@ -83,3 +82,49 @@ func KubernetesVersionToImageTag(version string) string { allowed := regexp.MustCompile(`[^-a-zA-Z0-9_\.]`) return allowed.ReplaceAllString(version, "_") } + +// KubernetesIsCIVersion checks if user requested CI version +func KubernetesIsCIVersion(version string) bool { + subs := kubeBucketPrefixes.FindAllStringSubmatch(version, 1) + if len(subs) == 1 && len(subs[0]) == 4 && strings.HasPrefix(subs[0][2], "ci") { + return true + } + return false +} + +// Internal helper: split version parts, +// Return base URL and cleaned-up version +func splitVersion(version string) (string, string, error) { + var urlSuffix string + subs := kubeBucketPrefixes.FindAllStringSubmatch(version, 1) + if len(subs) != 1 || len(subs[0]) != 4 { + return "", "", fmt.Errorf("invalid version %q", version) + } + + switch { + case strings.HasPrefix(subs[0][2], "ci"): + // Special case. CI images populated only by ci-cross area + urlSuffix = "ci-cross" + default: + urlSuffix = "release" + } + url := fmt.Sprintf("%s/%s", kubeReleaseBucketURL, urlSuffix) + return url, subs[0][3], nil +} + +// Internal helper: return content of URL +func fetchFromURL(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("unable to get URL %q: %s", url, err.Error()) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unable to fetch file. URL: %q Status: %v", url, resp.Status) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("unable to read content of URL %q: %s", url, err.Error()) + } + return strings.TrimSpace(string(body)), nil +} diff --git a/cmd/kubeadm/app/util/version_test.go b/cmd/kubeadm/app/util/version_test.go index 491ea24344a46..a333031a5b112 100644 --- a/cmd/kubeadm/app/util/version_test.go +++ b/cmd/kubeadm/app/util/version_test.go @@ -45,7 +45,7 @@ func TestValidVersion(t *testing.T) { "v1.6.0-alpha.0.536+d60d9f3269288f", "v1.5.0-alpha.0.1078+1044b6822497da-pull", "v1.5.0-alpha.1.822+49b9e32fad9f32-pull-gke-gci", - "v1.6.1_coreos.0", + "v1.6.1+coreos.0", } for _, s := range validVersions { ver, err := KubernetesReleaseVersion(s) @@ -165,3 +165,70 @@ func TestVersionToTag(t *testing.T) { } } } + +func TestSplitVersion(t *testing.T) { + type T struct { + input string + bucket string + label string + valid bool + } + cases := []T{ + // Release area + {"v1.7.0", "https://dl.k8s.io/release", "v1.7.0", true}, + {"v1.8.0-alpha.2.1231+afabd012389d53a", "https://dl.k8s.io/release", "v1.8.0-alpha.2.1231+afabd012389d53a", true}, + {"release/v1.7.0", "https://dl.k8s.io/release", "v1.7.0", true}, + {"release/latest-1.7", "https://dl.k8s.io/release", "latest-1.7", true}, + // CI builds area, lookup actual builds at ci-cross/*.txt + {"ci-cross/latest", "https://dl.k8s.io/ci-cross", "latest", true}, + {"ci/latest-1.7", "https://dl.k8s.io/ci-cross", "latest-1.7", true}, + // unknown label in default (release) area: splitVersion validate only areas. + {"unknown-1", "https://dl.k8s.io/release", "unknown-1", true}, + // unknown area, not valid input. + {"unknown/latest-1", "", "", false}, + } + + // kubeReleaseBucketURL can be overriden during network tests, thus ensure + // it will contain value corresponding to expected outcome for this unit test + kubeReleaseBucketURL = "https://dl.k8s.io" + + for _, tc := range cases { + bucket, label, err := splitVersion(tc.input) + switch { + case err != nil && tc.valid: + t.Errorf("splitVersion: unexpected error for %q. Error: %v", tc.input, err) + case err == nil && !tc.valid: + t.Errorf("splitVersion: error expected for key %q, but result is %q, %q", tc.input, bucket, label) + case bucket != tc.bucket: + t.Errorf("splitVersion: unexpected bucket result for key %q. Expected: %q Actual: %q", tc.input, tc.bucket, bucket) + case label != tc.label: + t.Errorf("splitVersion: unexpected label result for key %q. Expected: %q Actual: %q", tc.input, tc.label, label) + } + + } +} + +func TestKubernetesIsCIVersion(t *testing.T) { + type T struct { + input string + expected bool + } + cases := []T{ + {"", false}, + // Official releases + {"v1.0.0", false}, + {"release/v1.0.0", false}, + // CI builds + {"ci/latest-1", true}, + {"ci-cross/latest", true}, + } + + for _, tc := range cases { + result := KubernetesIsCIVersion(tc.input) + t.Logf("KubernetesIsCIVersion: Input: %q. Result: %v. Expected: %v", tc.input, result, tc.expected) + if result != tc.expected { + t.Errorf("failed KubernetesIsCIVersion: Input: %q. Result: %v. Expected: %v", tc.input, result, tc.expected) + } + } + +} diff --git a/cmd/kubeadm/test/certs/util.go b/cmd/kubeadm/test/certs/util.go index 30ef1d303ae66..4f3235010cf03 100644 --- a/cmd/kubeadm/test/certs/util.go +++ b/cmd/kubeadm/test/certs/util.go @@ -19,6 +19,7 @@ package certs import ( "crypto/rsa" "crypto/x509" + "net" "testing" "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil" @@ -35,6 +36,13 @@ func SetupCertificateAuthorithy(t *testing.T) (*x509.Certificate, *rsa.PrivateKe return caCert, caKey } +// AssertCertificateIsCa is a utility function for kubeadm testing that asserts if a given certificate is a CA +func AssertCertificateIsCa(t *testing.T, cert *x509.Certificate) { + if !cert.IsCA { + t.Error("cert is not a valida CA") + } +} + // AssertCertificateIsSignedByCa is a utility function for kubeadm testing that asserts if a given certificate is signed // by the expected CA func AssertCertificateIsSignedByCa(t *testing.T, cert *x509.Certificate, signingCa *x509.Certificate) { @@ -77,3 +85,50 @@ func AssertCertificateHasClientAuthUsage(t *testing.T, cert *x509.Certificate) { } t.Error("cert has not ClientAuth usage as expected") } + +// AssertCertificateHasServerAuthUsage is a utility function for kubeadm testing that asserts if a given certificate has +// the expected ExtKeyUsageServerAuth +func AssertCertificateHasServerAuthUsage(t *testing.T, cert *x509.Certificate) { + for i := range cert.ExtKeyUsage { + if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth { + return + } + } + t.Error("cert is not a ServerAuth") +} + +// AssertCertificateHasDNSNames is a utility function for kubeadm testing that asserts if a given certificate has +// the expected DNSNames +func AssertCertificateHasDNSNames(t *testing.T, cert *x509.Certificate, DNSNames ...string) { + for _, DNSName := range DNSNames { + found := false + for _, val := range cert.DNSNames { + if val == DNSName { + found = true + break + } + } + + if !found { + t.Errorf("cert does not contain DNSName %s", DNSName) + } + } +} + +// AssertCertificateHasIPAddresses is a utility function for kubeadm testing that asserts if a given certificate has +// the expected IPAddresses +func AssertCertificateHasIPAddresses(t *testing.T, cert *x509.Certificate, IPAddresses ...net.IP) { + for _, IPAddress := range IPAddresses { + found := false + for _, val := range cert.IPAddresses { + if val.Equal(IPAddress) { + found = true + break + } + } + + if !found { + t.Errorf("cert does not contain IPAddress %s", IPAddress) + } + } +} diff --git a/cmd/kubeadm/test/util.go b/cmd/kubeadm/test/util.go index fe8ea38f781db..916e72bf512ea 100644 --- a/cmd/kubeadm/test/util.go +++ b/cmd/kubeadm/test/util.go @@ -76,6 +76,17 @@ func SetupMasterConfigurationFile(t *testing.T, tmpdir string, cfg *kubeadmapi.M return cfgPath } +// SetupEmptyFiles is a utility function for kubeadm testing that creates one or more empty files (touch) +func SetupEmptyFiles(t *testing.T, tmpdir string, fileNames ...string) { + for _, fileName := range fileNames { + newFile, err := os.Create(filepath.Join(tmpdir, fileName)) + if err != nil { + t.Fatalf("Error creating file %s in %s: %v", fileName, tmpdir, err) + } + newFile.Close() + } +} + // SetupPkiDirWithCertificateAuthorithy is a utility function for kubeadm testing that creates a // CertificateAuthorithy cert/key pair into /pki subfolder of a given temporary directory. // The funtion returns the path of the created pki. diff --git a/cmd/kubectl/BUILD b/cmd/kubectl/BUILD index da36e7fcb0f6f..a4c15004d4322 100644 --- a/cmd/kubectl/BUILD +++ b/cmd/kubectl/BUILD @@ -7,12 +7,17 @@ load("//pkg/version:def.bzl", "version_x_defs") go_binary( name = "kubectl", - gc_linkopts = [ - "-linkmode", - "external", - "-extldflags", - "-static", - ], + gc_linkopts = select({ + # Mac OS X doesn't support static binaries: + # https://developer.apple.com/library/content/qa/qa1118/_index.html + "@io_bazel_rules_go//go/platform:darwin_amd64": [], + "//conditions:default": [ + "-linkmode", + "external", + "-extldflags", + "-static", + ], + }), library = ":go_default_library", visibility = [ "//build/visible_to:COMMON_release", diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 71dd2850fe782..e0ef19961d368 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -41,6 +41,7 @@ go_library( "//pkg/features:go_default_library", "//pkg/kubelet:go_default_library", "//pkg/kubelet/apis/kubeletconfig:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", "//pkg/kubelet/certificate:go_default_library", diff --git a/cmd/kubelet/app/options/BUILD b/cmd/kubelet/app/options/BUILD index 2cb6e59bdb6e5..9b19054d13241 100644 --- a/cmd/kubelet/app/options/BUILD +++ b/cmd/kubelet/app/options/BUILD @@ -12,11 +12,10 @@ go_library( "options.go", ], deps = [ - "//pkg/api:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//pkg/features:go_default_library", "//pkg/kubelet/apis/kubeletconfig:go_default_library", - "//pkg/kubelet/apis/kubeletconfig/install:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", "//pkg/kubelet/apis/kubeletconfig/validation:go_default_library", "//pkg/util/taints:go_default_library", diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 118d9c620a51c..406ac6ea4260d 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -25,14 +25,12 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/flag" utilflag "k8s.io/apiserver/pkg/util/flag" - "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" - kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/validation" - // Need to make sure the kubeletconfig api is installed so defaulting funcs work - _ "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/install" + kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" + kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/validation" utiltaints "k8s.io/kubernetes/pkg/util/taints" "github.com/spf13/pflag" @@ -123,8 +121,12 @@ func NewKubeletFlags() *KubeletFlags { KubeConfig: flag.NewStringFlag("/var/lib/kubelet/kubeconfig"), ContainerRuntimeOptions: *NewContainerRuntimeOptions(), CertDirectory: "/var/run/kubernetes", - CloudProvider: v1alpha1.AutoDetectCloudProvider, RootDirectory: v1alpha1.DefaultRootDir, + // DEPRECATED: auto detecting cloud providers goes against the initiative + // for out-of-tree cloud providers as we'll now depend on cAdvisor integrations + // with cloud providers instead of in the core repo. + // More details here: https://github.com/kubernetes/kubernetes/issues/50986 + CloudProvider: v1alpha1.AutoDetectCloudProvider, } } @@ -142,10 +144,14 @@ func ValidateKubeletFlags(f *KubeletFlags) error { // NewKubeletConfiguration will create a new KubeletConfiguration with default values func NewKubeletConfiguration() (*kubeletconfig.KubeletConfiguration, error) { + scheme, _, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + return nil, err + } versioned := &v1alpha1.KubeletConfiguration{} - api.Scheme.Default(versioned) + scheme.Default(versioned) config := &kubeletconfig.KubeletConfiguration{} - if err := api.Scheme.Convert(versioned, config, nil); err != nil { + if err := scheme.Convert(versioned, config, nil); err != nil { return nil, err } return config, nil @@ -218,7 +224,7 @@ func (f *KubeletFlags) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&f.CertDirectory, "cert-dir", f.CertDirectory, "The directory where the TLS certs are located. "+ "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") - fs.StringVar(&f.CloudProvider, "cloud-provider", f.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider.") + fs.StringVar(&f.CloudProvider, "cloud-provider", f.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider (deprecated). Specify empty string for running with no cloud provider, this will be the default in upcoming releases.") fs.StringVar(&f.CloudConfigFile, "cloud-config", f.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") fs.StringVar(&f.RootDirectory, "root-dir", f.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).") @@ -290,8 +296,8 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat fs.MarkDeprecated("maximum-dead-containers-per-container", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.") fs.Int32Var(&c.MaxContainerCount, "maximum-dead-containers", c.MaxContainerCount, "Maximum number of old instances of containers to retain globally. Each container takes up some disk space. To disable, set to a negative number.") fs.MarkDeprecated("maximum-dead-containers", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.") - fs.Int32Var(&c.CAdvisorPort, "cadvisor-port", c.CAdvisorPort, "The port of the localhost cAdvisor endpoint") - fs.Int32Var(&c.HealthzPort, "healthz-port", c.HealthzPort, "The port of the localhost healthz endpoint") + fs.Int32Var(&c.CAdvisorPort, "cadvisor-port", c.CAdvisorPort, "The port of the localhost cAdvisor endpoint (set to 0 to disable)") + fs.Int32Var(&c.HealthzPort, "healthz-port", c.HealthzPort, "The port of the localhost healthz endpoint (set to 0 to disable)") fs.Var(componentconfig.IPVar{Val: &c.HealthzBindAddress}, "healthz-bind-address", "The IP address for the healthz server to serve on. (set to 0.0.0.0 for all interfaces)") fs.Int32Var(&c.OOMScoreAdj, "oom-score-adj", c.OOMScoreAdj, "The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]") fs.BoolVar(&c.RegisterNode, "register-node", c.RegisterNode, "Register the node with the apiserver. If --kubeconfig is not provided, this flag is irrelevant, as the Kubelet won't have an apiserver to register with. Default=true.") diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 852bd12bf0a66..13abcd9ff7396 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -61,9 +61,7 @@ import ( ) // ProbeVolumePlugins collects all volume plugins into an easy to use list. -// PluginDir specifies the directory to search for additional third party -// volume plugins. -func ProbeVolumePlugins(pluginDir string) []volume.VolumePlugin { +func ProbeVolumePlugins() []volume.VolumePlugin { allPlugins := []volume.VolumePlugin{} // The list of plugins to probe is decided by the kubelet binary, not @@ -88,7 +86,6 @@ func ProbeVolumePlugins(pluginDir string) []volume.VolumePlugin { allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...) allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, flexvolume.ProbeVolumePlugins(pluginDir)...) allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...) allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...) allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) @@ -102,6 +99,13 @@ func ProbeVolumePlugins(pluginDir string) []volume.VolumePlugin { return allPlugins } +// GetDynamicPluginProber gets the probers of dynamically discoverable plugins +// for kubelet. +// Currently only Flexvolume plugins are dynamically discoverable. +func GetDynamicPluginProber(pluginDir string) volume.DynamicPluginProber { + return flexvolume.GetDynamicPluginProber(pluginDir) +} + // ProbeNetworkPlugins collects all compiled-in plugins func ProbeNetworkPlugins(pluginDir, cniConfDir, cniBinDir string) []network.NetworkPlugin { allPlugins := []network.NetworkPlugin{} diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 69a2a5d8c1ff5..ddd653bcb6ae6 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -60,6 +60,7 @@ import ( "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet" kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/certificate" @@ -150,22 +151,22 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err } return &kubelet.Dependencies{ - Auth: nil, // default does not enforce auth[nz] - CAdvisorInterface: nil, // cadvisor.New launches background processes (bg http.ListenAndServe, and some bg cleaners), not set here - Cloud: nil, // cloud provider might start background processes - ContainerManager: nil, - DockerClient: dockerClient, - KubeClient: nil, - ExternalKubeClient: nil, - EventClient: nil, - Mounter: mounter, - NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir, s.CNIConfDir, s.CNIBinDir), - OOMAdjuster: oom.NewOOMAdjuster(), - OSInterface: kubecontainer.RealOS{}, - Writer: writer, - VolumePlugins: ProbeVolumePlugins(s.VolumePluginDir), - TLSOptions: tlsOptions, - }, nil + Auth: nil, // default does not enforce auth[nz] + CAdvisorInterface: nil, // cadvisor.New launches background processes (bg http.ListenAndServe, and some bg cleaners), not set here + Cloud: nil, // cloud provider might start background processes + ContainerManager: nil, + DockerClient: dockerClient, + KubeClient: nil, + ExternalKubeClient: nil, + EventClient: nil, + Mounter: mounter, + NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir, s.CNIConfDir, s.CNIBinDir), + OOMAdjuster: oom.NewOOMAdjuster(), + OSInterface: kubecontainer.RealOS{}, + Writer: writer, + VolumePlugins: ProbeVolumePlugins(), + DynamicPluginProber: GetDynamicPluginProber(s.VolumePluginDir), + TLSOptions: tlsOptions}, nil } // Run runs the specified KubeletServer with the given Dependencies. This should never exit. @@ -188,24 +189,34 @@ func checkPermissions() error { return nil } -func setConfigz(cz *configz.Config, kc *kubeletconfiginternal.KubeletConfiguration) { - tmp := kubeletconfigv1alpha1.KubeletConfiguration{} - api.Scheme.Convert(kc, &tmp, nil) - cz.Set(tmp) +func setConfigz(cz *configz.Config, kc *kubeletconfiginternal.KubeletConfiguration) error { + scheme, _, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + return err + } + versioned := kubeletconfigv1alpha1.KubeletConfiguration{} + if err := scheme.Convert(kc, &versioned, nil); err != nil { + return err + } + cz.Set(versioned) + return nil } -func initConfigz(kc *kubeletconfiginternal.KubeletConfiguration) (*configz.Config, error) { +func initConfigz(kc *kubeletconfiginternal.KubeletConfiguration) error { cz, err := configz.New("kubeletconfig") - if err == nil { - setConfigz(cz, kc) - } else { + if err != nil { glog.Errorf("unable to register configz: %s", err) + return err } - return cz, err + if err := setConfigz(cz, kc); err != nil { + glog.Errorf("unable to register config: %s", err) + return err + } + return nil } // makeEventRecorder sets up kubeDeps.Recorder if its nil. Its a no-op otherwise. -func makeEventRecorder(s *kubeletconfiginternal.KubeletConfiguration, kubeDeps *kubelet.Dependencies, nodeName types.NodeName) { +func makeEventRecorder(kubeDeps *kubelet.Dependencies, nodeName types.NodeName) { if kubeDeps.Recorder != nil { return } @@ -250,7 +261,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { } // Register current configuration with /configz endpoint - _, err = initConfigz(&s.KubeletConfiguration) + err = initConfigz(&s.KubeletConfiguration) if err != nil { glog.Errorf("unable to register KubeletConfiguration with configz, error: %v", err) } @@ -272,6 +283,10 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { } } + if s.CloudProvider == kubeletconfigv1alpha1.AutoDetectCloudProvider { + glog.Warning("--cloud-provider=auto-detect is deprecated. The desired cloud provider should be set explicitly") + } + if kubeDeps.Cloud == nil { if !cloudprovider.IsExternal(s.CloudProvider) && s.CloudProvider != kubeletconfigv1alpha1.AutoDetectCloudProvider { cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) @@ -384,7 +399,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { } // Setup event recorder if required. - makeEventRecorder(&s.KubeletConfiguration, kubeDeps, nodeName) + makeEventRecorder(kubeDeps, nodeName) if kubeDeps.ContainerManager == nil { if s.CgroupsPerQOS && s.CgroupRoot == "" { @@ -616,7 +631,7 @@ func RunKubelet(kubeFlags *options.KubeletFlags, kubeCfg *kubeletconfiginternal. return err } // Setup event recorder if required. - makeEventRecorder(kubeCfg, kubeDeps, nodeName) + makeEventRecorder(kubeDeps, nodeName) // TODO(mtaufen): I moved the validation of these fields here, from UnsecuredKubeletConfig, // so that I could remove the associated fields from KubeletConfiginternal. I would @@ -734,7 +749,7 @@ func parseResourceList(m kubeletconfiginternal.ConfigurationMap) (v1.ResourceLis for k, v := range m { switch v1.ResourceName(k) { // CPU, memory and local storage resources are supported. - case v1.ResourceCPU, v1.ResourceMemory, v1.ResourceStorage: + case v1.ResourceCPU, v1.ResourceMemory, v1.ResourceEphemeralStorage: q, err := resource.ParseQuantity(v) if err != nil { return nil, err @@ -742,12 +757,7 @@ func parseResourceList(m kubeletconfiginternal.ConfigurationMap) (v1.ResourceLis if q.Sign() == -1 { return nil, fmt.Errorf("resource quantity for %q cannot be negative: %v", k, v) } - // storage specified in configuration map is mapped to ResourceStorageScratch API - if v1.ResourceName(k) == v1.ResourceStorage { - rl[v1.ResourceStorageScratch] = q - } else { - rl[v1.ResourceName(k)] = q - } + rl[v1.ResourceName(k)] = q default: return nil, fmt.Errorf("cannot reserve %q resource", k) } @@ -756,7 +766,8 @@ func parseResourceList(m kubeletconfiginternal.ConfigurationMap) (v1.ResourceLis } // BootstrapKubeletConfigController constructs and bootstrap a configuration controller -func BootstrapKubeletConfigController(flags *options.KubeletFlags, +func BootstrapKubeletConfigController( + flags *options.KubeletFlags, defaultConfig *kubeletconfiginternal.KubeletConfiguration) (*kubeletconfiginternal.KubeletConfiguration, *kubeletconfig.Controller, error) { var err error // Alpha Dynamic Configuration Implementation; this section only loads config from disk, it does not contact the API server @@ -777,7 +788,10 @@ func BootstrapKubeletConfigController(flags *options.KubeletFlags, } // get the latest KubeletConfiguration checkpoint from disk, or load the init or default config if no valid checkpoints exist - kubeletConfigController := kubeletconfig.NewController(initConfigDir, dynamicConfigDir, defaultConfig) + kubeletConfigController, err := kubeletconfig.NewController(initConfigDir, dynamicConfigDir, defaultConfig) + if err != nil { + return nil, nil, fmt.Errorf("failed to construct controller, error: %v", err) + } kubeletConfig, err := kubeletConfigController.Bootstrap() if err != nil { return nil, nil, fmt.Errorf("failed to determine a valid configuration, error: %v", err) diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go index d26d213e5eb7f..989c33361f869 100644 --- a/cmd/kubelet/kubelet.go +++ b/cmd/kubelet/kubelet.go @@ -89,10 +89,16 @@ func main() { } // construct a KubeletServer from kubeletFlags and kubeletConfig - kubeletServer := &options.KubeletServer{KubeletFlags: *kubeletFlags, KubeletConfiguration: *kubeletConfig} + kubeletServer := &options.KubeletServer{ + KubeletFlags: *kubeletFlags, + KubeletConfiguration: *kubeletConfig, + } // use kubeletServer to construct the default KubeletDeps kubeletDeps, err := app.UnsecuredDependencies(kubeletServer) + if err != nil { + die(err) + } // add the kubelet config controller to kubeletDeps kubeletDeps.KubeletConfigController = kubeletConfigController diff --git a/cmd/mungedocs/BUILD b/cmd/mungedocs/BUILD deleted file mode 100644 index 1cc809f4159c9..0000000000000 --- a/cmd/mungedocs/BUILD +++ /dev/null @@ -1,70 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", - "go_test", -) - -go_binary( - name = "mungedocs", - library = ":go_default_library", -) - -go_test( - name = "go_default_test", - srcs = [ - "analytics_test.go", - "example_syncer_test.go", - "headers_test.go", - "kubectl_dash_f_test.go", - "links_test.go", - "preformatted_test.go", - "toc_test.go", - "util_test.go", - "whitespace_test.go", - ], - data = [ - "mungedocs.go", - ":testdata", - "//docs:srcs", - ], - library = ":go_default_library", - deps = ["//vendor/github.com/stretchr/testify/assert:go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = [ - "analytics.go", - "example_syncer.go", - "headers.go", - "kubectl_dash_f.go", - "links.go", - "mungedocs.go", - "preformatted.go", - "toc.go", - "util.go", - "whitespace.go", - ], - deps = ["//vendor/github.com/spf13/pflag:go_default_library"], -) - -filegroup( - name = "testdata", - srcs = glob(["testdata/*"]), -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/mungedocs/README.md b/cmd/mungedocs/README.md deleted file mode 100644 index 5fe3ed106cf45..0000000000000 --- a/cmd/mungedocs/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Documentation Mungers - -Basically this is like lint/gofmt for md docs. - -It basically does the following: -- iterate over all files in the given doc root. -- for each file split it into a slice (mungeLines) of lines (mungeLine) -- a mungeline has metadata about each line typically determined by a 'fast' regex. - - metadata contains things like 'is inside a preformatted block' - - contains a markdown header - - has a link to another file - - etc.. - - if you have a really slow regex with a lot of backtracking you might want to write a fast one to limit how often you run the slow one. -- each munger is then called in turn - - they are given the mungeLines - - they create an entirely new set of mungeLines with their modifications - - the new set is returned -- the new set is then fed into the next munger. -- in the end we might commit the end mungeLines to the file or not (--verify) - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cmd/mungedocs/README.md?pixel)]() diff --git a/cmd/mungedocs/analytics.go b/cmd/mungedocs/analytics.go deleted file mode 100644 index a7eaefa0803d4..0000000000000 --- a/cmd/mungedocs/analytics.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "strings" -) - -const analyticsMungeTag = "GENERATED_ANALYTICS" -const analyticsLinePrefix = "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/" - -func updateAnalytics(fileName string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - fileName, err := makeRepoRelative(fileName, fileName) - if err != nil { - return mlines, err - } - - link := fmt.Sprintf(analyticsLinePrefix+"%s?pixel)]()", fileName) - insertLines := getMungeLines(link) - mlines, err = removeMacroBlock(analyticsMungeTag, mlines) - if err != nil { - return mlines, err - } - - // Remove floating analytics links not surrounded by the munge tags. - for _, mline := range mlines { - if mline.preformatted || mline.header || mline.beginTag || mline.endTag { - out = append(out, mline) - continue - } - if strings.HasPrefix(mline.data, analyticsLinePrefix) { - continue - } - out = append(out, mline) - } - out = appendMacroBlock(out, analyticsMungeTag) - out, err = updateMacroBlock(out, analyticsMungeTag, insertLines) - if err != nil { - return mlines, err - } - return out, nil -} diff --git a/cmd/mungedocs/analytics_test.go b/cmd/mungedocs/analytics_test.go deleted file mode 100644 index b97feef86eb82..0000000000000 --- a/cmd/mungedocs/analytics_test.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestAnalytics(t *testing.T) { - b := beginMungeTag("GENERATED_ANALYTICS") - e := endMungeTag("GENERATED_ANALYTICS") - var cases = []struct { - in string - expected string - }{ - { - "aoeu", - "aoeu" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + "\n" + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()", - "aoeu" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n", - "aoeu" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n", - "aoeu" + "\n" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "prefix" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + - "\n" + "suffix", - "prefix" + "\n" + "suffix" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n", - "aoeu" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - out, err := updateAnalytics("path/to/file-name.md", in) - assert.NoError(t, err) - if !expected.Equal(out) { - t.Errorf("Case %d Expected \n\n%v\n\n but got \n\n%v\n\n", i, expected.String(), out.String()) - } - } -} diff --git a/cmd/mungedocs/example_syncer.go b/cmd/mungedocs/example_syncer.go deleted file mode 100644 index c15255be43948..0000000000000 --- a/cmd/mungedocs/example_syncer.go +++ /dev/null @@ -1,121 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "io/ioutil" - "regexp" - "strings" -) - -const exampleToken = "EXAMPLE" - -const exampleLineStart = " -// -// ```yaml -// foo: -// bar: -// ``` -// -// [Download example](../../examples/guestbook/frontend-service.yaml?raw=true) -// -func syncExamples(filePath string, mlines mungeLines) (mungeLines, error) { - var err error - type exampleTag struct { - token string - linkText string - fileType string - } - exampleTags := []exampleTag{} - - // collect all example Tags - for _, mline := range mlines { - if mline.preformatted || !mline.beginTag { - continue - } - line := mline.data - if !strings.HasPrefix(line, exampleLineStart) { - continue - } - match := exampleMungeTagRE.FindStringSubmatch(line) - if len(match) < 4 { - err = fmt.Errorf("Found unparsable EXAMPLE munge line %v", line) - return mlines, err - } - tag := exampleTag{ - token: exampleToken + " " + match[1], - linkText: match[1], - fileType: match[3], - } - exampleTags = append(exampleTags, tag) - } - // update all example Tags - for _, tag := range exampleTags { - ft := "" - if tag.fileType == "json" { - ft = "json" - } - if tag.fileType == "yaml" { - ft = "yaml" - } - example, err := exampleContent(filePath, tag.linkText, ft) - if err != nil { - return mlines, err - } - mlines, err = updateMacroBlock(mlines, tag.token, example) - if err != nil { - return mlines, err - } - } - return mlines, nil -} - -// exampleContent retrieves the content of the file at linkPath -func exampleContent(filePath, linkPath, fileType string) (mungeLines, error) { - repoRel, err := makeRepoRelative(linkPath, filePath) - if err != nil { - return nil, err - } - - fileRel, err := makeFileRelative(linkPath, filePath) - if err != nil { - return nil, err - } - - dat, err := ioutil.ReadFile(repoRel) - if err != nil { - return nil, err - } - - // remove leading and trailing spaces and newlines - trimmedFileContent := strings.TrimSpace(string(dat)) - content := fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s?raw=true)", fileType, trimmedFileContent, fileRel) - out := getMungeLines(content) - return out, nil -} diff --git a/cmd/mungedocs/example_syncer_test.go b/cmd/mungedocs/example_syncer_test.go deleted file mode 100644 index 4c2505350415b..0000000000000 --- a/cmd/mungedocs/example_syncer_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_syncExamples(t *testing.T) { - var podExample = `apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 -` - var textExample = `some text -` - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "\n\n", - "\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml?raw=true)\n\n", - }, - { - "\n\n", - "\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml?raw=true)\n\n", - }, - { - "\n\n", - "\n\n```\n" + textExample + "```\n\n[Download example](testdata/example.txt?raw=true)\n\n", - }, - } - repoRoot = "" - for _, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := syncExamples("filename.md", in) - assert.NoError(t, err) - if !expected.Equal(actual) { - t.Errorf("Expected example \n'%q' but got \n'%q'", expected.String(), actual.String()) - } - } -} diff --git a/cmd/mungedocs/headers.go b/cmd/mungedocs/headers.go deleted file mode 100644 index e23ae7536e5ee..0000000000000 --- a/cmd/mungedocs/headers.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "regexp" -) - -var headerRegex = regexp.MustCompile(`^(#+)\s*(.*)$`) - -func fixHeaderLine(mlines mungeLines, newlines mungeLines, linenum int) mungeLines { - var out mungeLines - - mline := mlines[linenum] - line := mlines[linenum].data - - matches := headerRegex.FindStringSubmatch(line) - if matches == nil { - out = append(out, mline) - return out - } - - // There must be a blank line before the # (unless first line in file) - if linenum != 0 { - newlen := len(newlines) - if newlines[newlen-1].data != "" { - out = append(out, blankMungeLine) - } - } - - // There must be a space AFTER the ##'s - newline := fmt.Sprintf("%s %s", matches[1], matches[2]) - newmline := newMungeLine(newline) - out = append(out, newmline) - - // The next line needs to be a blank line (unless last line in file) - if len(mlines) > linenum+1 && mlines[linenum+1].data != "" { - out = append(out, blankMungeLine) - } - return out -} - -// Header lines need whitespace around them and after the #s. -func updateHeaderLines(filePath string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - for i, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - if !mline.header { - out = append(out, mline) - continue - } - newLines := fixHeaderLine(mlines, out, i) - out = append(out, newLines...) - } - return out, nil -} diff --git a/cmd/mungedocs/headers_test.go b/cmd/mungedocs/headers_test.go deleted file mode 100644 index d30c9b32a144f..0000000000000 --- a/cmd/mungedocs/headers_test.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestHeaderLines(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "# ok", - "# ok", - }, - { - "## ok", - "## ok", - }, - { - "##### ok", - "##### ok", - }, - { - "##fix", - "## fix", - }, - { - "foo\n\n##fix\n\nbar", - "foo\n\n## fix\n\nbar", - }, - { - "foo\n##fix\nbar", - "foo\n\n## fix\n\nbar", - }, - { - "foo\n```\n##fix\n```\nbar", - "foo\n```\n##fix\n```\nbar", - }, - { - "foo\n#fix1\n##fix2\nbar", - "foo\n\n# fix1\n\n## fix2\n\nbar", - }, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updateHeaderLines("filename.md", in) - assert.NoError(t, err) - if !actual.Equal(expected) { - t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) - } - } -} diff --git a/cmd/mungedocs/kubectl_dash_f.go b/cmd/mungedocs/kubectl_dash_f.go deleted file mode 100644 index 2dae48cc8574d..0000000000000 --- a/cmd/mungedocs/kubectl_dash_f.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - "path" - "strings" -) - -// Looks for lines that have kubectl commands with -f flags and files that -// don't exist. -func updateKubectlFileTargets(file string, mlines mungeLines) (mungeLines, error) { - var errors []string - for i, mline := range mlines { - if !mline.preformatted { - continue - } - if err := lookForKubectl(mline.data, i); err != nil { - errors = append(errors, err.Error()) - } - } - err := error(nil) - if len(errors) != 0 { - err = fmt.Errorf("%s", strings.Join(errors, "\n")) - } - return mlines, err -} - -func lookForKubectl(line string, lineNum int) error { - fields := strings.Fields(line) - for i := range fields { - if fields[i] == "kubectl" { - return gotKubectl(lineNum, fields, i) - } - } - return nil -} - -func gotKubectl(lineNum int, fields []string, fieldNum int) error { - for i := fieldNum + 1; i < len(fields); i++ { - switch fields[i] { - case "create", "update", "replace", "delete": - return gotCommand(lineNum, fields, i) - } - } - return nil -} - -func gotCommand(lineNum int, fields []string, fieldNum int) error { - for i := fieldNum + 1; i < len(fields); i++ { - if strings.HasPrefix(fields[i], "-f") { - return gotDashF(lineNum, fields, i) - } - } - return nil -} - -func gotDashF(lineNum int, fields []string, fieldNum int) error { - target := "" - if fields[fieldNum] == "-f" { - if fieldNum+1 == len(fields) { - return fmt.Errorf("ran out of fields after '-f'") - } - target = fields[fieldNum+1] - } else { - target = fields[fieldNum][2:] - } - // Turn dirs into file-like names. - target = strings.TrimRight(target, "/") - - // Now exclude special-cases - - if target == "-" || target == "FILENAME" { - // stdin and "FILENAME" are OK - return nil - } - if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") { - // URLs are ok - return nil - } - if strings.HasPrefix(target, "./") { - // Same-dir files are usually created in the same example - return nil - } - if strings.HasPrefix(target, "~/") { - // Home directory may also be created by the same example - return nil - } - if strings.HasPrefix(target, "/") { - // Absolute paths tend to be /tmp/* and created in the same example. - return nil - } - if strings.HasPrefix(target, "$") { - // Allow the start of the target to be an environment - // variable that points to the root of the kubernetes - // repo. - split := strings.SplitN(target, "/", 2) - if len(split) == 2 { - target = split[1] - } - } - - // If we got here we expect the file to exist. - _, err := os.Stat(path.Join(repoRoot, target)) - if os.IsNotExist(err) { - return fmt.Errorf("%d: target file %q does not exist", lineNum, target) - } - return err -} diff --git a/cmd/mungedocs/kubectl_dash_f_test.go b/cmd/mungedocs/kubectl_dash_f_test.go deleted file mode 100644 index 6f18fd547a408..0000000000000 --- a/cmd/mungedocs/kubectl_dash_f_test.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import "testing" - -func TestKubectlDashF(t *testing.T) { - var cases = []struct { - in string - ok bool - }{ - // No match - {"", true}, - { - "Foo\nBar\n", - true, - }, - { - "Foo\nkubectl blah blech\nBar", - true, - }, - { - "Foo\n```shell\nkubectl blah blech\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create blech\n```\nBar", - true, - }, - // Special cases - { - "Foo\n```\nkubectl -blah create -f -\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f-\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f FILENAME\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -fFILENAME\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f http://google.com/foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -fhttp://google.com/foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f ./foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f./foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f /foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f/foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f~/foobar\n```\nBar", - true, - }, - // Real checks - { - "Foo\n```\nkubectl -blah create -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -fmungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah update -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah update -fmungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah replace -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah replace -fmungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah delete -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah delete -fmungedocs.go\n```\nBar", - true, - }, - // Failures - { - "Foo\n```\nkubectl -blah delete -f does_not_exist\n```\nBar", - false, - }, - { - "Foo\n```\nkubectl -blah delete -fdoes_not_exist\n```\nBar", - false, - }, - } - for i, c := range cases { - repoRoot = "" - in := getMungeLines(c.in) - _, err := updateKubectlFileTargets("filename.md", in) - if err != nil && c.ok { - t.Errorf("case[%d]: expected success, got %v", i, err) - } - if err == nil && !c.ok { - t.Errorf("case[%d]: unexpected success", i) - } - } -} diff --git a/cmd/mungedocs/links.go b/cmd/mungedocs/links.go deleted file mode 100644 index c05cdfa1874b7..0000000000000 --- a/cmd/mungedocs/links.go +++ /dev/null @@ -1,238 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - "fmt" - "net/url" - "os" - "path" - "regexp" - "strings" -) - -var ( - // Finds markdown links of the form [foo](bar "alt-text"). - linkRE = regexp.MustCompile(`\[([^]]*)\]\(([^)]*)\)`) - // Finds markdown link typos of the form (foo)[bar] - badLinkRE = regexp.MustCompile(`\([^]()]*\)\[[^]()]*\]`) - // Splits the link target into link target and alt-text. - altTextRE = regexp.MustCompile(`([^)]*)( ".*")`) -) - -func processLink(in string, filePath string) (string, error) { - var errs []string - out := linkRE.ReplaceAllStringFunc(in, func(in string) string { - var err error - match := linkRE.FindStringSubmatch(in) - if match == nil { - errs = append(errs, fmt.Sprintf("Detected this line had a link, but unable to parse, %v", in)) - return "" - } - // match[0] is the entire expression; - visibleText := match[1] - linkText := match[2] - altText := "" - if parts := altTextRE.FindStringSubmatch(linkText); parts != nil { - linkText = parts[1] - altText = parts[2] - } - - // clean up some random garbage I found in our docs. - linkText = strings.Trim(linkText, " ") - linkText = strings.Trim(linkText, "\n") - linkText = strings.Trim(linkText, " ") - - u, terr := url.Parse(linkText) - if terr != nil { - errs = append(errs, fmt.Sprintf("link %q is unparsable: %v", linkText, terr)) - return in - } - - if u.Host != "" && u.Host != "github.com" { - // We only care about relative links and links within github. - return in - } - - suggestedVisibleText := visibleText - if u.Path != "" && !strings.HasPrefix(linkText, "TODO:") { - newPath, targetExists := checkPath(filePath, path.Clean(u.Path)) - if !targetExists { - errs = append(errs, fmt.Sprintf("%q: target not found", linkText)) - return in - } - u.Path = newPath - if strings.HasPrefix(u.Path, "/") { - u.Host = "github.com" - u.Scheme = "https" - } else { - // Remove host and scheme from relative paths - u.Host = "" - u.Scheme = "" - } - // Make the visible text show the absolute path if it's - // not nested in or beneath the current directory. - if strings.HasPrefix(u.Path, "..") { - dir := path.Dir(filePath) - suggestedVisibleText, err = makeRepoRelative(path.Join(dir, u.Path), filePath) - if err != nil { - errs = append(errs, fmt.Sprintf("%q: unable to make path relative", filePath)) - return in - } - } else { - suggestedVisibleText = u.Path - } - var unescaped string - if unescaped, err = url.QueryUnescape(u.String()); err != nil { - // Remove %28 type stuff, be nice to humans. - // And don't fight with the toc generator. - linkText = unescaped - } else { - linkText = u.String() - } - } - // If the current visible text is trying to be a file name, use - // the correct file name. - if strings.HasSuffix(visibleText, ".md") && !strings.ContainsAny(visibleText, ` '"`+"`") { - visibleText = suggestedVisibleText - } - - return fmt.Sprintf("[%s](%s)", visibleText, linkText+altText) - }) - if len(errs) != 0 { - return "", errors.New(strings.Join(errs, ",")) - } - return out, nil -} - -// updateLinks assumes lines has links in markdown syntax, and verifies that -// any relative links actually point to files that exist. -func updateLinks(filePath string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - allErrs := []string{} - - for lineNum, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - if badMatch := badLinkRE.FindString(mline.data); badMatch != "" { - allErrs = append(allErrs, - fmt.Sprintf("On line %d: found backwards markdown link %q", lineNum, badMatch)) - } - if !mline.link { - out = append(out, mline) - continue - } - line, err := processLink(mline.data, filePath) - if err != nil { - var s = fmt.Sprintf("On line %d: %s", lineNum, err.Error()) - err := errors.New(s) - allErrs = append(allErrs, err.Error()) - } - ml := newMungeLine(line) - out = append(out, ml) - } - err := error(nil) - if len(allErrs) != 0 { - err = fmt.Errorf("%s", strings.Join(allErrs, "\n")) - } - return out, err -} - -// We have to append together before path.Clean will be able to tell that stuff -// like ../docs isn't needed. -func cleanPath(dirPath, linkPath string) string { - clean := path.Clean(path.Join(dirPath, linkPath)) - if strings.HasPrefix(clean, dirPath+"/") { - out := strings.TrimPrefix(clean, dirPath+"/") - if out != linkPath { - fmt.Printf("%s -> %s\n", linkPath, out) - } - return out - } - return linkPath -} - -func checkPath(filePath, linkPath string) (newPath string, ok bool) { - dir := path.Dir(filePath) - absFilePrefixes := []string{ - "/kubernetes/kubernetes/blob/master/", - "/kubernetes/kubernetes/tree/master/", - } - for _, prefix := range absFilePrefixes { - if strings.HasPrefix(linkPath, prefix) { - linkPath = strings.TrimPrefix(linkPath, prefix) - // Now linkPath is relative to the root of the repo. The below - // loop that adds ../ at the beginning of the path should find - // the right path. - break - } - } - if strings.HasPrefix(linkPath, "/") { - // These links might go to e.g. the github issues page, or a - // file at a particular revision, or another github project - // entirely. - return linkPath, true - } - linkPath = cleanPath(dir, linkPath) - - // Fast exit if the link is already correct. - if info, err := os.Stat(path.Join(dir, linkPath)); err == nil { - if info.IsDir() { - return linkPath + "/", true - } - return linkPath, true - } - - for strings.HasPrefix(linkPath, "../") { - linkPath = strings.TrimPrefix(linkPath, "../") - } - - // Fix - vs _ automatically - nameMungers := []func(string) string{ - func(s string) string { return s }, - func(s string) string { return strings.Replace(s, "-", "_", -1) }, - func(s string) string { return strings.Replace(s, "_", "-", -1) }, - } - // Fix being moved into/out of admin (replace "admin" with directory - // you're doing mass movements to/from). - pathMungers := []func(string) string{ - func(s string) string { return s }, - func(s string) string { return path.Join("admin", s) }, - func(s string) string { return strings.TrimPrefix(s, "admin/") }, - } - - for _, namer := range nameMungers { - for _, pather := range pathMungers { - newPath = pather(namer(linkPath)) - for i := 0; i < 7; i++ { - // The file must exist. - target := path.Join(dir, newPath) - if info, err := os.Stat(target); err == nil { - if info.IsDir() { - return newPath + "/", true - } - return cleanPath(dir, newPath), true - } - newPath = path.Join("..", newPath) - } - } - } - return linkPath, false -} diff --git a/cmd/mungedocs/links_test.go b/cmd/mungedocs/links_test.go deleted file mode 100644 index d5c761e051bad..0000000000000 --- a/cmd/mungedocs/links_test.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -var _ = fmt.Printf - -func TestBadLinks(t *testing.T) { - var cases = []struct { - in string - }{ - {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/NOTREADME.md)"}, - {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/docs/NOTREADME.md)"}, - {"[NOTREADME](../NOTREADME.md)"}, - } - for _, c := range cases { - in := getMungeLines(c.in) - _, err := updateLinks("filename.md", in) - assert.Error(t, err) - } -} -func TestGoodLinks(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - {"[README](https://lwn.net)", - "[README](https://lwn.net)"}, - // _ to - - {"[README](https://github.com/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/test_dashes.md)", - "[README](../../cmd/mungedocs/testdata/test-dashes.md)"}, - // - to _ - {"[README](../../cmd/mungedocs/testdata/test-underscores.md)", - "[README](../../cmd/mungedocs/testdata/test_underscores.md)"}, - - // Does this even make sense? i dunno - {"[README](/docs/README.md)", - "[README](https://github.com/docs/README.md)"}, - {"[README](/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/README.md)", - "[README](../../cmd/mungedocs/testdata/README.md)"}, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updateLinks("filename.md", in) - assert.NoError(t, err) - if !actual.Equal(expected) { - t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) - } - } -} diff --git a/cmd/mungedocs/mungedocs.go b/cmd/mungedocs/mungedocs.go deleted file mode 100644 index 4ae44daff7c6c..0000000000000 --- a/cmd/mungedocs/mungedocs.go +++ /dev/null @@ -1,239 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" - - flag "github.com/spf13/pflag" -) - -// This needs to be updated when we cut a new release series. -const latestReleaseBranch = "release-1.5" - -var ( - verbose = flag.Bool("verbose", false, "On verification failure, emit pre-munge and post-munge versions.") - verify = flag.Bool("verify", false, "Exit with status 1 if files would have needed changes but do not change.") - norecurse = flag.Bool("norecurse", false, "Only process the files of --root-dir.") - upstream = flag.String("upstream", "upstream", "The name of the upstream Git remote to pull from") - rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.") - // "repo-root" seems like a dumb name, this is the relative path (from rootDir) to get to the repoRoot - relRoot = flag.String("repo-root", "..", `Appended to --root-dir to get the repository root. -It's done this way so that generally you just have to set --root-dir. -Examples: - * --root-dir=docs/ --repo-root=.. means the repository root is ./ - * --root-dir=/usr/local/long/path/repo/docs/ --repo-root=.. means the repository root is /usr/local/long/path/repo/ - * --root-dir=/usr/local/long/path/repo/docs/admin --repo-root=../.. means the repository root is /usr/local/long/path/repo/`) - skipMunges = flag.String("skip-munges", "", "Comma-separated list of munges to *not* run. Available munges are: "+availableMungeList) - repoRoot string - - ErrChangesNeeded = errors.New("mungedocs: changes required") - - // All of the munge operations to perform. - // TODO: allow selection from command line. (e.g., just check links in the examples directory.) - allMunges = []munge{ - // Simple "check something" functions must run first. - {"preformat-balance", checkPreformatBalance}, - // Functions which modify state. - {"remove-whitespace", updateWhitespace}, - {"table-of-contents", updateTOC}, - {"md-links", updateLinks}, - {"blank-lines-surround-preformatted", updatePreformatted}, - {"header-lines", updateHeaderLines}, - {"analytics", updateAnalytics}, - {"kubectl-dash-f", updateKubectlFileTargets}, - {"sync-examples", syncExamples}, - } - availableMungeList = func() string { - names := []string{} - for _, m := range allMunges { - names = append(names, m.name) - } - return strings.Join(names, ",") - }() -) - -// a munge processes a document, returning an updated document xor an error. -// The fn is NOT allowed to mutate 'before', if changes are needed it must copy -// data into a new byte array and return that. -type munge struct { - name string - fn func(filePath string, mlines mungeLines) (after mungeLines, err error) -} - -type fileProcessor struct { - // Which munge functions should we call? - munges []munge - - // Are we allowed to make changes? - verifyOnly bool -} - -// Either change a file or verify that it needs no changes (according to modify argument) -func (f fileProcessor) visit(path string) error { - if !strings.HasSuffix(path, ".md") { - return nil - } - - fileBytes, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - mungeLines := getMungeLines(string(fileBytes)) - - modificationsMade := false - errFound := false - filePrinted := false - for _, munge := range f.munges { - after, err := munge.fn(path, mungeLines) - if err != nil || !after.Equal(mungeLines) { - if !filePrinted { - fmt.Printf("%s\n----\n", path) - filePrinted = true - } - fmt.Printf("%s:\n", munge.name) - if *verbose { - if len(mungeLines) <= 20 { - fmt.Printf("INPUT: <<<%v>>>\n", mungeLines) - fmt.Printf("MUNGED: <<<%v>>>\n", after) - } else { - fmt.Printf("not printing failed chunk: too many lines\n") - } - } - if err != nil { - fmt.Println(err) - errFound = true - } else { - fmt.Println("contents were modified") - modificationsMade = true - } - fmt.Println("") - } - mungeLines = after - } - - // Write out new file with any changes. - if modificationsMade { - if f.verifyOnly { - // We're not allowed to make changes. - return ErrChangesNeeded - } - ioutil.WriteFile(path, mungeLines.Bytes(), 0644) - } - if errFound { - return ErrChangesNeeded - } - - return nil -} - -func newWalkFunc(fp *fileProcessor, changesNeeded *bool) filepath.WalkFunc { - return func(path string, info os.FileInfo, err error) error { - stat, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - if path != *rootDir && stat.IsDir() && *norecurse { - return filepath.SkipDir - } - if err := fp.visit(path); err != nil { - *changesNeeded = true - if err != ErrChangesNeeded { - return err - } - } - return nil - } -} - -func wantedMunges() (filtered []munge) { - skipList := strings.Split(*skipMunges, ",") - skipped := map[string]bool{} - for _, m := range skipList { - if len(m) > 0 { - skipped[m] = true - } - } - for _, m := range allMunges { - if !skipped[m.name] { - filtered = append(filtered, m) - } else { - // Remove from the map so we can verify that everything - // requested was in fact valid. - delete(skipped, m.name) - } - } - if len(skipped) != 0 { - fmt.Fprintf(os.Stderr, "ERROR: requested to skip %v, but these are not valid munges. (valid: %v)\n", skipped, availableMungeList) - os.Exit(1) - } - return filtered -} - -func main() { - var err error - flag.Parse() - - if *rootDir == "" { - fmt.Fprintf(os.Stderr, "usage: %s [--help] [--verify] [--norecurse] --root-dir [--skip-munges=] [--upstream=] \n", flag.Arg(0)) - os.Exit(1) - } - - repoRoot = path.Join(*rootDir, *relRoot) - repoRoot, err = filepath.Abs(repoRoot) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) - os.Exit(2) - } - - fp := fileProcessor{ - munges: wantedMunges(), - verifyOnly: *verify, - } - - // For each markdown file under source docs root, process the doc. - // - If any error occurs: exit with failure (exit >1). - // - If verify is true: exit 0 if no changes needed, exit 1 if changes - // needed. - // - If verify is false: exit 0 if changes successfully made or no - // changes needed, exit 1 if manual changes are needed. - var changesNeeded bool - - err = filepath.Walk(*rootDir, newWalkFunc(&fp, &changesNeeded)) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) - os.Exit(2) - } - if changesNeeded { - if *verify { - fmt.Fprintf(os.Stderr, "FAIL: changes needed but not made due to --verify\n") - } else { - fmt.Fprintf(os.Stderr, "FAIL: some manual changes are still required.\n") - } - os.Exit(1) - } -} diff --git a/cmd/mungedocs/preformatted.go b/cmd/mungedocs/preformatted.go deleted file mode 100644 index 582ba981a109d..0000000000000 --- a/cmd/mungedocs/preformatted.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import "fmt" - -// Blocks of ``` need to have blank lines on both sides or they don't look -// right in HTML. -func updatePreformatted(filePath string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - inpreformat := false - for i, mline := range mlines { - if !inpreformat && mline.preformatted { - if i == 0 || out[len(out)-1].data != "" { - out = append(out, blankMungeLine) - } - // start of a preformat block - inpreformat = true - } - out = append(out, mline) - if inpreformat && !mline.preformatted { - if i >= len(mlines)-2 || mlines[i+1].data != "" { - out = append(out, blankMungeLine) - } - inpreformat = false - } - } - return out, nil -} - -// If the file ends on a preformatted line, there must have been an imbalance. -func checkPreformatBalance(filePath string, mlines mungeLines) (mungeLines, error) { - if len(mlines) > 0 && mlines[len(mlines)-1].preformatted { - return mlines, fmt.Errorf("unbalanced triple backtick delimiters") - } - return mlines, nil -} diff --git a/cmd/mungedocs/preformatted_test.go b/cmd/mungedocs/preformatted_test.go deleted file mode 100644 index 718e002976ee4..0000000000000 --- a/cmd/mungedocs/preformatted_test.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestPreformatted(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "```\nbob\n```", - "\n```\nbob\n```\n\n", - }, - { - "```\nbob\n```\n```\nnotbob\n```\n", - "\n```\nbob\n```\n\n```\nnotbob\n```\n\n", - }, - { - "```bob```\n", - "```bob```\n", - }, - { - " ```\n bob\n ```", - "\n ```\n bob\n ```\n\n", - }, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updatePreformatted("filename.md", in) - assert.NoError(t, err) - if !actual.Equal(expected) { - t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) - } - } -} - -func TestPreformattedImbalance(t *testing.T) { - var cases = []struct { - in string - ok bool - }{ - {"", true}, - {"```\nin\n```", true}, - {"```\nin\n```\nout", true}, - {"```", false}, - {"```\nin\n```\nout\n```", false}, - } - for i, c := range cases { - in := getMungeLines(c.in) - out, err := checkPreformatBalance("filename.md", in) - if err != nil && c.ok { - t.Errorf("case[%d]: expected success", i) - } - if err == nil && !c.ok { - t.Errorf("case[%d]: expected failure", i) - } - // Even in case of misformat, return all the text, - // so that the user's work is not lost. - if !equalMungeLines(out, in) { - t.Errorf("case[%d]: expected munged text to be identical to input text", i) - } - } -} - -func equalMungeLines(a, b mungeLines) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} diff --git a/cmd/mungedocs/testdata/README.md b/cmd/mungedocs/testdata/README.md deleted file mode 100644 index 7b57bd29ea8af..0000000000000 --- a/cmd/mungedocs/testdata/README.md +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/cmd/mungedocs/testdata/example.txt b/cmd/mungedocs/testdata/example.txt deleted file mode 100644 index 7b57bd29ea8af..0000000000000 --- a/cmd/mungedocs/testdata/example.txt +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/cmd/mungedocs/testdata/pod.yaml b/cmd/mungedocs/testdata/pod.yaml deleted file mode 100644 index 89920b83a9af6..0000000000000 --- a/cmd/mungedocs/testdata/pod.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 \ No newline at end of file diff --git a/cmd/mungedocs/testdata/test-dashes.md b/cmd/mungedocs/testdata/test-dashes.md deleted file mode 100644 index 7b57bd29ea8af..0000000000000 --- a/cmd/mungedocs/testdata/test-dashes.md +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/cmd/mungedocs/testdata/test_underscores.md b/cmd/mungedocs/testdata/test_underscores.md deleted file mode 100644 index 7b57bd29ea8af..0000000000000 --- a/cmd/mungedocs/testdata/test_underscores.md +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/cmd/mungedocs/toc.go b/cmd/mungedocs/toc.go deleted file mode 100644 index 649a954bf5abc..0000000000000 --- a/cmd/mungedocs/toc.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "regexp" - "strings" -) - -const tocMungeTag = "GENERATED_TOC" - -var r = regexp.MustCompile("[^A-Za-z0-9-]") - -// inserts/updates a table of contents in markdown file. -// -// First, builds a ToC. -// Then, finds the magic macro block tags and replaces anything between those with -// the ToC, thereby updating any previously inserted ToC. -// -// TODO(erictune): put this in own package with tests -func updateTOC(filePath string, mlines mungeLines) (mungeLines, error) { - toc := buildTOC(mlines) - updatedMarkdown, err := updateMacroBlock(mlines, tocMungeTag, toc) - if err != nil { - return mlines, err - } - return updatedMarkdown, nil -} - -// builds table of contents for markdown file -// -// First scans for all section headers (lines that begin with "#" but not within code quotes) -// and builds a table of contents from those. Assumes bookmarks for those will be -// like #each-word-in-heading-in-lowercases-with-dashes-instead-of-spaces. -// builds the ToC. - -func buildTOC(mlines mungeLines) mungeLines { - var out mungeLines - bookmarks := map[string]int{} - - for _, mline := range mlines { - if mline.preformatted || !mline.header { - continue - } - // Add a blank line after the munge start tag - if len(out) == 0 { - out = append(out, blankMungeLine) - } - line := mline.data - noSharps := strings.TrimLeft(line, "#") - numSharps := len(line) - len(noSharps) - heading := strings.Trim(noSharps, " \n") - if numSharps > 0 { - indent := strings.Repeat(" ", numSharps-1) - bookmark := strings.Replace(strings.ToLower(heading), " ", "-", -1) - // remove symbols (except for -) in bookmarks - bookmark = r.ReplaceAllString(bookmark, "") - // Incremental counter for duplicate bookmarks - next := bookmarks[bookmark] - bookmarks[bookmark] = next + 1 - if next > 0 { - bookmark = fmt.Sprintf("%s-%d", bookmark, next) - } - tocLine := fmt.Sprintf("%s- [%s](#%s)", indent, heading, bookmark) - out = append(out, newMungeLine(tocLine)) - } - - } - // Add a blank line before the munge end tag - if len(out) != 0 { - out = append(out, blankMungeLine) - } - return out -} diff --git a/cmd/mungedocs/toc_test.go b/cmd/mungedocs/toc_test.go deleted file mode 100644 index 5d7e27cdcdee1..0000000000000 --- a/cmd/mungedocs/toc_test.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_buildTOC(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - {"Lorem ipsum\ndolor sit amet\n", ""}, - { - "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n", - "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", - }, - { - "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n```bash\n#!/bin/sh\n```", - "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", - }, - { - "# Title\nLorem ipsum \n## Section Heading\n### Ok, why doesn't this work? ...add 4 *more* `symbols`!\ndolor sit amet\n", - "\n- [Title](#title)\n - [Section Heading](#section-heading)\n - [Ok, why doesn't this work? ...add 4 *more* `symbols`!](#ok-why-doesnt-this-work-add-4-more-symbols)\n\n", - }, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual := buildTOC(in) - if !expected.Equal(actual) { - t.Errorf("Case[%d] Expected TOC '%v' but got '%v'", i, expected.String(), actual.String()) - } - } -} - -func Test_updateTOC(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "Lorem ipsum\ndolor sit amet\n", - "Lorem ipsum\ndolor sit amet\n", - }, - { - "# Title\nLorem ipsum \n**table of contents**\n\nold cruft\n\n## Section Heading\ndolor sit amet\n", - "# Title\nLorem ipsum \n**table of contents**\n\n\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n\n## Section Heading\ndolor sit amet\n", - }, - } - for _, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updateTOC("filename.md", in) - assert.NoError(t, err) - if !expected.Equal(actual) { - t.Errorf("Expected TOC '%v' but got '%v'", expected.String(), actual.String()) - } - } -} diff --git a/cmd/mungedocs/util.go b/cmd/mungedocs/util.go deleted file mode 100644 index c25e1d1976ecc..0000000000000 --- a/cmd/mungedocs/util.go +++ /dev/null @@ -1,291 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "path" - "path/filepath" - "regexp" - "strings" - "unicode" -) - -// Replaces the text between matching "beginMark" and "endMark" within the -// document represented by "lines" with "insertThis". -// -// Delimiters should occupy own line. -// Returns copy of document with modifications. -func updateMacroBlock(mlines mungeLines, token string, insertThis mungeLines) (mungeLines, error) { - beginMark := beginMungeTag(token) - endMark := endMungeTag(token) - var out mungeLines - betweenBeginAndEnd := false - for _, mline := range mlines { - if mline.preformatted && !betweenBeginAndEnd { - out = append(out, mline) - continue - } - line := mline.data - if mline.beginTag && line == beginMark { - if betweenBeginAndEnd { - return nil, fmt.Errorf("found second begin mark while updating macro blocks") - } - betweenBeginAndEnd = true - out = append(out, mline) - } else if mline.endTag && line == endMark { - if !betweenBeginAndEnd { - return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") - } - betweenBeginAndEnd = false - out = append(out, insertThis...) - out = append(out, mline) - } else { - if !betweenBeginAndEnd { - out = append(out, mline) - } - } - } - if betweenBeginAndEnd { - return nil, fmt.Errorf("never found closing end mark while updating macro blocks") - } - return out, nil -} - -// Tests that a document, represented as a slice of lines, has a line. Ignores -// leading and trailing space. -func hasLine(lines mungeLines, needle string) bool { - for _, mline := range lines { - haystack := strings.TrimSpace(mline.data) - if haystack == needle { - return true - } - } - return false -} - -func removeMacroBlock(token string, mlines mungeLines) (mungeLines, error) { - beginMark := beginMungeTag(token) - endMark := endMungeTag(token) - var out mungeLines - betweenBeginAndEnd := false - for _, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - line := mline.data - if mline.beginTag && line == beginMark { - if betweenBeginAndEnd { - return nil, fmt.Errorf("found second begin mark while updating macro blocks") - } - betweenBeginAndEnd = true - } else if mline.endTag && line == endMark { - if !betweenBeginAndEnd { - return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") - } - betweenBeginAndEnd = false - } else { - if !betweenBeginAndEnd { - out = append(out, mline) - } - } - } - if betweenBeginAndEnd { - return nil, fmt.Errorf("never found closing end mark while updating macro blocks") - } - return out, nil -} - -// Add a macro block to the beginning of a set of lines -func prependMacroBlock(token string, mlines mungeLines) mungeLines { - beginLine := newMungeLine(beginMungeTag(token)) - endLine := newMungeLine(endMungeTag(token)) - out := mungeLines{beginLine, endLine} - if len(mlines) > 0 && mlines[0].data != "" { - out = append(out, blankMungeLine) - } - return append(out, mlines...) -} - -// Add a macro block to the end of a set of lines -func appendMacroBlock(mlines mungeLines, token string) mungeLines { - beginLine := newMungeLine(beginMungeTag(token)) - endLine := newMungeLine(endMungeTag(token)) - out := mlines - if len(mlines) > 0 && mlines[len(mlines)-1].data != "" { - out = append(out, blankMungeLine) - } - return append(out, beginLine, endLine) -} - -// Tests that a document, represented as a slice of lines, has a macro block. -func hasMacroBlock(lines mungeLines, token string) bool { - beginMark := beginMungeTag(token) - endMark := endMungeTag(token) - - foundBegin := false - for _, mline := range lines { - if mline.preformatted { - continue - } - if !mline.beginTag && !mline.endTag { - continue - } - line := mline.data - switch { - case !foundBegin && line == beginMark: - foundBegin = true - case foundBegin && line == endMark: - return true - } - } - return false -} - -// Returns the canonical begin-tag for a given description. This does not -// include the trailing newline. -func beginMungeTag(desc string) string { - return fmt.Sprintf("", desc) -} - -// Returns the canonical end-tag for a given description. This does not -// include the trailing newline. -func endMungeTag(desc string) string { - return fmt.Sprintf("", desc) -} - -type mungeLine struct { - data string - preformatted bool - header bool - link bool - beginTag bool - endTag bool -} - -type mungeLines []mungeLine - -func (m1 mungeLines) Equal(m2 mungeLines) bool { - if len(m1) != len(m2) { - return false - } - for i := range m1 { - if m1[i].data != m2[i].data { - return false - } - } - return true -} - -func (mlines mungeLines) String() string { - slice := []string{} - for _, mline := range mlines { - slice = append(slice, mline.data) - } - s := strings.Join(slice, "\n") - // We need to tack on an extra newline at the end of the file - return s + "\n" -} - -func (mlines mungeLines) Bytes() []byte { - return []byte(mlines.String()) -} - -var ( - // Finds all preformatted block start/stops. - preformatRE = regexp.MustCompile("^\\s*```") - notPreformatRE = regexp.MustCompile("^\\s*```.*```") - // Is this line a header? - mlHeaderRE = regexp.MustCompile(`^#`) - // Is there a link on this line? - mlLinkRE = regexp.MustCompile(`\[[^]]*\]\([^)]*\)`) - beginTagRE = regexp.MustCompile(` + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: newrelic-config +type: Opaque +data: + config: {{config_data}} +``` + +[Download example](newrelic-config-template.yaml?raw=true) + + +The script will encode the config file and write it to `newrelic-config.yaml`. + +Finally, submit the config to the cluster: + +```console +$ kubectl create -f examples/newrelic-infrastructure/newrelic-config.yaml +``` + +### Step 2: Create the DaemonSet definition. + +The DaemonSet definition instructs Kubernetes to place a newrelic Infrastructure agent on each Kubernetes node. + + + +```yaml +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: newrelic-infra-agent + labels: + tier: monitoring + app: newrelic-infra-agent + version: v1 +spec: + template: + metadata: + labels: + name: newrelic + spec: + # Filter to specific nodes: + # nodeSelector: + # app: newrelic + hostPID: true + hostIPC: true + hostNetwork: true + containers: + - resources: + requests: + cpu: 0.15 + securityContext: + privileged: true + image: newrelic/infrastructure + name: newrelic + command: [ "bash", "-c", "source /etc/kube-nr-infra/config && /usr/bin/newrelic-infra" ] + volumeMounts: + - name: newrelic-config + mountPath: /etc/kube-nr-infra + readOnly: true + - name: dev + mountPath: /dev + - name: run + mountPath: /var/run/docker.sock + - name: log + mountPath: /var/log + - name: host-root + mountPath: /host + readOnly: true + volumes: + - name: newrelic-config + secret: + secretName: newrelic-config + - name: dev + hostPath: + path: /dev + - name: run + hostPath: + path: /var/run/docker.sock + - name: log + hostPath: + path: /var/log + - name: host-root + hostPath: + path: / +``` + +[Download example](newrelic-infra-daemonset.yaml?raw=true) + + +The daemonset instructs Kubernetes to spawn pods on each node, mapping /dev/, /run/, and /var/log to the container. It also maps the entire kube node / to /host/ in the container with a read-only mount. It also maps the secrets we set up earlier to /etc/kube-newrelic/config, and sources them in the startup script, configuring the agent properly. + +#### DaemonSet customization + +- There are more environment variables for fine tuning the infrastructure agent's operation (or a yaml file that you'd have to construct). See [Infrastructure Agent Environment Variables][(https://docs.newrelic.com/docs/infrastructure/new-relic-infrastructure/configuration/configure-infrastructure-agent) for the full list. + + +### Known issues + +It's a bit cludgy to define the environment variables like we do here in these config files. There is [another issue](https://github.com/kubernetes/kubernetes/issues/4710) to discuss adding mapping secrets to environment variables in Kubernetes. (Personally I don't like that method and prefer to use the config secrets.) + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/newrelic/README.md?pixel)]() + diff --git a/examples/newrelic-infrastructure/config-to-secret.sh b/examples/newrelic-infrastructure/config-to-secret.sh new file mode 100755 index 0000000000000..520c71990b015 --- /dev/null +++ b/examples/newrelic-infrastructure/config-to-secret.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Encodes the environment variables into a Kubernetes secret. + +BASE64_ENC=$(cat nrconfig.env | base64 | tr -d '\n') +sed -e "s#{{config_data}}#${BASE64_ENC}#g" ./newrelic-config-template.yaml > newrelic-config.yaml diff --git a/examples/newrelic-infrastructure/newrelic-config-template.yaml b/examples/newrelic-infrastructure/newrelic-config-template.yaml new file mode 100644 index 0000000000000..361a307924070 --- /dev/null +++ b/examples/newrelic-infrastructure/newrelic-config-template.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: newrelic-config +type: Opaque +data: + config: {{config_data}} diff --git a/examples/newrelic-infrastructure/newrelic-infra-daemonset.yaml b/examples/newrelic-infrastructure/newrelic-infra-daemonset.yaml new file mode 100644 index 0000000000000..395ab97223a5d --- /dev/null +++ b/examples/newrelic-infrastructure/newrelic-infra-daemonset.yaml @@ -0,0 +1,58 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: newrelic-infra-agent + labels: + tier: monitoring + app: newrelic-infra-agent + version: v1 +spec: + template: + metadata: + labels: + name: newrelic + spec: + # Filter to specific nodes: + # nodeSelector: + # app: newrelic + hostPID: true + hostIPC: true + hostNetwork: true + containers: + - resources: + requests: + cpu: 0.15 + securityContext: + privileged: true + image: newrelic/infrastructure + name: newrelic + command: [ "bash", "-c", "source /etc/kube-nr-infra/config && /usr/bin/newrelic-infra" ] + volumeMounts: + - name: newrelic-config + mountPath: /etc/kube-nr-infra + readOnly: true + - name: dev + mountPath: /dev + - name: run + mountPath: /var/run/docker.sock + - name: log + mountPath: /var/log + - name: host-root + mountPath: /host + readOnly: true + volumes: + - name: newrelic-config + secret: + secretName: newrelic-config + - name: dev + hostPath: + path: /dev + - name: run + hostPath: + path: /var/run/docker.sock + - name: log + hostPath: + path: /var/log + - name: host-root + hostPath: + path: / diff --git a/examples/newrelic-infrastructure/nrconfig.env b/examples/newrelic-infrastructure/nrconfig.env new file mode 100644 index 0000000000000..ced09727caf41 --- /dev/null +++ b/examples/newrelic-infrastructure/nrconfig.env @@ -0,0 +1,9 @@ +#--REQUIRED-- +# Put your license key in this variable +export NRIA_LICENSE_KEY=REPLACE_LICENSE_KEY_HERE +# +#--OPTIONAL-- +# Set to 1 for debug output in the log +export NRIA_VERBOSE=0 +# Can log to any file, but will not create directories +export NRIA_LOG_FILE=/var/log/nr-infra.log diff --git a/examples/newrelic/newrelic-daemonset.yaml b/examples/newrelic/newrelic-daemonset.yaml index 24adb382378b5..fd46972204959 100644 --- a/examples/newrelic/newrelic-daemonset.yaml +++ b/examples/newrelic/newrelic-daemonset.yaml @@ -52,6 +52,7 @@ spec: - name: run hostPath: path: /var/run/docker.sock + type: Socket - name: sys hostPath: path: /sys diff --git a/examples/oms/omsagent-daemonset.yaml b/examples/oms/omsagent-daemonset.yaml index bf59b370edbfe..8ab523f06032a 100644 --- a/examples/oms/omsagent-daemonset.yaml +++ b/examples/oms/omsagent-daemonset.yaml @@ -27,4 +27,5 @@ spec: volumes: - name: docker-sock hostPath: - path: /var/run/docker.sock \ No newline at end of file + path: /var/run/docker.sock + type: Socket diff --git a/examples/podsecuritypolicy/rbac/bindings.yaml b/examples/podsecuritypolicy/rbac/bindings.yaml index 13b8ac3c4ac5d..17a686755a82b 100644 --- a/examples/podsecuritypolicy/rbac/bindings.yaml +++ b/examples/podsecuritypolicy/rbac/bindings.yaml @@ -1,6 +1,6 @@ # privilegedPSP gives the privilegedPSP role # to the group privileged. -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: privileged-psp-users @@ -15,7 +15,7 @@ roleRef: --- # restrictedPSP grants the restrictedPSP role to # the groups restricted and privileged. -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: restricted-psp-users @@ -33,7 +33,7 @@ roleRef: --- # edit grants edit role to the groups # restricted and privileged. -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: edit diff --git a/examples/podsecuritypolicy/rbac/roles.yaml b/examples/podsecuritypolicy/rbac/roles.yaml index 43aecf2a09a46..b7ee13db4729c 100644 --- a/examples/podsecuritypolicy/rbac/roles.yaml +++ b/examples/podsecuritypolicy/rbac/roles.yaml @@ -1,6 +1,6 @@ # restrictedPSP grants access to use # the restricted PSP. -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: restricted-psp-user @@ -16,7 +16,7 @@ rules: --- # privilegedPSP grants access to use the privileged # PSP. -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: privileged-psp-user diff --git a/examples/sysdig-cloud/sysdig-daemonset.yaml b/examples/sysdig-cloud/sysdig-daemonset.yaml index e1fc1534a75a9..3cfd4a54f8dbb 100644 --- a/examples/sysdig-cloud/sysdig-daemonset.yaml +++ b/examples/sysdig-cloud/sysdig-daemonset.yaml @@ -16,6 +16,7 @@ spec: - name: docker-sock hostPath: path: /var/run/docker.sock + type: Socket - name: dev-vol hostPath: path: /dev diff --git a/examples/sysdig-cloud/sysdig-rc.yaml b/examples/sysdig-cloud/sysdig-rc.yaml index d088cd5355b25..dfef08de1ad76 100644 --- a/examples/sysdig-cloud/sysdig-rc.yaml +++ b/examples/sysdig-cloud/sysdig-rc.yaml @@ -14,6 +14,7 @@ spec: - name: docker-sock hostPath: path: /var/run/docker.sock + type: Socket - name: dev-vol hostPath: path: /dev diff --git a/examples/volumes/flexvolume/README.md b/examples/volumes/flexvolume/README.md index 33ce85b526e5d..a872b6524865f 100644 --- a/examples/volumes/flexvolume/README.md +++ b/examples/volumes/flexvolume/README.md @@ -1 +1 @@ -This file has moved to [https://github.com/kubernetes/examples/blob/master/staging/volumes/flexvolume/README.md](https://github.com/kubernetes/examples/blob/master/staging/volumes/flexvolume/README.md) +This file has moved to [https://github.com/kubernetes/community/blob/master/contributors/devel/flexvolume.md](https://github.com/kubernetes/community/blob/master/contributors/devel/flexvolume.md) diff --git a/federation/apis/core/v1/conversion.go b/federation/apis/core/v1/conversion.go index 8ce8f3a55acca..2e9ed1e7411bb 100644 --- a/federation/apis/core/v1/conversion.go +++ b/federation/apis/core/v1/conversion.go @@ -17,8 +17,6 @@ limitations under the License. package v1 import ( - "fmt" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/pkg/api/v1" ) @@ -59,24 +57,6 @@ func addConversionFuncs(scheme *runtime.Scheme) error { return err } - // Add field label conversions for kinds having selectable nothing but ObjectMeta fields. - for _, kind := range []string{ - "Service", - } { - err = scheme.AddFieldLabelConversionFunc("v1", kind, - func(label, value string) (string, string, error) { - switch label { - case "metadata.namespace", - "metadata.name": - return label, value, nil - default: - return "", "", fmt.Errorf("field label %q not supported for %q", label, kind) - } - }) - if err != nil { - return err - } - } if err := v1.AddFieldLabelConversionsForEvent(scheme); err != nil { return nil } diff --git a/federation/apis/federation/v1beta1/BUILD b/federation/apis/federation/v1beta1/BUILD index 9db2ea32cf780..a659903c83f2d 100644 --- a/federation/apis/federation/v1beta1/BUILD +++ b/federation/apis/federation/v1beta1/BUILD @@ -8,7 +8,6 @@ load( go_library( name = "go_default_library", srcs = [ - "conversion.go", "defaults.go", "doc.go", "generated.pb.go", diff --git a/federation/apis/federation/v1beta1/register.go b/federation/apis/federation/v1beta1/register.go index 10932175f589e..b14f5b1b06168 100644 --- a/federation/apis/federation/v1beta1/register.go +++ b/federation/apis/federation/v1beta1/register.go @@ -40,7 +40,7 @@ func init() { // We only register manually written functions here. The registration of the // generated functions takes place in the generated files. The separation // makes the code compile even when the generated files are missing. - localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs, addConversionFuncs) + localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs) } func addKnownTypes(scheme *runtime.Scheme) error { diff --git a/federation/apis/federation/v1beta1/zz_generated.conversion.go b/federation/apis/federation/v1beta1/zz_generated.conversion.go index c663146f3e819..1f9e733ddcdc0 100644 --- a/federation/apis/federation/v1beta1/zz_generated.conversion.go +++ b/federation/apis/federation/v1beta1/zz_generated.conversion.go @@ -127,11 +127,7 @@ func Convert_v1beta1_ClusterList_To_federation_ClusterList(in *ClusterList, out func autoConvert_federation_ClusterList_To_v1beta1_ClusterList(in *federation.ClusterList, out *ClusterList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]Cluster, 0) - } else { - out.Items = *(*[]Cluster)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]Cluster)(unsafe.Pointer(&in.Items)) return nil } @@ -152,11 +148,7 @@ func Convert_v1beta1_ClusterSpec_To_federation_ClusterSpec(in *ClusterSpec, out } func autoConvert_federation_ClusterSpec_To_v1beta1_ClusterSpec(in *federation.ClusterSpec, out *ClusterSpec, s conversion.Scope) error { - if in.ServerAddressByClientCIDRs == nil { - out.ServerAddressByClientCIDRs = make([]ServerAddressByClientCIDR, 0) - } else { - out.ServerAddressByClientCIDRs = *(*[]ServerAddressByClientCIDR)(unsafe.Pointer(&in.ServerAddressByClientCIDRs)) - } + out.ServerAddressByClientCIDRs = *(*[]ServerAddressByClientCIDR)(unsafe.Pointer(&in.ServerAddressByClientCIDRs)) out.SecretRef = (*v1.LocalObjectReference)(unsafe.Pointer(in.SecretRef)) return nil } diff --git a/federation/apis/openapi-spec/swagger.json b/federation/apis/openapi-spec/swagger.json index b0f0e4363dded..3336486d3d10c 100644 --- a/federation/apis/openapi-spec/swagger.json +++ b/federation/apis/openapi-spec/swagger.json @@ -9844,6 +9844,16 @@ } } }, + "io.k8s.api.core.v1.ClientIPConfig": { + "description": "ClientIPConfig represents the configurations of Client IP based session affinity.", + "properties": { + "timeoutSeconds": { + "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", + "type": "integer", + "format": "int32" + } + } + }, "io.k8s.api.core.v1.ConfigMap": { "description": "ConfigMap holds configuration data for pods to consume.", "properties": { @@ -10246,7 +10256,7 @@ "$ref": "#/definitions/io.k8s.api.core.v1.ObjectFieldSelector" }, "resourceFieldRef": { - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.", + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.", "$ref": "#/definitions/io.k8s.api.core.v1.ResourceFieldSelector" }, "secretKeyRef": { @@ -10605,7 +10615,11 @@ ], "properties": { "path": { - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "type": "string" + }, + "type": { + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", "type": "string" } } @@ -10630,6 +10644,10 @@ "description": "Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi", "type": "string" }, + "initiatorName": { + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection.", + "type": "string" + }, "iqn": { "description": "Target iSCSI Qualified Name.", "type": "string" @@ -10881,9 +10899,7 @@ "properties": { "key": { "description": "The label key that the selector applies to.", - "type": "string", - "x-kubernetes-patch-merge-key": "key", - "x-kubernetes-patch-strategy": "merge" + "type": "string" }, "operator": { "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", @@ -11221,7 +11237,7 @@ "$ref": "#/definitions/io.k8s.api.core.v1.Volume" }, "x-kubernetes-patch-merge-key": "name", - "x-kubernetes-patch-strategy": "merge" + "x-kubernetes-patch-strategy": "merge,retainKeys" } } }, @@ -11868,6 +11884,10 @@ "description": "Supports \"ClientIP\" and \"None\". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies", "type": "string" }, + "sessionAffinityConfig": { + "description": "sessionAffinityConfig contains the configurations of session affinity.", + "$ref": "#/definitions/io.k8s.api.core.v1.SessionAffinityConfig" + }, "type": { "description": "type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. \"ExternalName\" maps to the specified externalName. \"ClusterIP\" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object. If clusterIP is \"None\", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a stable IP. \"NodePort\" builds on ClusterIP and allocates a port on every node which routes to the clusterIP. \"LoadBalancer\" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services---service-types", "type": "string" @@ -11883,6 +11903,15 @@ } } }, + "io.k8s.api.core.v1.SessionAffinityConfig": { + "description": "SessionAffinityConfig represents the configurations of session affinity.", + "properties": { + "clientIP": { + "description": "clientIP contains the configurations of Client IP based session affinity.", + "$ref": "#/definitions/io.k8s.api.core.v1.ClientIPConfig" + } + } + }, "io.k8s.api.core.v1.StorageOSVolumeSource": { "description": "Represents a StorageOS persistent volume resource.", "properties": { @@ -11933,9 +11962,7 @@ }, "key": { "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", - "type": "string", - "x-kubernetes-patch-merge-key": "key", - "x-kubernetes-patch-strategy": "merge" + "type": "string" }, "operator": { "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", @@ -12269,7 +12296,7 @@ "collisionCount": { "description": "Count of hash collisions for the DaemonSet. The DaemonSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision.", "type": "integer", - "format": "int64" + "format": "int32" }, "currentNumberScheduled": { "description": "The number of nodes that are running at least 1 daemon pod and are supposed to run the daemon pod. More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/", @@ -12503,6 +12530,7 @@ }, "strategy": { "description": "The deployment strategy to use to replace existing pods with new ones.", + "x-kubernetes-patch-strategy": "retainKeys", "$ref": "#/definitions/io.k8s.api.extensions.v1beta1.DeploymentStrategy" }, "template": { @@ -12522,7 +12550,7 @@ "collisionCount": { "description": "Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet.", "type": "integer", - "format": "int64" + "format": "int32" }, "conditions": { "description": "Represents the latest available observations of a deployment's current state.", @@ -13414,7 +13442,9 @@ "type": "array", "items": { "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Initializer" - } + }, + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" }, "result": { "description": "If result is set with the Failure field, the object will be persisted to storage and then deleted, ensuring that other clients can observe the deletion.", diff --git a/federation/apis/swagger-spec/extensions_v1beta1.json b/federation/apis/swagger-spec/extensions_v1beta1.json index 07185c5cd8ea8..80b71dc366579 100644 --- a/federation/apis/swagger-spec/extensions_v1beta1.json +++ b/federation/apis/swagger-spec/extensions_v1beta1.json @@ -5186,10 +5186,18 @@ "properties": { "path": { "type": "string", - "description": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + "description": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" + }, + "type": { + "$ref": "v1.HostPathType", + "description": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath" } } }, + "v1.HostPathType": { + "id": "v1.HostPathType", + "properties": {} + }, "v1.EmptyDirVolumeSource": { "id": "v1.EmptyDirVolumeSource", "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", @@ -5400,6 +5408,10 @@ "secretRef": { "$ref": "v1.LocalObjectReference", "description": "CHAP secret for iSCSI target and initiator authentication" + }, + "initiatorName": { + "type": "string", + "description": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection." } } }, @@ -6282,7 +6294,7 @@ }, "resourceFieldRef": { "$ref": "v1.ResourceFieldSelector", - "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported." + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported." }, "configMapKeyRef": { "$ref": "v1.ConfigMapKeySelector", @@ -6953,7 +6965,7 @@ }, "collisionCount": { "type": "integer", - "format": "int64", + "format": "int32", "description": "Count of hash collisions for the DaemonSet. The DaemonSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision." } } @@ -7212,7 +7224,7 @@ }, "collisionCount": { "type": "integer", - "format": "int64", + "format": "int32", "description": "Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet." } } diff --git a/federation/apis/swagger-spec/v1.json b/federation/apis/swagger-spec/v1.json index de5cbc562de16..54ca88fed9a84 100644 --- a/federation/apis/swagger-spec/v1.json +++ b/federation/apis/swagger-spec/v1.json @@ -5128,6 +5128,10 @@ "publishNotReadyAddresses": { "type": "boolean", "description": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery. This field will replace the service.alpha.kubernetes.io/tolerate-unready-endpoints when that annotation is deprecated and all clients have been converted to use this field." + }, + "sessionAffinityConfig": { + "$ref": "v1.SessionAffinityConfig", + "description": "sessionAffinityConfig contains the configurations of session affinity." } } }, @@ -5162,6 +5166,27 @@ } } }, + "v1.SessionAffinityConfig": { + "id": "v1.SessionAffinityConfig", + "description": "SessionAffinityConfig represents the configurations of session affinity.", + "properties": { + "clientIP": { + "$ref": "v1.ClientIPConfig", + "description": "clientIP contains the configurations of Client IP based session affinity." + } + } + }, + "v1.ClientIPConfig": { + "id": "v1.ClientIPConfig", + "description": "ClientIPConfig represents the configurations of Client IP based session affinity.", + "properties": { + "timeoutSeconds": { + "type": "integer", + "format": "int32", + "description": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be \u003e0 \u0026\u0026 \u003c=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours)." + } + } + }, "v1.ServiceStatus": { "id": "v1.ServiceStatus", "description": "ServiceStatus represents the current status of a service.", diff --git a/federation/cmd/federation-apiserver/app/BUILD b/federation/cmd/federation-apiserver/app/BUILD index 1b7fd40efb3a5..1b43b5d483330 100644 --- a/federation/cmd/federation-apiserver/app/BUILD +++ b/federation/cmd/federation-apiserver/app/BUILD @@ -66,6 +66,7 @@ go_library( "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/api/apps/v1beta2:go_default_library", "//vendor/k8s.io/api/autoscaling/v1:go_default_library", "//vendor/k8s.io/api/batch/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/federation/cmd/federation-apiserver/app/server.go b/federation/cmd/federation-apiserver/app/server.go index fd169f7471b9b..9f25c9a8bcd80 100644 --- a/federation/cmd/federation-apiserver/app/server.go +++ b/federation/cmd/federation-apiserver/app/server.go @@ -30,6 +30,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + appsv1beta2 "k8s.io/api/apps/v1beta2" apiv1 "k8s.io/api/core/v1" extensionsapiv1beta1 "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -286,6 +287,12 @@ func defaultResourceConfig() *serverstorage.ResourceConfig { extensionsapiv1beta1.SchemeGroupVersion.WithResource("ingresses"), extensionsapiv1beta1.SchemeGroupVersion.WithResource("replicasets"), ) + // All apps resources except these are disabled by default. + rc.EnableResources( + appsv1beta2.SchemeGroupVersion.WithResource("daemonsets"), + appsv1beta2.SchemeGroupVersion.WithResource("deployments"), + appsv1beta2.SchemeGroupVersion.WithResource("replicasets"), + ) return rc } diff --git a/federation/docs/api-reference/extensions/v1beta1/definitions.html b/federation/docs/api-reference/extensions/v1beta1/definitions.html index 63e825ed50878..c3828c216fe7c 100755 --- a/federation/docs/api-reference/extensions/v1beta1/definitions.html +++ b/federation/docs/api-reference/extensions/v1beta1/definitions.html @@ -482,7 +482,7 @@

v1beta1.DeploymentStatus

collisionCount

Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet.

false

-

integer (int64)

+

integer (int32)

@@ -675,7 +675,7 @@

v1beta1.DaemonSetStatus

collisionCount

Count of hash collisions for the DaemonSet. The DaemonSet controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ControllerRevision.

false

-

integer (int64)

+

integer (int32)

@@ -2361,6 +2361,13 @@

v1.ISCSIVolumeSource

v1.LocalObjectReference

+ +

initiatorName

+

Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface <target portal>:<volume name> will be created for the connection.

+

false

+

string

+ + @@ -2941,7 +2948,7 @@

v1.EnvVarSource

resourceFieldRef

-

Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.

+

Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.

false

v1.ResourceFieldSelector

@@ -4868,6 +4875,10 @@

v1.ScaleIOVolumeSource

+ +
+

v1.HostPathType

+

v1.Initializers

@@ -6227,11 +6238,18 @@

v1.HostPathVolumeSource

path

-

Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath

+

Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath

true

string

+ +

type

+

Type for HostPath Volume Defaults to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath

+

false

+

v1.HostPathType

+ + diff --git a/federation/docs/api-reference/v1/definitions.html b/federation/docs/api-reference/v1/definitions.html index b4ba4127bbd4c..f02c87030702e 100755 --- a/federation/docs/api-reference/v1/definitions.html +++ b/federation/docs/api-reference/v1/definitions.html @@ -2049,6 +2049,40 @@

v1.EventList

+
+
+

v1.ClientIPConfig

+
+

ClientIPConfig represents the configurations of Client IP based session affinity.

+
+ +++++++ + + + + + + + + + + + + + + + + + + +
NameDescriptionRequiredSchemaDefault

timeoutSeconds

timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && ⇐86400(for 1 day) if ServiceAffinity == "ClientIP". Default value is 10800(for 3 hours).

false

integer (int32)

+

v1.APIResource

@@ -2233,6 +2267,13 @@

v1.ServiceSpec

boolean

false

+ +

sessionAffinityConfig

+

sessionAffinityConfig contains the configurations of session affinity.

+

false

+

v1.SessionAffinityConfig

+ + @@ -2333,6 +2374,40 @@

v1.StatusCause

+
+
+

v1.SessionAffinityConfig

+
+

SessionAffinityConfig represents the configurations of session affinity.

+
+ +++++++ + + + + + + + + + + + + + + + + + + +
NameDescriptionRequiredSchemaDefault

clientIP

clientIP contains the configurations of Client IP based session affinity.

false

v1.ClientIPConfig

+

any

diff --git a/federation/pkg/federation-controller/job/jobcontroller.go b/federation/pkg/federation-controller/job/jobcontroller.go index d3977182fc1e9..368ca1c396cee 100644 --- a/federation/pkg/federation-controller/job/jobcontroller.go +++ b/federation/pkg/federation-controller/job/jobcontroller.go @@ -468,12 +468,12 @@ func (fjc *FederationJobController) reconcileJob(key string) (reconciliationStat for _, condition := range currentLjob.Status.Conditions { if condition.Type == batchv1.JobComplete { if fedStatusCompleteCondition == nil || - fedStatusCompleteCondition.LastTransitionTime.Before(condition.LastTransitionTime) { + fedStatusCompleteCondition.LastTransitionTime.Before(&condition.LastTransitionTime) { fedStatusCompleteCondition = &condition } } else if condition.Type == batchv1.JobFailed { if fedStatusFailedCondition == nil || - fedStatusFailedCondition.LastTransitionTime.Before(condition.LastTransitionTime) { + fedStatusFailedCondition.LastTransitionTime.Before(&condition.LastTransitionTime) { fedStatusFailedCondition = &condition } } @@ -484,7 +484,7 @@ func (fjc *FederationJobController) reconcileJob(key string) (reconciliationStat } } if currentLjob.Status.CompletionTime != nil { - if fedStatus.CompletionTime == nil || fedStatus.CompletionTime.Before(*currentLjob.Status.CompletionTime) { + if fedStatus.CompletionTime == nil || fedStatus.CompletionTime.Before(currentLjob.Status.CompletionTime) { fedStatus.CompletionTime = currentLjob.Status.CompletionTime } } diff --git a/federation/pkg/federation-controller/service/dns/dns.go b/federation/pkg/federation-controller/service/dns/dns.go index 73167f85268f7..e478d76e6bfa8 100644 --- a/federation/pkg/federation-controller/service/dns/dns.go +++ b/federation/pkg/federation-controller/service/dns/dns.go @@ -182,6 +182,7 @@ func (s *ServiceDNSController) workerFunction() bool { for _, clusterIngress := range ingress.Items { err = s.ensureDNSRecords(clusterIngress.Cluster, service) if err != nil { + runtime.HandleError(fmt.Errorf("Error when ensuring DNS records for service %s/%s: %v", service.Namespace, service.Name, err)) s.deliverService(service, 0, true) } } diff --git a/federation/pkg/kubefed/init/init.go b/federation/pkg/kubefed/init/init.go index 31f000e3e2cd7..2599c45843964 100644 --- a/federation/pkg/kubefed/init/init.go +++ b/federation/pkg/kubefed/init/init.go @@ -284,7 +284,7 @@ func (i *initFederation) Run(cmdOut io.Writer, config util.AdminConfig) error { return err } // If the error is type NoRBACAPIError, We continue to create the rest of - // the resources, without the SA and roles (in the abscense of RBAC support). + // the resources, without the SA and roles (in the absence of RBAC support). rbacAvailable = false } diff --git a/federation/pkg/kubefed/util/util.go b/federation/pkg/kubefed/util/util.go index eba6349dfb053..427bbc2798230 100644 --- a/federation/pkg/kubefed/util/util.go +++ b/federation/pkg/kubefed/util/util.go @@ -262,7 +262,7 @@ func buildConfigFromSecret(secret *api.Secret, serverAddress string) (*restclien // GetVersionedClientForRBACOrFail discovers the versioned rbac APIs and gets the versioned // clientset for either the preferred version or the first listed version (if no preference listed) -// TODO: We need to evaluate the usage of RESTMapper interface to achieve te same functionality +// TODO: We need to evaluate the usage of RESTMapper interface to achieve the same functionality func GetVersionedClientForRBACOrFail(hostFactory cmdutil.Factory) (client.Interface, error) { discoveryclient, err := hostFactory.DiscoveryClient() if err != nil { diff --git a/hack/.golint_failures b/hack/.golint_failures index dd91722987e2d..32330dca2e086 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -1,7 +1,6 @@ cluster/images/etcd-version-monitor cmd/cloud-controller-manager/app cmd/cloud-controller-manager/app/options -cmd/genslateyaml cmd/genutils cmd/gke-certificates-controller/app cmd/hyperkube @@ -29,7 +28,6 @@ cmd/kubectl/app cmd/kubelet/app cmd/kubelet/app/options cmd/kubemark -cmd/mungedocs examples/guestbook-go federation/apis/core federation/apis/core/v1 @@ -109,7 +107,6 @@ pkg/apis/authorization/v1 pkg/apis/authorization/v1beta1 pkg/apis/authorization/validation pkg/apis/autoscaling -pkg/apis/autoscaling/v2alpha1 pkg/apis/autoscaling/validation pkg/apis/batch pkg/apis/batch/validation @@ -282,6 +279,7 @@ pkg/kubelet/server/portforward pkg/kubelet/server/remotecommand pkg/kubelet/server/stats pkg/kubelet/server/streaming +pkg/kubelet/stats pkg/kubelet/status pkg/kubelet/status/testing pkg/kubelet/sysctl @@ -753,6 +751,28 @@ staging/src/k8s.io/client-go/util/integer staging/src/k8s.io/client-go/util/jsonpath staging/src/k8s.io/client-go/util/retry staging/src/k8s.io/client-go/util/testing +staging/src/k8s.io/code-generator/cmd/client-gen/args +staging/src/k8s.io/code-generator/cmd/client-gen/generators/fake +staging/src/k8s.io/code-generator/cmd/client-gen/generators/scheme +staging/src/k8s.io/code-generator/cmd/client-gen/types +staging/src/k8s.io/code-generator/cmd/conversion-gen/generators +staging/src/k8s.io/code-generator/cmd/go-to-protobuf/protobuf +staging/src/k8s.io/code-generator/cmd/informer-gen/generators +staging/src/k8s.io/code-generator/cmd/lister-gen/generators +staging/src/k8s.io/code-generator/test/apis/testgroup +staging/src/k8s.io/code-generator/test/apis/testgroup/v1 +staging/src/k8s.io/code-generator/test/clientset/internal +staging/src/k8s.io/code-generator/test/clientset/internal/fake +staging/src/k8s.io/code-generator/test/clientset/internal/scheme +staging/src/k8s.io/code-generator/test/clientset/internal/typed/testgroup/internalversion +staging/src/k8s.io/code-generator/test/clientset/internal/typed/testgroup/internalversion/fake +staging/src/k8s.io/code-generator/test/clientset/versioned +staging/src/k8s.io/code-generator/test/clientset/versioned/fake +staging/src/k8s.io/code-generator/test/clientset/versioned/scheme +staging/src/k8s.io/code-generator/test/clientset/versioned/typed/testgroup/v1 +staging/src/k8s.io/code-generator/test/clientset/versioned/typed/testgroup/v1/fake +staging/src/k8s.io/code-generator/test/informers/externalversions/internalinterfaces +staging/src/k8s.io/code-generator/test/informers/internalversion/internalinterfaces staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1 staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/validation @@ -764,28 +784,6 @@ staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister staging/src/k8s.io/kube-aggregator/pkg/controllers/status staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice staging/src/k8s.io/kube-aggregator/pkg/registry/apiservice/etcd -staging/src/k8s.io/kube-gen/cmd/client-gen/args -staging/src/k8s.io/kube-gen/cmd/client-gen/generators/fake -staging/src/k8s.io/kube-gen/cmd/client-gen/generators/scheme -staging/src/k8s.io/kube-gen/cmd/client-gen/types -staging/src/k8s.io/kube-gen/cmd/conversion-gen/generators -staging/src/k8s.io/kube-gen/cmd/go-to-protobuf/protobuf -staging/src/k8s.io/kube-gen/cmd/informer-gen/generators -staging/src/k8s.io/kube-gen/cmd/lister-gen/generators -staging/src/k8s.io/kube-gen/test/apis/testgroup -staging/src/k8s.io/kube-gen/test/apis/testgroup/v1 -staging/src/k8s.io/kube-gen/test/clientset/internal -staging/src/k8s.io/kube-gen/test/clientset/internal/fake -staging/src/k8s.io/kube-gen/test/clientset/internal/scheme -staging/src/k8s.io/kube-gen/test/clientset/internal/typed/testgroup/internalversion -staging/src/k8s.io/kube-gen/test/clientset/internal/typed/testgroup/internalversion/fake -staging/src/k8s.io/kube-gen/test/clientset/versioned -staging/src/k8s.io/kube-gen/test/clientset/versioned/fake -staging/src/k8s.io/kube-gen/test/clientset/versioned/scheme -staging/src/k8s.io/kube-gen/test/clientset/versioned/typed/testgroup/v1 -staging/src/k8s.io/kube-gen/test/clientset/versioned/typed/testgroup/v1/fake -staging/src/k8s.io/kube-gen/test/informers/externalversions/internalinterfaces -staging/src/k8s.io/kube-gen/test/informers/internalversion/internalinterfaces staging/src/k8s.io/metrics/pkg/apis/custom_metrics staging/src/k8s.io/metrics/pkg/apis/custom_metrics/v1alpha1 staging/src/k8s.io/metrics/pkg/apis/metrics diff --git a/hack/e2e.go b/hack/e2e.go index bc58a0ad8cf74..c016926c4a494 100644 --- a/hack/e2e.go +++ b/hack/e2e.go @@ -25,6 +25,7 @@ import ( "log" "os" "os/exec" + "os/signal" "path/filepath" "strings" "time" @@ -81,12 +82,21 @@ func main() { } func wait(cmd string, args ...string) error { + sigChannel := make(chan os.Signal, 1) + signal.Notify(sigChannel, os.Interrupt) + c := exec.Command(cmd, args...) c.Stdout = os.Stdout c.Stderr = os.Stderr if err := c.Start(); err != nil { return err } + go func() { + sig := <-sigChannel + if err := c.Process.Signal(sig); err != nil { + log.Fatalf("could not send %s signal %s: %v", cmd, sig, err) + } + }() return c.Wait() } diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index 005ee32820577..79e9dbdff5dfd 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -134,7 +134,6 @@ kube::golang::test_targets() { cmd/genkubedocs cmd/genman cmd/genyaml - cmd/mungedocs cmd/genswaggertypedocs cmd/linkcheck federation/cmd/genfeddocs diff --git a/hack/lib/protoc.sh b/hack/lib/protoc.sh index b1e8cb60c956b..980684cbd8ad7 100644 --- a/hack/lib/protoc.sh +++ b/hack/lib/protoc.sh @@ -28,7 +28,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh" function kube::protoc::generate_proto() { kube::golang::setup_env local bins=( - vendor/k8s.io/kube-gen/cmd/go-to-protobuf/protoc-gen-gogo + vendor/k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo ) make -C "${KUBE_ROOT}" WHAT="${bins[*]}" diff --git a/hack/lib/util.sh b/hack/lib/util.sh index 5154806ac6e22..8e96ad462b1d0 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -331,7 +331,7 @@ kube::util::group-version-to-pkg-path() { return fi - # "v1" is the API GroupVersion + # "v1" is the API GroupVersion if [[ "${group_version}" == "v1" ]]; then echo "vendor/k8s.io/api/core/v1" return @@ -437,6 +437,23 @@ kube::util::git_upstream_remote_name() { head -n 1 | awk '{print $1}' } +# Ensures the current directory is a git tree for doing things like restoring or +# validating godeps +kube::util::create-fake-git-tree() { + local -r target_dir=${1:-$(pwd)} + + pushd "${target_dir}" >/dev/null + git init >/dev/null + git config --local user.email "nobody@k8s.io" + git config --local user.name "$0" + git add . >/dev/null + git commit -q -m "Snapshot" >/dev/null + if (( ${KUBE_VERBOSE:-5} >= 6 )); then + kube::log::status "${target_dir} is now a git tree." + fi + popd >/dev/null +} + # Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist # and are checked out to the referenced rev. kube::util::godep_restored() { @@ -505,7 +522,7 @@ kube::util::ensure_godep_version() { kube::util::ensure-temp-dir mkdir -p "${KUBE_TEMP}/go/src" - GOPATH="${KUBE_TEMP}/go" go get -d -u github.com/tools/godep 2>/dev/null + GOPATH="${KUBE_TEMP}/go" go get -d -u github.com/tools/godep pushd "${KUBE_TEMP}/go/src/github.com/tools/godep" >/dev/null git checkout -q "${GODEP_VERSION}" GOPATH="${KUBE_TEMP}/go" go install . @@ -539,7 +556,7 @@ kube::util::go_install_from_commit() { kube::util::ensure-temp-dir mkdir -p "${KUBE_TEMP}/go/src" - GOPATH="${KUBE_TEMP}/go" go get -d -u "${pkg}" 2>/dev/null + GOPATH="${KUBE_TEMP}/go" go get -d -u "${pkg}" ( cd "${KUBE_TEMP}/go/src/${pkg}" git checkout -q "${commit}" diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 01b2541942a71..7fa308657bc52 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -31,6 +31,8 @@ KUBELET_AUTHORIZATION_WEBHOOK=${KUBELET_AUTHORIZATION_WEBHOOK:-""} KUBELET_AUTHENTICATION_WEBHOOK=${KUBELET_AUTHENTICATION_WEBHOOK:-""} POD_MANIFEST_PATH=${POD_MANIFEST_PATH:-"/var/run/kubernetes/static-pods"} KUBELET_FLAGS=${KUBELET_FLAGS:-""} +# many dev environments run with swap on, so we don't fail in this env +FAIL_SWAP_ON=${FAIL_SWAP_ON:-"false"} # Name of the network plugin, eg: "kubenet" NET_PLUGIN=${NET_PLUGIN:-""} # Place the config files and binaries required by NET_PLUGIN in these directory, @@ -112,6 +114,11 @@ if [ "${CLOUD_PROVIDER}" == "openstack" ]; then fi fi +# warn if users are running with swap allowed +if [ "${FAIL_SWAP_ON}" == "false" ]; then + echo "WARNING : The kubelet is configured to not fail if swap is enabled; production deployments should disable swap." +fi + if [ "$(id -u)" != "0" ]; then echo "WARNING : This script MAY be run as root for docker socket / iptables functionality; if failures occur, retry as root." 2>&1 fi @@ -665,6 +672,7 @@ function start_kubelet { --eviction-soft=${EVICTION_SOFT} \ --eviction-pressure-transition-period=${EVICTION_PRESSURE_TRANSITION_PERIOD} \ --pod-manifest-path="${POD_MANIFEST_PATH}" \ + --fail-swap-on="${FAIL_SWAP_ON}" \ ${auth_args} \ ${dns_args} \ ${cni_conf_dir_args} \ diff --git a/hack/make-rules/helpers/cache_go_dirs.sh b/hack/make-rules/helpers/cache_go_dirs.sh index dd385e622d529..a16f78b8c0299 100755 --- a/hack/make-rules/helpers/cache_go_dirs.sh +++ b/hack/make-rules/helpers/cache_go_dirs.sh @@ -39,7 +39,7 @@ function kfind() { # include the "special" vendor directories which are actually part # of the Kubernetes source tree - generators will use these for # including certain core API concepts. - find -H . ./vendor/k8s.io/apimachinery ./vendor/k8s.io/apiserver ./vendor/k8s.io/kube-aggregator ./vendor/k8s.io/apiextensions-apiserver ./vendor/k8s.io/metrics ./vendor/k8s.io/sample-apiserver ./vendor/k8s.io/api ./vendor/k8s.io/client-go ./vendor/k8s.io/kube-gen \ + find -H . ./vendor/k8s.io/apimachinery ./vendor/k8s.io/apiserver ./vendor/k8s.io/kube-aggregator ./vendor/k8s.io/apiextensions-apiserver ./vendor/k8s.io/metrics ./vendor/k8s.io/sample-apiserver ./vendor/k8s.io/api ./vendor/k8s.io/client-go ./vendor/k8s.io/code-generator \ \( \ -not \( \ \( \ diff --git a/hack/make-rules/test-cmd-util.sh b/hack/make-rules/test-cmd-util.sh index 0cd61775f1ff2..a7ac65f198fe6 100755 --- a/hack/make-rules/test-cmd-util.sh +++ b/hack/make-rules/test-cmd-util.sh @@ -41,9 +41,11 @@ IMAGE_NGINX="gcr.io/google-containers/nginx:1.7.9" IMAGE_DEPLOYMENT_R1="gcr.io/google-containers/nginx:test-cmd" # deployment-revision1.yaml IMAGE_DEPLOYMENT_R2="$IMAGE_NGINX" # deployment-revision2.yaml IMAGE_PERL="gcr.io/google-containers/perl" -IMAGE_DAEMONSET_R1="gcr.io/google-containers/pause:2.0" +IMAGE_PAUSE_V2="gcr.io/google-containers/pause:2.0" IMAGE_DAEMONSET_R2="gcr.io/google-containers/pause:latest" IMAGE_DAEMONSET_R2_2="gcr.io/google-containers/nginx:test-cmd" # rollingupdate-daemonset-rv2.yaml +IMAGE_STATEFULSET_R1="gcr.io/google_containers/nginx-slim:0.7" +IMAGE_STATEFULSET_R2="gcr.io/google_containers/nginx-slim:0.8" # Expose kubectl directly for readability PATH="${KUBE_OUTPUT_HOSTBIN}":$PATH @@ -173,6 +175,9 @@ function cleanup() kube::etcd::cleanup rm -rf "${KUBE_TEMP}" + local junit_dir="${KUBE_JUNIT_REPORT_DIR:-/tmp/junit-results}" + echo "junit report dir:" ${junit_dir} + kube::log::status "Clean up complete" } @@ -927,6 +932,30 @@ run_kubectl_apply_tests() { kubectl delete pods test-pod "${kube_flags[@]}" + ## kubectl apply should be able to clear defaulted fields. + # Pre-Condition: no deployment exists + kube::test::get_object_assert deployments "{{range.items}}{{$id_field}}:{{end}}" '' + # Command: apply a deployment "test-deployment-retainkeys" (doesn't exist) should create this deployment + kubectl apply -f hack/testdata/retainKeys/deployment/deployment-before.yaml "${kube_flags[@]}" + # Post-Condition: deployment "test-deployment-retainkeys" created + kube::test::get_object_assert deployments "{{range.items}}{{$id_field}}{{end}}" 'test-deployment-retainkeys' + # Post-Condition: deployment "test-deployment-retainkeys" has defaulted fields + [[ "$(kubectl get deployments test-deployment-retainkeys -o yaml "${kube_flags[@]}" | grep RollingUpdate)" ]] + [[ "$(kubectl get deployments test-deployment-retainkeys -o yaml "${kube_flags[@]}" | grep maxSurge)" ]] + [[ "$(kubectl get deployments test-deployment-retainkeys -o yaml "${kube_flags[@]}" | grep maxUnavailable)" ]] + [[ "$(kubectl get deployments test-deployment-retainkeys -o yaml "${kube_flags[@]}" | grep emptyDir)" ]] + # Command: apply a deployment "test-deployment-retainkeys" should clear + # defaulted fields and successfully update the deployment + [[ "$(kubectl apply -f hack/testdata/retainKeys/deployment/deployment-after.yaml "${kube_flags[@]}")" ]] + # Post-Condition: deployment "test-deployment-retainkeys" has updated fields + [[ "$(kubectl get deployments test-deployment-retainkeys -o yaml "${kube_flags[@]}" | grep Recreate)" ]] + ! [[ "$(kubectl get deployments test-deployment-retainkeys -o yaml "${kube_flags[@]}" | grep RollingUpdate)" ]] + [[ "$(kubectl get deployments test-deployment-retainkeys -o yaml "${kube_flags[@]}" | grep hostPath)" ]] + ! [[ "$(kubectl get deployments test-deployment-retainkeys -o yaml "${kube_flags[@]}" | grep emptyDir)" ]] + # Clean up + kubectl delete deployments test-deployment-retainkeys "${kube_flags[@]}" + + ## kubectl apply -f with label selector should only apply matching objects # Pre-Condition: no POD exists kube::test::get_object_assert pods "{{range.items}}{{$id_field}}:{{end}}" '' @@ -1234,31 +1263,6 @@ run_kubectl_run_tests() { set +o errexit } -run_kubectl_using_deprecated_commands_test() { - set -o nounset - set -o errexit - - create_and_use_new_namespace - kube::log::status "Testing kubectl using deprecated commands" - ## `kubectl run-container` should function identical to `kubectl run`, but it - ## should also print a deprecation warning. - # Pre-Condition: no Job exists - kube::test::get_object_assert jobs "{{range.items}}{{$id_field}}:{{end}}" '' - # Command - output_message=$(kubectl 2>&1 run-container pi --generator=job/v1 "--image=$IMAGE_PERL" --restart=OnFailure -- perl -Mbignum=bpi -wle 'print bpi(15)' "${kube_flags[@]}") - # Ensure that the user is warned their command is deprecated. - kube::test::if_has_string "${output_message}" 'deprecated' - # Post-Condition: Job "pi" is created - kube::test::get_object_assert jobs "{{range.items}}{{$id_field}}:{{end}}" 'pi:' - # Clean up - kubectl delete jobs pi "${kube_flags[@]}" - # Post-condition: no pods exist. - kube::test::get_object_assert pods "{{range.items}}{{$id_field}}:{{end}}" '' - - set +o nounset - set +o errexit -} - run_kubectl_get_tests() { set -o nounset set -o errexit @@ -1884,7 +1888,7 @@ run_recursive_resources_tests() { # Create a deployment (revision 1) kubectl create -f hack/testdata/deployment-revision1.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Command output_message=$(kubectl convert --local -f hack/testdata/deployment-revision1.yaml --output-version=apps/v1beta1 -o go-template='{{ .apiVersion }}' "${kube_flags[@]}") echo $output_message @@ -2007,11 +2011,11 @@ run_recursive_resources_tests() { # Create deployments (revision 1) recursively from directory of YAML files ! kubectl create -f hack/testdata/recursive/deployment --recursive "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx0-deployment:nginx1-deployment:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_NGINX}:${IMAGE_NGINX}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_NGINX}:${IMAGE_NGINX}:" ## Rollback the deployments to revision 1 recursively output_message=$(! kubectl rollout undo -f hack/testdata/recursive/deployment --recursive --to-revision=1 2>&1 "${kube_flags[@]}") # Post-condition: nginx0 & nginx1 should be a no-op, and since nginx2 is malformed, it should error - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_NGINX}:${IMAGE_NGINX}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_NGINX}:${IMAGE_NGINX}:" kube::test::if_has_string "${output_message}" "Object 'Kind' is missing" ## Pause the deployments recursively PRESERVE_ERR_FILE=true @@ -2646,8 +2650,8 @@ run_rc_tests() { # Create a deployment kubectl create -f hack/testdata/deployment-multicontainer-resources.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx-deployment-resources:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set the deployment's cpu limits kubectl set resources deployment nginx-deployment-resources --limits=cpu=100m "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{(index .spec.template.spec.containers 0).resources.limits.cpu}}:{{end}}" "100m:" @@ -2778,27 +2782,27 @@ run_deployment_tests() { # Create a deployment (revision 1) kubectl create -f hack/testdata/deployment-revision1.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Rollback to revision 1 - should be no-op kubectl rollout undo deployment nginx --to-revision=1 "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Update the deployment (revision 2) kubectl apply -f hack/testdata/deployment-revision2.yaml "${kube_flags[@]}" - kube::test::get_object_assert deployment.extensions "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment.extensions "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" # Rollback to revision 1 with dry-run - should be no-op kubectl rollout undo deployment nginx --dry-run=true "${kube_flags[@]}" | grep "test-cmd" - kube::test::get_object_assert deployment.extensions "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment.extensions "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" # Rollback to revision 1 kubectl rollout undo deployment nginx --to-revision=1 "${kube_flags[@]}" sleep 1 - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Rollback to revision 1000000 - should be no-op kubectl rollout undo deployment nginx --to-revision=1000000 "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Rollback to last revision kubectl rollout undo deployment nginx "${kube_flags[@]}" sleep 1 - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" # Pause the deployment kubectl-with-retry rollout pause deployment nginx "${kube_flags[@]}" # A paused deployment cannot be rolled back @@ -2824,30 +2828,35 @@ run_deployment_tests() { # Create a deployment kubectl create -f hack/testdata/deployment-multicontainer.yaml "${kube_flags[@]}" kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx-deployment:' - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set the deployment's image kubectl set image deployment nginx-deployment nginx="${IMAGE_DEPLOYMENT_R2}" "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set non-existing container should fail ! kubectl set image deployment nginx-deployment redis=redis "${kube_flags[@]}" # Set image of deployments without specifying name kubectl set image deployments --all nginx="${IMAGE_DEPLOYMENT_R1}" "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set image of a deployment specified by file kubectl set image -f hack/testdata/deployment-multicontainer.yaml nginx="${IMAGE_DEPLOYMENT_R2}" "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set image of a local file without talking to the server kubectl set image -f hack/testdata/deployment-multicontainer.yaml nginx="${IMAGE_DEPLOYMENT_R1}" "${kube_flags[@]}" --local -o yaml - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_PERL}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R2}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PERL}:" # Set image of all containers of the deployment kubectl set image deployment nginx-deployment "*"="${IMAGE_DEPLOYMENT_R1}" "${kube_flags[@]}" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" - kube::test::get_object_assert deployment "{{range.items}}{{$deployment_second_image_field}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + # Set image of all containners of the deployment again when image not change + kubectl set image deployment nginx-deployment "*"="${IMAGE_DEPLOYMENT_R1}" "${kube_flags[@]}" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + kube::test::get_object_assert deployment "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" + # Clean up kubectl delete deployment nginx-deployment "${kube_flags[@]}" @@ -3025,35 +3034,35 @@ run_daemonset_history_tests() { # Command # Create a DaemonSet (revision 1) kubectl apply -f hack/testdata/rollingupdate-daemonset.yaml --record "${kube_flags[@]}" - kube::test::get_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-daemonset.yaml --record.*" + kube::test::wait_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-daemonset.yaml --record.*" # Rollback to revision 1 - should be no-op kubectl rollout undo daemonset --to-revision=1 "${kube_flags[@]}" - kube::test::get_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R1}:" + kube::test::get_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_PAUSE_V2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "1" # Update the DaemonSet (revision 2) kubectl apply -f hack/testdata/rollingupdate-daemonset-rv2.yaml --record "${kube_flags[@]}" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "2" kube::test::wait_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-daemonset-rv2.yaml --record.*" # Rollback to revision 1 with dry-run - should be no-op kubectl rollout undo daemonset --dry-run=true "${kube_flags[@]}" - kube::test::get_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" - kube::test::get_object_assert daemonset "{{range.items}}{{$daemonset_image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" + kube::test::get_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" + kube::test::get_object_assert daemonset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "2" # Rollback to revision 1 kubectl rollout undo daemonset --to-revision=1 "${kube_flags[@]}" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R1}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_PAUSE_V2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "1" # Rollback to revision 1000000 - should fail output_message=$(! kubectl rollout undo daemonset --to-revision=1000000 "${kube_flags[@]}" 2>&1) kube::test::if_has_string "${output_message}" "unable to find specified revision" - kube::test::get_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R1}:" + kube::test::get_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_PAUSE_V2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "1" # Rollback to last revision kubectl rollout undo daemonset "${kube_flags[@]}" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" - kube::test::wait_object_assert daemonset "{{range.items}}{{$daemonset_image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_DAEMONSET_R2}:" + kube::test::wait_object_assert daemonset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_DAEMONSET_R2_2}:" kube::test::get_object_assert daemonset "{{range.items}}{{$container_len}}{{end}}" "2" # Clean up kubectl delete -f hack/testdata/rollingupdate-daemonset.yaml "${kube_flags[@]}" @@ -3062,6 +3071,58 @@ run_daemonset_history_tests() { set +o errexit } +run_statefulset_history_tests() { + set -o nounset + set -o errexit + + create_and_use_new_namespace + kube::log::status "Testing kubectl(v1:statefulsets, v1:controllerrevisions)" + + ### Test rolling back a StatefulSet + # Pre-condition: no statefulset or its pods exists + kube::test::get_object_assert statefulset "{{range.items}}{{$id_field}}:{{end}}" '' + # Command + # Create a StatefulSet (revision 1) + kubectl apply -f hack/testdata/rollingupdate-statefulset.yaml --record "${kube_flags[@]}" + kube::test::wait_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-statefulset.yaml --record.*" + # Rollback to revision 1 - should be no-op + kubectl rollout undo statefulset --to-revision=1 "${kube_flags[@]}" + kube::test::get_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R1}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "1" + # Update the statefulset (revision 2) + kubectl apply -f hack/testdata/rollingupdate-statefulset-rv2.yaml --record "${kube_flags[@]}" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R2}:" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PAUSE_V2}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "2" + kube::test::wait_object_assert controllerrevisions "{{range.items}}{{$annotations_field}}:{{end}}" ".*rollingupdate-statefulset-rv2.yaml --record.*" + # Rollback to revision 1 with dry-run - should be no-op + kubectl rollout undo statefulset --dry-run=true "${kube_flags[@]}" + kube::test::get_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R2}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PAUSE_V2}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "2" + # Rollback to revision 1 + kubectl rollout undo statefulset --to-revision=1 "${kube_flags[@]}" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R1}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "1" + # Rollback to revision 1000000 - should fail + output_message=$(! kubectl rollout undo statefulset --to-revision=1000000 "${kube_flags[@]}" 2>&1) + kube::test::if_has_string "${output_message}" "unable to find specified revision" + kube::test::get_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R1}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "1" + # Rollback to last revision + kubectl rollout undo statefulset "${kube_flags[@]}" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field0}}:{{end}}" "${IMAGE_STATEFULSET_R2}:" + kube::test::wait_object_assert statefulset "{{range.items}}{{$image_field1}}:{{end}}" "${IMAGE_PAUSE_V2}:" + kube::test::get_object_assert statefulset "{{range.items}}{{$container_len}}{{end}}" "2" + # Clean up - delete newest configuration + kubectl delete -f hack/testdata/rollingupdate-statefulset-rv2.yaml "${kube_flags[@]}" + # Post-condition: no pods from statefulset controller + wait-for-pods-with-label "app=nginx-statefulset" "" + + set +o nounset + set +o errexit +} + run_multi_resources_tests() { set -o nounset set -o errexit @@ -3639,7 +3700,7 @@ run_stateful_set_tests() { # Pre-condition: no statefulset exists kube::test::get_object_assert statefulset "{{range.items}}{{$id_field}}:{{end}}" '' # Command: create statefulset - kubectl create -f hack/testdata/nginx-statefulset.yaml "${kube_flags[@]}" + kubectl create -f hack/testdata/rollingupdate-statefulset.yaml "${kube_flags[@]}" ### Scale statefulset test with current-replicas and replicas # Pre-condition: 0 replicas @@ -3656,7 +3717,7 @@ run_stateful_set_tests() { wait-for-pods-with-label "app=nginx-statefulset" "nginx-0" ### Clean up - kubectl delete -f hack/testdata/nginx-statefulset.yaml "${kube_flags[@]}" + kubectl delete -f hack/testdata/rollingupdate-statefulset.yaml "${kube_flags[@]}" # Post-condition: no pods from statefulset controller wait-for-pods-with-label "app=nginx-statefulset" "" @@ -4241,15 +4302,13 @@ runTests() { deployment_replicas=".spec.replicas" secret_data=".data" secret_type=".type" - deployment_image_field="(index .spec.template.spec.containers 0).image" - deployment_second_image_field="(index .spec.template.spec.containers 1).image" change_cause_annotation='.*kubernetes.io/change-cause.*' pdb_min_available=".spec.minAvailable" pdb_max_unavailable=".spec.maxUnavailable" template_generation_field=".spec.templateGeneration" container_len="(len .spec.template.spec.containers)" - daemonset_image_field0="(index .spec.template.spec.containers 0).image" - daemonset_image_field1="(index .spec.template.spec.containers 1).image" + image_field0="(index .spec.template.spec.containers 0).image" + image_field1="(index .spec.template.spec.containers 1).image" # Make sure "default" namespace exists. if kube::test::if_supports_resource "${namespaces}" ; then @@ -4343,7 +4402,6 @@ runTests() { # run for federation apiserver as well. record_command run_kubectl_apply_tests record_command run_kubectl_run_tests - record_command run_kubectl_using_deprecated_commands_test record_command run_kubectl_create_filter_tests fi @@ -4456,7 +4514,6 @@ runTests() { record_command run_service_tests fi - ################## # DaemonSets # ################## @@ -4494,16 +4551,17 @@ runTests() { record_command run_rs_tests fi - ################# # Stateful Sets # ################# if kube::test::if_supports_resource "${statefulsets}" ; then record_command run_stateful_set_tests + if kube::test::if_supports_resource "${controllerrevisions}"; then + record_command run_statefulset_history_tests + fi fi - ###################### # Lists # ###################### diff --git a/hack/make-rules/test.sh b/hack/make-rules/test.sh index e243b22aba772..f7f0d812f76b1 100755 --- a/hack/make-rules/test.sh +++ b/hack/make-rules/test.sh @@ -274,11 +274,11 @@ runTests() { # separate files. # ignore paths: - # vendor/k8s.io/kube-gen/cmd/generator: is fragile when run under coverage, so ignore it for now. + # vendor/k8s.io/code-generator/cmd/generator: is fragile when run under coverage, so ignore it for now. # https://github.com/kubernetes/kubernetes/issues/24967 # vendor/k8s.io/client-go/1.4/rest: causes cover internal errors # https://github.com/golang/go/issues/16540 - cover_ignore_dirs="vendor/k8s.io/kube-gen/cmd/generator|vendor/k8s.io/client-go/1.4/rest" + cover_ignore_dirs="vendor/k8s.io/code-generator/cmd/generator|vendor/k8s.io/client-go/1.4/rest" for path in $(echo $cover_ignore_dirs | sed 's/|/ /g'); do echo -e "skipped\tk8s.io/kubernetes/$path" done diff --git a/hack/staging-import-restrictions.json b/hack/staging-import-restrictions.json index 3d421f2449581..51bd3e90100dd 100644 --- a/hack/staging-import-restrictions.json +++ b/hack/staging-import-restrictions.json @@ -14,23 +14,23 @@ ] }, { - "baseImportPath": "./vendor/k8s.io/kube-gen/", + "baseImportPath": "./vendor/k8s.io/code-generator/", "ignoredSubTrees": [ - "./vendor/k8s.io/kube-gen/test" + "./vendor/k8s.io/code-generator/test" ], "allowedImports": [ "k8s.io/gengo", - "k8s.io/kube-gen", + "k8s.io/code-generator", "k8s.io/kube-openapi" ] }, { - "baseImportPath": "./vendor/k8s.io/kube-gen/test/", + "baseImportPath": "./vendor/k8s.io/code-generator/test/", "allowedImports": [ "k8s.io/apimachinery", "k8s.io/client-go", "k8s.io/gengo", - "k8s.io/kube-gen/test", + "k8s.io/code-generator/test", "k8s.io/kube-openapi" ] }, diff --git a/hack/testdata/retainKeys/deployment/deployment-after.yaml b/hack/testdata/retainKeys/deployment/deployment-after.yaml new file mode 100644 index 0000000000000..15689a599590f --- /dev/null +++ b/hack/testdata/retainKeys/deployment/deployment-after.yaml @@ -0,0 +1,22 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: test-deployment-retainkeys +spec: + strategy: + type: Recreate + replicas: 1 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 + volumes: + - name: test-volume + hostPath: + path: /data diff --git a/hack/testdata/retainKeys/deployment/deployment-before.yaml b/hack/testdata/retainKeys/deployment/deployment-before.yaml new file mode 100644 index 0000000000000..e58a0c34a14e0 --- /dev/null +++ b/hack/testdata/retainKeys/deployment/deployment-before.yaml @@ -0,0 +1,18 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: test-deployment-retainkeys +spec: + replicas: 1 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 + volumes: + - name: test-volume diff --git a/hack/testdata/rollingupdate-statefulset-rv2.yaml b/hack/testdata/rollingupdate-statefulset-rv2.yaml new file mode 100644 index 0000000000000..fec5493ab6975 --- /dev/null +++ b/hack/testdata/rollingupdate-statefulset-rv2.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: nginx +spec: + selector: + matchLabels: + app: nginx-statefulset + updateStrategy: + type: RollingUpdate + serviceName: "nginx" + replicas: 0 + template: + metadata: + labels: + app: nginx-statefulset + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: nginx + image: gcr.io/google_containers/nginx-slim:0.8 + ports: + - containerPort: 80 + name: web + command: + - sh + - -c + - 'while true; do sleep 1; done' + - name: pause + image: gcr.io/google-containers/pause:2.0 + ports: + - containerPort: 81 + name: web-2 diff --git a/hack/testdata/nginx-statefulset.yaml b/hack/testdata/rollingupdate-statefulset.yaml similarity index 50% rename from hack/testdata/nginx-statefulset.yaml rename to hack/testdata/rollingupdate-statefulset.yaml index 299e407dfa3f3..2acbf0f322b9f 100644 --- a/hack/testdata/nginx-statefulset.yaml +++ b/hack/testdata/rollingupdate-statefulset.yaml @@ -1,26 +1,13 @@ -# A headless service to create DNS records -apiVersion: v1 -kind: Service -metadata: - annotations: - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" - name: nginx - labels: - app: nginx-statefulset -spec: - ports: - - port: 80 - name: web - # *.nginx.default.svc.cluster.local - clusterIP: None - selector: - app: nginx-statefulset ---- -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta2 kind: StatefulSet metadata: name: nginx spec: + selector: + matchLabels: + app: nginx-statefulset + updateStrategy: + type: RollingUpdate serviceName: "nginx" replicas: 0 template: @@ -28,7 +15,7 @@ spec: labels: app: nginx-statefulset spec: - terminationGracePeriodSeconds: 0 + terminationGracePeriodSeconds: 5 containers: - name: nginx image: gcr.io/google_containers/nginx-slim:0.7 diff --git a/hack/update-all.sh b/hack/update-all.sh index e999df36772d6..b966eaa3a8f9e 100755 --- a/hack/update-all.sh +++ b/hack/update-all.sh @@ -46,7 +46,7 @@ done trap 'exit 1' SIGINT if $SILENT ; then - echo "Running in the silent mode, run with -v if you want to see script logs." + echo "Running in silent mode, run with -v if you want to see script logs." fi if ! $ALL ; then @@ -77,7 +77,7 @@ BASH_TARGETS=" update-bazel" for t in $BASH_TARGETS; do - echo -e "${color_yellow}Updating $t${color_norm}" + echo -e "${color_yellow}Running $t${color_norm}" if $SILENT ; then if ! bash "$KUBE_ROOT/hack/$t.sh" 1> /dev/null; then echo -e "${color_red}Running $t FAILED${color_norm}" diff --git a/hack/update-bazel.sh b/hack/update-bazel.sh index b0f8d3e21d9e0..c3f9c13255528 100755 --- a/hack/update-bazel.sh +++ b/hack/update-bazel.sh @@ -25,8 +25,10 @@ source "${KUBE_ROOT}/hack/lib/init.sh" rm -f "${KUBE_ROOT}/pkg/generated/openapi/zz_generated.openapi.go" # The git commit sha1s here should match the values in $KUBE_ROOT/WORKSPACE. -kube::util::go_install_from_commit github.com/kubernetes/repo-infra/kazel 9ba3a24eeffafa3a794b9e7312103424e7da26e9 +kube::util::go_install_from_commit github.com/kubernetes/repo-infra/kazel 4eaf9e671bbb549fb4ec292cf251f921d7ef80ac kube::util::go_install_from_commit github.com/bazelbuild/rules_go/go/tools/gazelle/gazelle 82483596ec203eb9c1849937636f4cbed83733eb +touch "${KUBE_ROOT}/vendor/BUILD" + gazelle fix -build_file_name=BUILD,BUILD.bazel -external=vendored -mode=fix -repo_root="$(kube::realpath ${KUBE_ROOT})" kazel -root="$(kube::realpath ${KUBE_ROOT})" diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 3d35adfff7686..8fae4073c72d1 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -24,9 +24,9 @@ source "${KUBE_ROOT}/hack/lib/init.sh" kube::golang::setup_env BUILD_TARGETS=( - vendor/k8s.io/kube-gen/cmd/client-gen - vendor/k8s.io/kube-gen/cmd/lister-gen - vendor/k8s.io/kube-gen/cmd/informer-gen + vendor/k8s.io/code-generator/cmd/client-gen + vendor/k8s.io/code-generator/cmd/lister-gen + vendor/k8s.io/code-generator/cmd/informer-gen ) make -C "${KUBE_ROOT}" WHAT="${BUILD_TARGETS[*]}" @@ -122,8 +122,8 @@ ${informergen} \ # You may add additional calls of code generators like set-gen above. # call generation on sub-project for now -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/kube-gen/hack/update-codegen.sh -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/kube-aggregator/hack/update-codegen.sh -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/sample-apiserver/hack/update-codegen.sh -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/apiextensions-apiserver/hack/update-codegen.sh -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/metrics/hack/update-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/code-generator/hack/update-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/kube-aggregator/hack/update-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-apiserver/hack/update-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/apiextensions-apiserver/hack/update-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/metrics/hack/update-codegen.sh diff --git a/hack/update-generated-protobuf-dockerized.sh b/hack/update-generated-protobuf-dockerized.sh index 17716e936fd40..e81618fff6d34 100755 --- a/hack/update-generated-protobuf-dockerized.sh +++ b/hack/update-generated-protobuf-dockerized.sh @@ -24,8 +24,8 @@ source "${KUBE_ROOT}/hack/lib/init.sh" kube::golang::setup_env BINS=( - vendor/k8s.io/kube-gen/cmd/go-to-protobuf - vendor/k8s.io/kube-gen/cmd/go-to-protobuf/protoc-gen-gogo + vendor/k8s.io/code-generator/cmd/go-to-protobuf + vendor/k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo ) make -C "${KUBE_ROOT}" WHAT="${BINS[*]}" diff --git a/hack/update-generated-runtime-dockerized.sh b/hack/update-generated-runtime-dockerized.sh index febcc48dca1ee..37f1aa5b8b1d7 100755 --- a/hack/update-generated-runtime-dockerized.sh +++ b/hack/update-generated-runtime-dockerized.sh @@ -25,7 +25,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh" kube::golang::setup_env BINS=( - vendor/k8s.io/kube-gen/cmd/go-to-protobuf/protoc-gen-gogo + vendor/k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo ) make -C "${KUBE_ROOT}" WHAT="${BINS[*]}" diff --git a/hack/update-staging-godeps-dockerized.sh b/hack/update-staging-godeps-dockerized.sh index 458cbb211f247..bf4bbea8bba2f 100755 --- a/hack/update-staging-godeps-dockerized.sh +++ b/hack/update-staging-godeps-dockerized.sh @@ -50,11 +50,16 @@ while getopts ":df" opt; do esac done +# Confirm this is running inside a docker container, as this will modify the git tree (unsafe to run outside of container) kube::util::ensure_dockerized kube::golang::setup_env +# Ensure we have a simple gopath so that we can modify it, and that no staging repos have made their way in kube::util::ensure_single_dir_gopath kube::util::ensure_no_staging_repos_in_gopath +# Confirm we have the right godep version installed kube::util::ensure_godep_version v79 +# Create a fake git repo the root of the repo to prevent godeps from complaining +kube::util::create-fake-git-tree "${KUBE_ROOT}" kube::log::status "Checking whether godeps are restored" if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then @@ -87,27 +92,13 @@ function diffGodepManifest() { fi } -# Create a fake git repo for staging to prevent godeps from complaining -pushd "${KUBE_ROOT}" >/dev/null - git init >/dev/null - git config --local user.email "nobody@k8s.io" - git config --local user.name "$0" - git add . >/dev/null - git commit -q -m "Snapshot" >/dev/null -popd >/dev/null - # move into staging and save the dependencies for everything in order mkdir -p "${TMP_GOPATH}/src/k8s.io" for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/" - pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null - git init >/dev/null - git config --local user.email "nobody@k8s.io" - git config --local user.name "$0" - git add . >/dev/null - git commit -q -m "Snapshot" >/dev/null - popd >/dev/null + # Create a fake git tree for the staging repo to prevent godeps from complaining + kube::util::create-fake-git-tree "${TMP_GOPATH}/src/k8s.io/${repo}" updateGodepManifest diffGodepManifest diff --git a/hack/verify-api-groups.sh b/hack/verify-api-groups.sh index bbfcb0f02002b..88de8b7f6ab35 100755 --- a/hack/verify-api-groups.sh +++ b/hack/verify-api-groups.sh @@ -71,7 +71,7 @@ groups_without_codegen=( "imagepolicy" "admission" ) -client_gen_file="${KUBE_ROOT}/vendor/k8s.io/kube-gen/cmd/client-gen/main.go" +client_gen_file="${KUBE_ROOT}/vendor/k8s.io/code-generator/cmd/client-gen/main.go" for group_dirname in "${group_dirnames[@]}"; do if ! grep -q "${group_dirname}/" "${client_gen_file}" ; then diff --git a/hack/verify-bazel.sh b/hack/verify-bazel.sh index 87c3eacdb6d36..e51e8ccf689e1 100755 --- a/hack/verify-bazel.sh +++ b/hack/verify-bazel.sh @@ -20,12 +20,19 @@ set -o pipefail export KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" +if [[ ! -f "${KUBE_ROOT}/vendor/BUILD" ]]; then + echo "${KUBE_ROOT}/vendor/BUILD does not exist." + echo + echo "Run ./hack/update-bazel.sh" + exit 1 +fi + # Remove generated files prior to running kazel. # TODO(spxtr): Remove this line once Bazel is the only way to build. rm -f "${KUBE_ROOT}/pkg/generated/openapi/zz_generated.openapi.go" # The git commit sha1s here should match the values in $KUBE_ROOT/WORKSPACE. -kube::util::go_install_from_commit github.com/kubernetes/repo-infra/kazel 9ba3a24eeffafa3a794b9e7312103424e7da26e9 +kube::util::go_install_from_commit github.com/kubernetes/repo-infra/kazel 4eaf9e671bbb549fb4ec292cf251f921d7ef80ac kube::util::go_install_from_commit github.com/bazelbuild/rules_go/go/tools/gazelle/gazelle 82483596ec203eb9c1849937636f4cbed83733eb gazelle_diff=$(gazelle fix -build_file_name=BUILD,BUILD.bazel -external=vendored -mode=diff -repo_root="$(kube::realpath ${KUBE_ROOT})") diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh index 0e497fe4ffb09..af94a334ae035 100755 --- a/hack/verify-codegen.sh +++ b/hack/verify-codegen.sh @@ -27,10 +27,10 @@ kube::golang::setup_env # # Note: these must be before the main script call because the later calls the sub-project's # update-codegen.sh scripts. We wouldn't see any error on changes then. -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/kube-gen/hack/verify-codegen.sh -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/kube-aggregator/hack/verify-codegen.sh -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/sample-apiserver/hack/verify-codegen.sh -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/apiextensions-apiserver/hack/verify-codegen.sh -KUBEGEN_PKG=./vendor/k8s.io/kube-gen vendor/k8s.io/metrics/hack/verify-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/code-generator/hack/verify-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/kube-aggregator/hack/verify-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/sample-apiserver/hack/verify-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/apiextensions-apiserver/hack/verify-codegen.sh +CODEGEN_PKG=./vendor/k8s.io/code-generator vendor/k8s.io/metrics/hack/verify-codegen.sh "${KUBE_ROOT}/hack/update-codegen.sh" --verify-only diff --git a/hack/verify-import-boss.sh b/hack/verify-import-boss.sh index 67fdb86c806e3..7e112a4ac69d3 100755 --- a/hack/verify-import-boss.sh +++ b/hack/verify-import-boss.sh @@ -23,6 +23,6 @@ source "${KUBE_ROOT}/hack/lib/init.sh" kube::golang::setup_env -make -C "${KUBE_ROOT}" WHAT=vendor/k8s.io/kube-gen/cmd/import-boss +make -C "${KUBE_ROOT}" WHAT=vendor/k8s.io/code-generator/cmd/import-boss $(kube::util::find-binary "import-boss") --verify-only diff --git a/hack/verify-openapi-spec.sh b/hack/verify-openapi-spec.sh index ac56cea109507..d205347ccefdc 100755 --- a/hack/verify-openapi-spec.sh +++ b/hack/verify-openapi-spec.sh @@ -36,6 +36,7 @@ cp -a "${SPECROOT}" "${TMP_SPECROOT}" trap "cp -a ${TMP_SPECROOT} ${SPECROOT}/..; rm -rf ${_tmp}" EXIT SIGINT rm ${SPECROOT}/* cp ${TMP_SPECROOT}/BUILD ${SPECROOT}/BUILD +cp ${TMP_SPECROOT}/README.md ${SPECROOT}/README.md "${KUBE_ROOT}/hack/update-openapi-spec.sh" echo "diffing ${SPECROOT} against freshly generated openapi spec" diff --git a/pkg/api/OWNERS b/pkg/api/OWNERS index 7009d88bf109e..0605b27b2aa2c 100644 --- a/pkg/api/OWNERS +++ b/pkg/api/OWNERS @@ -3,6 +3,8 @@ approvers: - lavalamp - smarterclayton - thockin +- liggitt +# - bgrant0607 # manual escalations only reviewers: - thockin - lavalamp diff --git a/pkg/api/annotation_key_constants.go b/pkg/api/annotation_key_constants.go index 0d41438191fcc..e935424462b5a 100644 --- a/pkg/api/annotation_key_constants.go +++ b/pkg/api/annotation_key_constants.go @@ -47,6 +47,8 @@ const ( // CreatedByAnnotation represents the key used to store the spec(json) // used to create the resource. + // This field is deprecated in favor of ControllerRef (see #44407). + // TODO(#50720): Remove this field in v1.9. CreatedByAnnotation = "kubernetes.io/created-by" // PreferAvoidPodsAnnotationKey represents the key of preferAvoidPods data (json serialized) diff --git a/pkg/api/defaulting_test.go b/pkg/api/defaulting_test.go index f5ba8780e74fd..40c04f6f9b885 100644 --- a/pkg/api/defaulting_test.go +++ b/pkg/api/defaulting_test.go @@ -22,7 +22,7 @@ import ( "sort" "testing" - "github.com/google/gofuzz" + fuzz "github.com/google/gofuzz" apiv1 "k8s.io/api/core/v1" extensionsv1beta1 "k8s.io/api/extensions/v1beta1" @@ -123,8 +123,6 @@ func TestDefaulting(t *testing.T) { {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "RoleBindingList"}: {}, {Group: "settings.k8s.io", Version: "v1alpha1", Kind: "PodPreset"}: {}, {Group: "settings.k8s.io", Version: "v1alpha1", Kind: "PodPresetList"}: {}, - {Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "InitializerConfiguration"}: {}, - {Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "InitializerConfigurationList"}: {}, {Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "ExternalAdmissionHookConfiguration"}: {}, {Group: "admissionregistration.k8s.io", Version: "v1alpha1", Kind: "ExternalAdmissionHookConfigurationList"}: {}, {Group: "networking.k8s.io", Version: "v1", Kind: "NetworkPolicy"}: {}, diff --git a/pkg/api/fuzzer/fuzzer.go b/pkg/api/fuzzer/fuzzer.go index 9d0f94a267817..fb3712be64378 100644 --- a/pkg/api/fuzzer/fuzzer.go +++ b/pkg/api/fuzzer/fuzzer.go @@ -355,6 +355,15 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { r.Keyring = "/etc/ceph/keyring" } }, + func(obj *api.HostPathVolumeSource, c fuzz.Continue) { + c.FuzzNoCustom(obj) + types := []api.HostPathType{api.HostPathUnset, api.HostPathDirectoryOrCreate, api.HostPathDirectory, + api.HostPathFileOrCreate, api.HostPathFile, api.HostPathSocket, api.HostPathCharDev, api.HostPathBlockDev} + typeVol := types[c.Rand.Intn(len(types))] + if obj.Type == nil { + obj.Type = &typeVol + } + }, func(pv *api.PersistentVolume, c fuzz.Continue) { c.FuzzNoCustom(pv) // fuzz self without calling this function again types := []api.PersistentVolumePhase{api.VolumeAvailable, api.VolumePending, api.VolumeBound, api.VolumeReleased, api.VolumeFailed} diff --git a/pkg/api/helper/helpers.go b/pkg/api/helper/helpers.go index 93eb7a130dbc0..e181d4a249706 100644 --- a/pkg/api/helper/helpers.go +++ b/pkg/api/helper/helpers.go @@ -107,6 +107,7 @@ func IsResourceQuotaScopeValidForResource(scope api.ResourceQuotaScope, resource var standardContainerResources = sets.NewString( string(api.ResourceCPU), string(api.ResourceMemory), + string(api.ResourceEphemeralStorage), ) // IsStandardContainerResourceName returns true if the container can make a resource request @@ -169,11 +170,14 @@ func IsStandardLimitRangeType(str string) bool { var standardQuotaResources = sets.NewString( string(api.ResourceCPU), string(api.ResourceMemory), + string(api.ResourceEphemeralStorage), string(api.ResourceRequestsCPU), string(api.ResourceRequestsMemory), string(api.ResourceRequestsStorage), + string(api.ResourceRequestsEphemeralStorage), string(api.ResourceLimitsCPU), string(api.ResourceLimitsMemory), + string(api.ResourceLimitsEphemeralStorage), string(api.ResourcePods), string(api.ResourceQuotas), string(api.ResourceServices), @@ -194,10 +198,13 @@ func IsStandardQuotaResourceName(str string) bool { var standardResources = sets.NewString( string(api.ResourceCPU), string(api.ResourceMemory), + string(api.ResourceEphemeralStorage), string(api.ResourceRequestsCPU), string(api.ResourceRequestsMemory), + string(api.ResourceRequestsEphemeralStorage), string(api.ResourceLimitsCPU), string(api.ResourceLimitsMemory), + string(api.ResourceLimitsEphemeralStorage), string(api.ResourcePods), string(api.ResourceQuotas), string(api.ResourceServices), @@ -207,6 +214,8 @@ var standardResources = sets.NewString( string(api.ResourcePersistentVolumeClaims), string(api.ResourceStorage), string(api.ResourceRequestsStorage), + string(api.ResourceServicesNodePorts), + string(api.ResourceServicesLoadBalancers), ) // IsStandardResourceName returns true if the resource is known to the system diff --git a/pkg/api/persistentvolume/util.go b/pkg/api/persistentvolume/util.go index d4edd807a79f7..0a8de7c5e68f6 100644 --- a/pkg/api/persistentvolume/util.go +++ b/pkg/api/persistentvolume/util.go @@ -37,13 +37,27 @@ func VisitPVSecretNames(pv *api.PersistentVolume, visitor Visitor) bool { source := &pv.Spec.PersistentVolumeSource switch { case source.AzureFile != nil: - if len(source.AzureFile.SecretName) > 0 && !visitor(getClaimRefNamespace(pv), source.AzureFile.SecretName) { - return false + if source.AzureFile.SecretNamespace != nil && len(*source.AzureFile.SecretNamespace) > 0 { + if len(source.AzureFile.SecretName) > 0 && !visitor(*source.AzureFile.SecretNamespace, source.AzureFile.SecretName) { + return false + } + } else { + if len(source.AzureFile.SecretName) > 0 && !visitor(getClaimRefNamespace(pv), source.AzureFile.SecretName) { + return false + } } return true case source.CephFS != nil: - if source.CephFS.SecretRef != nil && !visitor(getClaimRefNamespace(pv), source.CephFS.SecretRef.Name) { - return false + if source.CephFS.SecretRef != nil { + // previously persisted PV objects use claimRef namespace + ns := getClaimRefNamespace(pv) + if len(source.CephFS.SecretRef.Namespace) > 0 { + // use the secret namespace if namespace is set + ns = source.CephFS.SecretRef.Namespace + } + if !visitor(ns, source.CephFS.SecretRef.Name) { + return false + } } case source.FlexVolume != nil: if source.FlexVolume.SecretRef != nil && !visitor(getClaimRefNamespace(pv), source.FlexVolume.SecretRef.Name) { diff --git a/pkg/api/persistentvolume/util_test.go b/pkg/api/persistentvolume/util_test.go index 87564d337233e..8e83f0644c6d3 100644 --- a/pkg/api/persistentvolume/util_test.go +++ b/pkg/api/persistentvolume/util_test.go @@ -30,17 +30,31 @@ import ( func TestPVSecrets(t *testing.T) { // Stub containing all possible secret references in a PV. // The names of the referenced secrets match struct paths detected by reflection. + secretNamespace := "Spec.PersistentVolumeSource.AzureFile.SecretNamespace" pvs := []*api.PersistentVolume{ {Spec: api.PersistentVolumeSpec{ ClaimRef: &api.ObjectReference{Namespace: "claimrefns", Name: "claimrefname"}, PersistentVolumeSource: api.PersistentVolumeSource{ - AzureFile: &api.AzureFileVolumeSource{ + AzureFile: &api.AzureFilePersistentVolumeSource{ SecretName: "Spec.PersistentVolumeSource.AzureFile.SecretName"}}}}, {Spec: api.PersistentVolumeSpec{ ClaimRef: &api.ObjectReference{Namespace: "claimrefns", Name: "claimrefname"}, PersistentVolumeSource: api.PersistentVolumeSource{ - CephFS: &api.CephFSVolumeSource{ - SecretRef: &api.LocalObjectReference{ + AzureFile: &api.AzureFilePersistentVolumeSource{ + SecretName: "Spec.PersistentVolumeSource.AzureFile.SecretName", + SecretNamespace: &secretNamespace}}}}, + {Spec: api.PersistentVolumeSpec{ + ClaimRef: &api.ObjectReference{Namespace: "claimrefns", Name: "claimrefname"}, + PersistentVolumeSource: api.PersistentVolumeSource{ + CephFS: &api.CephFSPersistentVolumeSource{ + SecretRef: &api.SecretReference{ + Name: "Spec.PersistentVolumeSource.CephFS.SecretRef", + Namespace: "cephfs"}}}}}, + {Spec: api.PersistentVolumeSpec{ + ClaimRef: &api.ObjectReference{Namespace: "claimrefns", Name: "claimrefname"}, + PersistentVolumeSource: api.PersistentVolumeSource{ + CephFS: &api.CephFSPersistentVolumeSource{ + SecretRef: &api.SecretReference{ Name: "Spec.PersistentVolumeSource.CephFS.SecretRef"}}}}}, {Spec: api.PersistentVolumeSpec{ ClaimRef: &api.ObjectReference{Namespace: "claimrefns", Name: "claimrefname"}, @@ -74,7 +88,6 @@ func TestPVSecrets(t *testing.T) { Name: "Spec.PersistentVolumeSource.StorageOS.SecretRef", Namespace: "storageosns"}}}}}, } - extractedNames := sets.NewString() extractedNamesWithNamespace := sets.NewString() for _, pv := range pvs { @@ -88,6 +101,7 @@ func TestPVSecrets(t *testing.T) { // excludedSecretPaths holds struct paths to fields with "secret" in the name that are not actually references to secret API objects excludedSecretPaths := sets.NewString( "Spec.PersistentVolumeSource.CephFS.SecretFile", + "Spec.PersistentVolumeSource.AzureFile.SecretNamespace", ) // expectedSecretPaths holds struct paths to fields with "secret" in the name that are references to secret API objects. // every path here should be represented as an example in the PV stub above, with the secret name set to the path. @@ -122,7 +136,9 @@ func TestPVSecrets(t *testing.T) { expectedNamespacedNames := sets.NewString( "claimrefns/Spec.PersistentVolumeSource.AzureFile.SecretName", + "Spec.PersistentVolumeSource.AzureFile.SecretNamespace/Spec.PersistentVolumeSource.AzureFile.SecretName", "claimrefns/Spec.PersistentVolumeSource.CephFS.SecretRef", + "cephfs/Spec.PersistentVolumeSource.CephFS.SecretRef", "claimrefns/Spec.PersistentVolumeSource.FlexVolume.SecretRef", "claimrefns/Spec.PersistentVolumeSource.RBD.SecretRef", "claimrefns/Spec.PersistentVolumeSource.ScaleIO.SecretRef", diff --git a/pkg/api/pod/util.go b/pkg/api/pod/util.go index 5489171586ac0..9dc4fa68ee9cb 100644 --- a/pkg/api/pod/util.go +++ b/pkg/api/pod/util.go @@ -218,8 +218,8 @@ func UpdatePodCondition(status *api.PodStatus, condition *api.PodCondition) bool isEqual := condition.Status == oldCondition.Status && condition.Reason == oldCondition.Reason && condition.Message == oldCondition.Message && - condition.LastProbeTime.Equal(oldCondition.LastProbeTime) && - condition.LastTransitionTime.Equal(oldCondition.LastTransitionTime) + condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) && + condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime) status.Conditions[conditionIndex] = *condition // Return true if one of the fields have changed. diff --git a/pkg/api/resource.go b/pkg/api/resource.go index 27b7fd6650069..ce1747d8e6be9 100644 --- a/pkg/api/resource.go +++ b/pkg/api/resource.go @@ -54,8 +54,8 @@ func (self *ResourceList) NvidiaGPU() *resource.Quantity { return &resource.Quantity{} } -func (self *ResourceList) StorageOverlay() *resource.Quantity { - if val, ok := (*self)[ResourceStorageOverlay]; ok { +func (self *ResourceList) StorageEphemeral() *resource.Quantity { + if val, ok := (*self)[ResourceEphemeralStorage]; ok { return &val } return &resource.Quantity{} diff --git a/pkg/api/resource/helpers.go b/pkg/api/resource/helpers.go index debdbe52818fb..eecc26ed222a9 100644 --- a/pkg/api/resource/helpers.go +++ b/pkg/api/resource/helpers.go @@ -88,10 +88,14 @@ func ExtractContainerResourceValue(fs *api.ResourceFieldSelector, container *api return convertResourceCPUToString(container.Resources.Limits.Cpu(), divisor) case "limits.memory": return convertResourceMemoryToString(container.Resources.Limits.Memory(), divisor) + case "limits.ephemeral-storage": + return convertResourceEphemeralStorageToString(container.Resources.Limits.StorageEphemeral(), divisor) case "requests.cpu": return convertResourceCPUToString(container.Resources.Requests.Cpu(), divisor) case "requests.memory": return convertResourceMemoryToString(container.Resources.Requests.Memory(), divisor) + case "requests.ephemeral-storage": + return convertResourceEphemeralStorageToString(container.Resources.Requests.StorageEphemeral(), divisor) } return "", fmt.Errorf("unsupported container resource : %v", fs.Resource) @@ -110,3 +114,10 @@ func convertResourceMemoryToString(memory *resource.Quantity, divisor resource.Q m := int64(math.Ceil(float64(memory.Value()) / float64(divisor.Value()))) return strconv.FormatInt(m, 10), nil } + +// convertResourceEphemeralStorageToString converts ephemeral storage value to the format of divisor and returns +// ceiling of the value. +func convertResourceEphemeralStorageToString(ephemeralStorage *resource.Quantity, divisor resource.Quantity) (string, error) { + m := int64(math.Ceil(float64(ephemeralStorage.Value()) / float64(divisor.Value()))) + return strconv.FormatInt(m, 10), nil +} diff --git a/pkg/api/resource/helpers_test.go b/pkg/api/resource/helpers_test.go index fbcc3688b60c8..cab00528709f7 100644 --- a/pkg/api/resource/helpers_test.go +++ b/pkg/api/resource/helpers_test.go @@ -30,9 +30,8 @@ func TestResourceHelpers(t *testing.T) { memoryLimit := resource.MustParse("10G") resourceSpec := api.ResourceRequirements{ Limits: api.ResourceList{ - "cpu": cpuLimit, - "memory": memoryLimit, - "kube.io/storage": memoryLimit, + api.ResourceCPU: cpuLimit, + api.ResourceMemory: memoryLimit, }, } if res := resourceSpec.Limits.Cpu(); res.Cmp(cpuLimit) != 0 { @@ -43,8 +42,7 @@ func TestResourceHelpers(t *testing.T) { } resourceSpec = api.ResourceRequirements{ Limits: api.ResourceList{ - "memory": memoryLimit, - "kube.io/storage": memoryLimit, + api.ResourceMemory: memoryLimit, }, } if res := resourceSpec.Limits.Cpu(); res.Value() != 0 { diff --git a/pkg/api/serialization_test.go b/pkg/api/serialization_test.go index ff7ca6ee3d7b2..f6e2daa85ea00 100644 --- a/pkg/api/serialization_test.go +++ b/pkg/api/serialization_test.go @@ -208,7 +208,6 @@ func TestRoundTripTypes(t *testing.T) { fuzzer := fuzzer.FuzzerFor(kapitesting.FuzzerFuncs, rand.NewSource(seed), api.Codecs) nonRoundTrippableTypes := map[schema.GroupVersionKind]bool{ - {Group: "componentconfig", Version: runtime.APIVersionInternal, Kind: "KubeletConfiguration"}: true, {Group: "componentconfig", Version: runtime.APIVersionInternal, Kind: "KubeProxyConfiguration"}: true, {Group: "componentconfig", Version: runtime.APIVersionInternal, Kind: "KubeSchedulerConfiguration"}: true, } diff --git a/pkg/api/testing/BUILD b/pkg/api/testing/BUILD index c914bec2270d7..659e97210cd47 100644 --- a/pkg/api/testing/BUILD +++ b/pkg/api/testing/BUILD @@ -24,6 +24,7 @@ go_library( "//pkg/apis/extensions:go_default_library", "//pkg/apis/extensions/fuzzer:go_default_library", "//pkg/apis/extensions/v1beta1:go_default_library", + "//pkg/apis/networking/fuzzer:go_default_library", "//pkg/apis/policy/fuzzer:go_default_library", "//pkg/apis/rbac/fuzzer:go_default_library", "//pkg/apis/storage/fuzzer:go_default_library", diff --git a/pkg/api/testing/fuzzer.go b/pkg/api/testing/fuzzer.go index 844b9f79d2b86..8fc37898d491d 100644 --- a/pkg/api/testing/fuzzer.go +++ b/pkg/api/testing/fuzzer.go @@ -19,7 +19,7 @@ package testing import ( "fmt" - "github.com/google/gofuzz" + fuzz "github.com/google/gofuzz" "k8s.io/api/core/v1" apitesting "k8s.io/apimachinery/pkg/api/testing" @@ -38,6 +38,7 @@ import ( "k8s.io/kubernetes/pkg/apis/extensions" extensionsfuzzer "k8s.io/kubernetes/pkg/apis/extensions/fuzzer" extensionsv1beta1 "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" + networkingfuzzer "k8s.io/kubernetes/pkg/apis/networking/fuzzer" policyfuzzer "k8s.io/kubernetes/pkg/apis/policy/fuzzer" rbacfuzzer "k8s.io/kubernetes/pkg/apis/rbac/fuzzer" storagefuzzer "k8s.io/kubernetes/pkg/apis/storage/fuzzer" @@ -102,4 +103,5 @@ var FuzzerFuncs = fuzzer.MergeFuzzerFuncs( certificatesfuzzer.Funcs, admissionregistrationfuzzer.Funcs, storagefuzzer.Funcs, + networkingfuzzer.Funcs, ) diff --git a/pkg/api/types.go b/pkg/api/types.go index 4d626b5d09593..8b8554396e66c 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -360,7 +360,7 @@ type PersistentVolumeSource struct { Cinder *CinderVolumeSource // CephFS represents a Ceph FS mount on the host that shares a pod's lifetime // +optional - CephFS *CephFSVolumeSource + CephFS *CephFSPersistentVolumeSource // FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. // +optional FC *FCVolumeSource @@ -369,7 +369,7 @@ type PersistentVolumeSource struct { Flocker *FlockerVolumeSource // AzureFile represents an Azure File Service mount on the host and bind mount to the pod. // +optional - AzureFile *AzureFileVolumeSource + AzureFile *AzureFilePersistentVolumeSource // VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine // +optional VsphereVolume *VsphereVirtualDiskVolumeSource @@ -455,6 +455,10 @@ type PersistentVolumeSpec struct { // means that this volume does not belong to any StorageClass. // +optional StorageClassName string + // A list of mount options, e.g. ["ro", "soft"]. Not validated - mount will + // simply fail if one is invalid. + // +optional + MountOptions []string } // PersistentVolumeReclaimPolicy describes a policy for end-of-life maintenance of persistent volumes @@ -597,10 +601,36 @@ const ( ClaimLost PersistentVolumeClaimPhase = "Lost" ) +type HostPathType string + +const ( + // For backwards compatible, leave it empty if unset + HostPathUnset HostPathType = "" + // If nothing exists at the given path, an empty directory will be created there + // as needed with file mode 0755, having the same group and ownership with Kubelet. + HostPathDirectoryOrCreate HostPathType = "DirectoryOrCreate" + // A directory must exist at the given path + HostPathDirectory HostPathType = "Directory" + // If nothing exists at the given path, an empty file will be created there + // as needed with file mode 0644, having the same group and ownership with Kubelet. + HostPathFileOrCreate HostPathType = "FileOrCreate" + // A file must exist at the given path + HostPathFile HostPathType = "File" + // A UNIX socket must exist at the given path + HostPathSocket HostPathType = "Socket" + // A character device must exist at the given path + HostPathCharDev HostPathType = "CharDevice" + // A block device must exist at the given path + HostPathBlockDev HostPathType = "BlockDevice" +) + // Represents a host path mapped into a pod. // Host path volumes do not support ownership management or SELinux relabeling. type HostPathVolumeSource struct { + // If the path is a symlink, it will follow the link to the real path. Path string + // Defaults to "" + Type *HostPathType } // Represents an empty directory for a pod. @@ -709,6 +739,11 @@ type ISCSIVolumeSource struct { // The secret is used if either DiscoveryCHAPAuth or SessionCHAPAuth is true // +optional SecretRef *LocalObjectReference + // Optional: Custom initiator name per volume. + // If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + // : will be created for the connection. + // +optional + InitiatorName *string } // Represents a Fibre Channel volume. @@ -988,6 +1023,40 @@ type CephFSVolumeSource struct { ReadOnly bool } +// SecretReference represents a Secret Reference. It has enough information to retrieve secret +// in any namespace +type SecretReference struct { + // Name is unique within a namespace to reference a secret resource. + // +optional + Name string + // Namespace defines the space within which the secret name must be unique. + // +optional + Namespace string +} + +// Represents a Ceph Filesystem mount that lasts the lifetime of a pod +// Cephfs volumes do not support ownership management or SELinux relabeling. +type CephFSPersistentVolumeSource struct { + // Required: Monitors is a collection of Ceph monitors + Monitors []string + // Optional: Used as the mounted root, rather than the full Ceph tree, default is / + // +optional + Path string + // Optional: User is the rados user name, default is admin + // +optional + User string + // Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + // +optional + SecretFile string + // Optional: SecretRef is reference to the authentication secret for User, default is empty. + // +optional + SecretRef *SecretReference + // Optional: Defaults to false (read/write). ReadOnly here will force + // the ReadOnly setting in VolumeMounts. + // +optional + ReadOnly bool +} + // Represents a Flocker volume mounted by the Flocker agent. // One and only one of datasetName and datasetUUID should be set. // Flocker volumes do not support ownership management or SELinux relabeling. @@ -1056,6 +1125,22 @@ type AzureFileVolumeSource struct { ReadOnly bool } +// AzureFile represents an Azure File Service mount on the host and bind mount to the pod. +type AzureFilePersistentVolumeSource struct { + // the name of secret that contains Azure Storage Account Name and Key + SecretName string + // Share Name + ShareName string + // Defaults to false (read/write). ReadOnly here will force + // the ReadOnly setting in VolumeMounts. + // +optional + ReadOnly bool + // the namespace of the secret that contains Azure Storage Account Name and Key + // default is the same as the Pod + // +optional + SecretNamespace *string +} + // Represents a vSphere volume resource. type VsphereVirtualDiskVolumeSource struct { // Path that identifies vSphere volume vmdk @@ -1389,7 +1474,7 @@ type EnvVarSource struct { // +optional FieldRef *ObjectFieldSelector // Selects a resource of the container: only resources limits and requests - // (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + // (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. // +optional ResourceFieldRef *ResourceFieldSelector // Selects a key of a ConfigMap. @@ -2034,9 +2119,7 @@ type PodAffinityTerm struct { // the labelSelector in the specified namespaces, where co-located is defined as running on a node // whose value of the label with key topologyKey matches that of any node on which any of the // selected pods is running. - // For PreferredDuringScheduling pod anti-affinity, empty topologyKey is interpreted as "all topologies" - // ("all topologies" here means all the topologyKeys indicated by scheduler command-line argument --failure-domains); - // for affinity and for RequiredDuringScheduling pod anti-affinity, empty topologyKey is not allowed. + // Empty topologyKey is not allowed. // +optional TopologyKey string } @@ -2581,6 +2664,31 @@ const ( ServiceAffinityNone ServiceAffinity = "None" ) +const ( + // DefaultClientIPServiceAffinitySeconds is the default timeout seconds + // of Client IP based session affinity - 3 hours. + DefaultClientIPServiceAffinitySeconds int32 = 10800 + // MaxClientIPServiceAffinitySeconds is the max timeout seconds + // of Client IP based session affinity - 1 day. + MaxClientIPServiceAffinitySeconds int32 = 86400 +) + +// SessionAffinityConfig represents the configurations of session affinity. +type SessionAffinityConfig struct { + // clientIP contains the configurations of Client IP based session affinity. + // +optional + ClientIP *ClientIPConfig +} + +// ClientIPConfig represents the configurations of Client IP based session affinity. +type ClientIPConfig struct { + // timeoutSeconds specifies the seconds of ClientIP type session sticky time. + // The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + // Default value is 10800(for 3 hours). + // +optional + TimeoutSeconds *int32 +} + // Service Type string describes ingress methods for a service type ServiceType string @@ -2708,6 +2816,10 @@ type ServiceSpec struct { // +optional SessionAffinity ServiceAffinity + // sessionAffinityConfig contains the configurations of session affinity. + // +optional + SessionAffinityConfig *SessionAffinityConfig + // Optional: If specified and supported by the platform, this will restrict traffic through the cloud-provider // load-balancer will be restricted to the specified client IPs. This field will be ignored if the // cloud-provider does not support the feature." @@ -3164,15 +3276,11 @@ const ( ResourceMemory ResourceName = "memory" // Volume size, in bytes (e,g. 5Gi = 5GiB = 5 * 1024 * 1024 * 1024) ResourceStorage ResourceName = "storage" - // Local Storage for overlay filesystem, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) - // The resource name for ResourceStorageOverlay is alpha and it can change across releases. - ResourceStorageOverlay ResourceName = "storage.kubernetes.io/overlay" - // Local Storage for scratch space, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) - // The resource name for ResourceStorageScratch is alpha and it can change across releases. - ResourceStorageScratch ResourceName = "storage.kubernetes.io/scratch" + // Local ephemeral storage, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) + // The resource name for ResourceEphemeralStorage is alpha and it can change across releases. + ResourceEphemeralStorage ResourceName = "ephemeral-storage" // NVIDIA GPU, in devices. Alpha, might change: although fractional and allowing values >1, only one whole device per node is assigned. ResourceNvidiaGPU ResourceName = "alpha.kubernetes.io/nvidia-gpu" - // Number of Pods that may be running on this Node: see ResourcePods ) const ( @@ -3722,10 +3830,14 @@ const ( ResourceRequestsMemory ResourceName = "requests.memory" // Storage request, in bytes ResourceRequestsStorage ResourceName = "requests.storage" + // Local ephemeral storage request, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) + ResourceRequestsEphemeralStorage ResourceName = "requests.ephemeral-storage" // CPU limit, in cores. (500m = .5 cores) ResourceLimitsCPU ResourceName = "limits.cpu" // Memory limit, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) ResourceLimitsMemory ResourceName = "limits.memory" + // Local ephemeral storage limit, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) + ResourceLimitsEphemeralStorage ResourceName = "limits.ephemeral-storage" ) // A ResourceQuotaScope defines a filter that must match each object tracked by a quota diff --git a/pkg/api/v1/conversion.go b/pkg/api/v1/conversion.go index 72b07cffd58e7..004b016d83ac6 100644 --- a/pkg/api/v1/conversion.go +++ b/pkg/api/v1/conversion.go @@ -154,32 +154,6 @@ func addConversionFuncs(scheme *runtime.Scheme) error { return err } - // Add field label conversions for kinds having selectable nothing but v1.ObjectMeta fields. - for _, k := range []string{ - "Endpoints", - "ResourceQuota", - "PersistentVolumeClaim", - "Service", - "ServiceAccount", - "ConfigMap", - } { - kind := k // don't close over range variables - err = scheme.AddFieldLabelConversionFunc("v1", kind, - func(label, value string) (string, string, error) { - switch label { - case "metadata.namespace", - "metadata.name": - return label, value, nil - default: - return "", "", fmt.Errorf("field label %q not supported for %q", label, kind) - } - }, - ) - if err != nil { - return err - } - } - // Add field conversion funcs. err = scheme.AddFieldLabelConversionFunc("v1", "Pod", func(label, value string) (string, string, error) { @@ -192,6 +166,7 @@ func addConversionFuncs(scheme *runtime.Scheme) error { "spec.nodeName", "spec.restartPolicy", "spec.serviceAccountName", + "spec.schedulerName", "status.phase", "status.hostIP", "status.podIP": @@ -236,19 +211,6 @@ func addConversionFuncs(scheme *runtime.Scheme) error { if err != nil { return err } - err = scheme.AddFieldLabelConversionFunc("v1", "PersistentVolume", - func(label, value string) (string, string, error) { - switch label { - case "metadata.name": - return label, value, nil - default: - return "", "", fmt.Errorf("field label not supported: %s", label) - } - }, - ) - if err != nil { - return err - } if err := AddFieldLabelConversionsForEvent(scheme); err != nil { return err } diff --git a/pkg/api/v1/defaults.go b/pkg/api/v1/defaults.go index 70c1d709efb71..8b062695f931d 100644 --- a/pkg/api/v1/defaults.go +++ b/pkg/api/v1/defaults.go @@ -369,3 +369,10 @@ func SetDefaults_ScaleIOVolumeSource(obj *v1.ScaleIOVolumeSource) { obj.FSType = "xfs" } } + +func SetDefaults_HostPathVolumeSource(obj *v1.HostPathVolumeSource) { + typeVol := v1.HostPathUnset + if obj.Type == nil { + obj.Type = &typeVol + } +} diff --git a/pkg/api/v1/defaults_test.go b/pkg/api/v1/defaults_test.go index 0e39bd55d9fe1..6b7027d1ae571 100644 --- a/pkg/api/v1/defaults_test.go +++ b/pkg/api/v1/defaults_test.go @@ -1277,3 +1277,25 @@ func TestSetDefaultSchedulerName(t *testing.T) { t.Errorf("Expected scheduler name: %+v\ngot: %+v\n", v1.DefaultSchedulerName, output.Spec.SchedulerName) } } + +func TestSetDefaultHostPathVolumeSource(t *testing.T) { + s := v1.PodSpec{} + s.Volumes = []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "foo"}, + }, + }, + } + pod := &v1.Pod{ + Spec: s, + } + output := roundTrip(t, runtime.Object(pod)) + pod2 := output.(*v1.Pod) + defaultType := pod2.Spec.Volumes[0].VolumeSource.HostPath.Type + expectedType := v1.HostPathUnset + + if defaultType == nil || *defaultType != expectedType { + t.Errorf("Expected v1.HostPathVolumeSource default type %v, got %v", expectedType, defaultType) + } +} diff --git a/pkg/api/v1/pod/util.go b/pkg/api/v1/pod/util.go index b5ddb1ca2d96b..9e869c573d840 100644 --- a/pkg/api/v1/pod/util.go +++ b/pkg/api/v1/pod/util.go @@ -348,8 +348,8 @@ func UpdatePodCondition(status *v1.PodStatus, condition *v1.PodCondition) bool { isEqual := condition.Status == oldCondition.Status && condition.Reason == oldCondition.Reason && condition.Message == oldCondition.Message && - condition.LastProbeTime.Equal(oldCondition.LastProbeTime) && - condition.LastTransitionTime.Equal(oldCondition.LastTransitionTime) + condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) && + condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime) status.Conditions[conditionIndex] = *condition // Return true if one of the fields have changed. diff --git a/pkg/api/v1/resource/helpers.go b/pkg/api/v1/resource/helpers.go index e97879d283047..e0610daae3cfe 100644 --- a/pkg/api/v1/resource/helpers.go +++ b/pkg/api/v1/resource/helpers.go @@ -153,10 +153,14 @@ func ExtractContainerResourceValue(fs *v1.ResourceFieldSelector, container *v1.C return convertResourceCPUToString(container.Resources.Limits.Cpu(), divisor) case "limits.memory": return convertResourceMemoryToString(container.Resources.Limits.Memory(), divisor) + case "limits.ephemeral-storage": + return convertResourceEphemeralStorageToString(container.Resources.Limits.StorageEphemeral(), divisor) case "requests.cpu": return convertResourceCPUToString(container.Resources.Requests.Cpu(), divisor) case "requests.memory": return convertResourceMemoryToString(container.Resources.Requests.Memory(), divisor) + case "requests.ephemeral-storage": + return convertResourceEphemeralStorageToString(container.Resources.Requests.StorageEphemeral(), divisor) } return "", fmt.Errorf("Unsupported container resource : %v", fs.Resource) @@ -176,6 +180,13 @@ func convertResourceMemoryToString(memory *resource.Quantity, divisor resource.Q return strconv.FormatInt(m, 10), nil } +// convertResourceEphemeralStorageToString converts ephemeral storage value to the format of divisor and returns +// ceiling of the value. +func convertResourceEphemeralStorageToString(ephemeralStorage *resource.Quantity, divisor resource.Quantity) (string, error) { + m := int64(math.Ceil(float64(ephemeralStorage.Value()) / float64(divisor.Value()))) + return strconv.FormatInt(m, 10), nil +} + // findContainerInPod finds a container by its name in the provided pod func findContainerInPod(pod *v1.Pod, containerName string) (*v1.Container, error) { for _, container := range pod.Spec.Containers { diff --git a/pkg/api/v1/resource/helpers_test.go b/pkg/api/v1/resource/helpers_test.go index 0cf00dd1f7b02..a0f41253f79c6 100644 --- a/pkg/api/v1/resource/helpers_test.go +++ b/pkg/api/v1/resource/helpers_test.go @@ -30,9 +30,8 @@ func TestResourceHelpers(t *testing.T) { memoryLimit := resource.MustParse("10G") resourceSpec := v1.ResourceRequirements{ Limits: v1.ResourceList{ - "cpu": cpuLimit, - "memory": memoryLimit, - "kube.io/storage": memoryLimit, + v1.ResourceCPU: cpuLimit, + v1.ResourceMemory: memoryLimit, }, } if res := resourceSpec.Limits.Cpu(); res.Cmp(cpuLimit) != 0 { @@ -43,8 +42,7 @@ func TestResourceHelpers(t *testing.T) { } resourceSpec = v1.ResourceRequirements{ Limits: v1.ResourceList{ - "memory": memoryLimit, - "kube.io/storage": memoryLimit, + v1.ResourceMemory: memoryLimit, }, } if res := resourceSpec.Limits.Cpu(); res.Value() != 0 { diff --git a/pkg/api/v1/validation/BUILD b/pkg/api/v1/validation/BUILD index b1bb82a924cc6..14612044e10c1 100644 --- a/pkg/api/v1/validation/BUILD +++ b/pkg/api/v1/validation/BUILD @@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( @@ -32,3 +33,15 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["validation_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + ], +) diff --git a/pkg/api/v1/validation/validation.go b/pkg/api/v1/validation/validation.go index c5ef5a431710d..109054f804ea8 100644 --- a/pkg/api/v1/validation/validation.go +++ b/pkg/api/v1/validation/validation.go @@ -46,16 +46,6 @@ func ValidateResourceRequirements(requirements *v1.ResourceRequirements, fldPath // Validate resource quantity. allErrs = append(allErrs, ValidateResourceQuantityValue(string(resourceName), quantity, fldPath)...) - // Check that request <= limit. - requestQuantity, exists := requirements.Requests[resourceName] - if exists { - // Ensure overcommit is allowed for the resource if request != limit - if quantity.Cmp(requestQuantity) != 0 && !v1helper.IsOvercommitAllowed(resourceName) { - allErrs = append(allErrs, field.Invalid(reqPath, requestQuantity.String(), fmt.Sprintf("must be equal to %s limit", resourceName))) - } else if quantity.Cmp(requestQuantity) < 0 { - allErrs = append(allErrs, field.Invalid(limPath, quantity.String(), fmt.Sprintf("must be greater than or equal to %s request", resourceName))) - } - } } for resourceName, quantity := range requirements.Requests { fldPath := reqPath.Key(string(resourceName)) @@ -63,6 +53,19 @@ func ValidateResourceRequirements(requirements *v1.ResourceRequirements, fldPath allErrs = append(allErrs, validateContainerResourceName(string(resourceName), fldPath)...) // Validate resource quantity. allErrs = append(allErrs, ValidateResourceQuantityValue(string(resourceName), quantity, fldPath)...) + + // Check that request <= limit. + limitQuantity, exists := requirements.Limits[resourceName] + if exists { + // For GPUs, not only requests can't exceed limits, they also can't be lower, i.e. must be equal. + if quantity.Cmp(limitQuantity) != 0 && !v1helper.IsOvercommitAllowed(resourceName) { + allErrs = append(allErrs, field.Invalid(reqPath, quantity.String(), fmt.Sprintf("must be equal to %s limit", resourceName))) + } else if quantity.Cmp(limitQuantity) > 0 { + allErrs = append(allErrs, field.Invalid(reqPath, quantity.String(), fmt.Sprintf("must be less than or equal to %s limit", resourceName))) + } + } else if resourceName == v1.ResourceNvidiaGPU { + allErrs = append(allErrs, field.Invalid(reqPath, quantity.String(), fmt.Sprintf("must be equal to %s request", v1.ResourceNvidiaGPU))) + } } return allErrs @@ -75,7 +78,7 @@ func validateContainerResourceName(value string, fldPath *field.Path) field.Erro return append(allErrs, field.Invalid(fldPath, value, "must be a standard resource for containers")) } } - return field.ErrorList{} + return allErrs } // ValidateResourceQuantityValue enforces that specified quantity is valid for specified resource @@ -122,7 +125,7 @@ func validateResourceName(value string, fldPath *field.Path) field.ErrorList { } } - return field.ErrorList{} + return allErrs } func ValidatePodLogOptions(opts *v1.PodLogOptions) field.ErrorList { diff --git a/pkg/api/v1/validation/validation_test.go b/pkg/api/v1/validation/validation_test.go new file mode 100644 index 0000000000000..2a33d7ebdc520 --- /dev/null +++ b/pkg/api/v1/validation/validation_test.go @@ -0,0 +1,179 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validation + +import ( + "testing" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func TestValidateResourceRequirements(t *testing.T) { + successCase := []struct { + Name string + requirements v1.ResourceRequirements + }{ + { + Name: "GPU only setting Limits", + requirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("10"), + }, + }, + }, + { + Name: "GPU setting Limits equals Requests", + requirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("10"), + }, + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("10"), + }, + }, + }, + { + Name: "Resources with GPU with Requests", + requirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("10"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("10G"), + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("1"), + }, + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("10"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("10G"), + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("1"), + }, + }, + }, + { + Name: "Resources with only Limits", + requirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("10"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("10G"), + v1.ResourceName("my.org/resource"): resource.MustParse("10"), + }, + }, + }, + { + Name: "Resources with only Requests", + requirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("10"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("10G"), + v1.ResourceName("my.org/resource"): resource.MustParse("10"), + }, + }, + }, + { + Name: "Resources with Requests Less Than Limits", + requirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("9"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("9G"), + v1.ResourceName("my.org/resource"): resource.MustParse("9"), + }, + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("10"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("10G"), + v1.ResourceName("my.org/resource"): resource.MustParse("9"), + }, + }, + }, + } + for _, tc := range successCase { + if errs := ValidateResourceRequirements(&tc.requirements, field.NewPath("resources")); len(errs) != 0 { + t.Errorf("%q unexpected error: %v", tc.Name, errs) + } + } + + errorCase := []struct { + Name string + requirements v1.ResourceRequirements + }{ + { + Name: "GPU only setting Requests", + requirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("10"), + }, + }, + }, + { + Name: "GPU setting Limits less than Requests", + requirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("10"), + }, + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("11"), + }, + }, + }, + { + Name: "GPU setting Limits larger than Requests", + requirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("10"), + }, + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceNvidiaGPU): resource.MustParse("9"), + }, + }, + }, + { + Name: "Resources with Requests Larger Than Limits", + requirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("10"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("10G"), + v1.ResourceName("my.org/resource"): resource.MustParse("10m"), + }, + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("9"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("9G"), + v1.ResourceName("my.org/resource"): resource.MustParse("9m"), + }, + }, + }, + { + Name: "Invalid Resources with Requests", + requirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName("my.org"): resource.MustParse("10m"), + }, + }, + }, + { + Name: "Invalid Resources with Limits", + requirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceName("my.org"): resource.MustParse("9m"), + }, + }, + }, + } + for _, tc := range errorCase { + if errs := ValidateResourceRequirements(&tc.requirements, field.NewPath("resources")); len(errs) == 0 { + t.Errorf("%q expected error", tc.Name) + } + } +} diff --git a/pkg/api/v1/zz_generated.conversion.go b/pkg/api/v1/zz_generated.conversion.go index 53c9d37395585..0d1057879b9ab 100644 --- a/pkg/api/v1/zz_generated.conversion.go +++ b/pkg/api/v1/zz_generated.conversion.go @@ -48,16 +48,22 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_api_AvoidPods_To_v1_AvoidPods, Convert_v1_AzureDiskVolumeSource_To_api_AzureDiskVolumeSource, Convert_api_AzureDiskVolumeSource_To_v1_AzureDiskVolumeSource, + Convert_v1_AzureFilePersistentVolumeSource_To_api_AzureFilePersistentVolumeSource, + Convert_api_AzureFilePersistentVolumeSource_To_v1_AzureFilePersistentVolumeSource, Convert_v1_AzureFileVolumeSource_To_api_AzureFileVolumeSource, Convert_api_AzureFileVolumeSource_To_v1_AzureFileVolumeSource, Convert_v1_Binding_To_api_Binding, Convert_api_Binding_To_v1_Binding, Convert_v1_Capabilities_To_api_Capabilities, Convert_api_Capabilities_To_v1_Capabilities, + Convert_v1_CephFSPersistentVolumeSource_To_api_CephFSPersistentVolumeSource, + Convert_api_CephFSPersistentVolumeSource_To_v1_CephFSPersistentVolumeSource, Convert_v1_CephFSVolumeSource_To_api_CephFSVolumeSource, Convert_api_CephFSVolumeSource_To_v1_CephFSVolumeSource, Convert_v1_CinderVolumeSource_To_api_CinderVolumeSource, Convert_api_CinderVolumeSource_To_v1_CinderVolumeSource, + Convert_v1_ClientIPConfig_To_api_ClientIPConfig, + Convert_api_ClientIPConfig_To_v1_ClientIPConfig, Convert_v1_ComponentCondition_To_api_ComponentCondition, Convert_api_ComponentCondition_To_v1_ComponentCondition, Convert_v1_ComponentStatus_To_api_ComponentStatus, @@ -336,6 +342,8 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_api_SecretList_To_v1_SecretList, Convert_v1_SecretProjection_To_api_SecretProjection, Convert_api_SecretProjection_To_v1_SecretProjection, + Convert_v1_SecretReference_To_api_SecretReference, + Convert_api_SecretReference_To_v1_SecretReference, Convert_v1_SecretVolumeSource_To_api_SecretVolumeSource, Convert_api_SecretVolumeSource_To_v1_SecretVolumeSource, Convert_v1_SecurityContext_To_api_SecurityContext, @@ -358,6 +366,8 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_api_ServiceSpec_To_v1_ServiceSpec, Convert_v1_ServiceStatus_To_api_ServiceStatus, Convert_api_ServiceStatus_To_v1_ServiceStatus, + Convert_v1_SessionAffinityConfig_To_api_SessionAffinityConfig, + Convert_api_SessionAffinityConfig_To_v1_SessionAffinityConfig, Convert_v1_StorageOSPersistentVolumeSource_To_api_StorageOSPersistentVolumeSource, Convert_api_StorageOSPersistentVolumeSource_To_v1_StorageOSPersistentVolumeSource, Convert_v1_StorageOSVolumeSource_To_api_StorageOSVolumeSource, @@ -507,6 +517,32 @@ func Convert_api_AzureDiskVolumeSource_To_v1_AzureDiskVolumeSource(in *api.Azure return autoConvert_api_AzureDiskVolumeSource_To_v1_AzureDiskVolumeSource(in, out, s) } +func autoConvert_v1_AzureFilePersistentVolumeSource_To_api_AzureFilePersistentVolumeSource(in *v1.AzureFilePersistentVolumeSource, out *api.AzureFilePersistentVolumeSource, s conversion.Scope) error { + out.SecretName = in.SecretName + out.ShareName = in.ShareName + out.ReadOnly = in.ReadOnly + out.SecretNamespace = (*string)(unsafe.Pointer(in.SecretNamespace)) + return nil +} + +// Convert_v1_AzureFilePersistentVolumeSource_To_api_AzureFilePersistentVolumeSource is an autogenerated conversion function. +func Convert_v1_AzureFilePersistentVolumeSource_To_api_AzureFilePersistentVolumeSource(in *v1.AzureFilePersistentVolumeSource, out *api.AzureFilePersistentVolumeSource, s conversion.Scope) error { + return autoConvert_v1_AzureFilePersistentVolumeSource_To_api_AzureFilePersistentVolumeSource(in, out, s) +} + +func autoConvert_api_AzureFilePersistentVolumeSource_To_v1_AzureFilePersistentVolumeSource(in *api.AzureFilePersistentVolumeSource, out *v1.AzureFilePersistentVolumeSource, s conversion.Scope) error { + out.SecretName = in.SecretName + out.ShareName = in.ShareName + out.ReadOnly = in.ReadOnly + out.SecretNamespace = (*string)(unsafe.Pointer(in.SecretNamespace)) + return nil +} + +// Convert_api_AzureFilePersistentVolumeSource_To_v1_AzureFilePersistentVolumeSource is an autogenerated conversion function. +func Convert_api_AzureFilePersistentVolumeSource_To_v1_AzureFilePersistentVolumeSource(in *api.AzureFilePersistentVolumeSource, out *v1.AzureFilePersistentVolumeSource, s conversion.Scope) error { + return autoConvert_api_AzureFilePersistentVolumeSource_To_v1_AzureFilePersistentVolumeSource(in, out, s) +} + func autoConvert_v1_AzureFileVolumeSource_To_api_AzureFileVolumeSource(in *v1.AzureFileVolumeSource, out *api.AzureFileVolumeSource, s conversion.Scope) error { out.SecretName = in.SecretName out.ShareName = in.ShareName @@ -579,6 +615,36 @@ func Convert_api_Capabilities_To_v1_Capabilities(in *api.Capabilities, out *v1.C return autoConvert_api_Capabilities_To_v1_Capabilities(in, out, s) } +func autoConvert_v1_CephFSPersistentVolumeSource_To_api_CephFSPersistentVolumeSource(in *v1.CephFSPersistentVolumeSource, out *api.CephFSPersistentVolumeSource, s conversion.Scope) error { + out.Monitors = *(*[]string)(unsafe.Pointer(&in.Monitors)) + out.Path = in.Path + out.User = in.User + out.SecretFile = in.SecretFile + out.SecretRef = (*api.SecretReference)(unsafe.Pointer(in.SecretRef)) + out.ReadOnly = in.ReadOnly + return nil +} + +// Convert_v1_CephFSPersistentVolumeSource_To_api_CephFSPersistentVolumeSource is an autogenerated conversion function. +func Convert_v1_CephFSPersistentVolumeSource_To_api_CephFSPersistentVolumeSource(in *v1.CephFSPersistentVolumeSource, out *api.CephFSPersistentVolumeSource, s conversion.Scope) error { + return autoConvert_v1_CephFSPersistentVolumeSource_To_api_CephFSPersistentVolumeSource(in, out, s) +} + +func autoConvert_api_CephFSPersistentVolumeSource_To_v1_CephFSPersistentVolumeSource(in *api.CephFSPersistentVolumeSource, out *v1.CephFSPersistentVolumeSource, s conversion.Scope) error { + out.Monitors = *(*[]string)(unsafe.Pointer(&in.Monitors)) + out.Path = in.Path + out.User = in.User + out.SecretFile = in.SecretFile + out.SecretRef = (*v1.SecretReference)(unsafe.Pointer(in.SecretRef)) + out.ReadOnly = in.ReadOnly + return nil +} + +// Convert_api_CephFSPersistentVolumeSource_To_v1_CephFSPersistentVolumeSource is an autogenerated conversion function. +func Convert_api_CephFSPersistentVolumeSource_To_v1_CephFSPersistentVolumeSource(in *api.CephFSPersistentVolumeSource, out *v1.CephFSPersistentVolumeSource, s conversion.Scope) error { + return autoConvert_api_CephFSPersistentVolumeSource_To_v1_CephFSPersistentVolumeSource(in, out, s) +} + func autoConvert_v1_CephFSVolumeSource_To_api_CephFSVolumeSource(in *v1.CephFSVolumeSource, out *api.CephFSVolumeSource, s conversion.Scope) error { out.Monitors = *(*[]string)(unsafe.Pointer(&in.Monitors)) out.Path = in.Path @@ -595,11 +661,7 @@ func Convert_v1_CephFSVolumeSource_To_api_CephFSVolumeSource(in *v1.CephFSVolume } func autoConvert_api_CephFSVolumeSource_To_v1_CephFSVolumeSource(in *api.CephFSVolumeSource, out *v1.CephFSVolumeSource, s conversion.Scope) error { - if in.Monitors == nil { - out.Monitors = make([]string, 0) - } else { - out.Monitors = *(*[]string)(unsafe.Pointer(&in.Monitors)) - } + out.Monitors = *(*[]string)(unsafe.Pointer(&in.Monitors)) out.Path = in.Path out.User = in.User out.SecretFile = in.SecretFile @@ -637,6 +699,26 @@ func Convert_api_CinderVolumeSource_To_v1_CinderVolumeSource(in *api.CinderVolum return autoConvert_api_CinderVolumeSource_To_v1_CinderVolumeSource(in, out, s) } +func autoConvert_v1_ClientIPConfig_To_api_ClientIPConfig(in *v1.ClientIPConfig, out *api.ClientIPConfig, s conversion.Scope) error { + out.TimeoutSeconds = (*int32)(unsafe.Pointer(in.TimeoutSeconds)) + return nil +} + +// Convert_v1_ClientIPConfig_To_api_ClientIPConfig is an autogenerated conversion function. +func Convert_v1_ClientIPConfig_To_api_ClientIPConfig(in *v1.ClientIPConfig, out *api.ClientIPConfig, s conversion.Scope) error { + return autoConvert_v1_ClientIPConfig_To_api_ClientIPConfig(in, out, s) +} + +func autoConvert_api_ClientIPConfig_To_v1_ClientIPConfig(in *api.ClientIPConfig, out *v1.ClientIPConfig, s conversion.Scope) error { + out.TimeoutSeconds = (*int32)(unsafe.Pointer(in.TimeoutSeconds)) + return nil +} + +// Convert_api_ClientIPConfig_To_v1_ClientIPConfig is an autogenerated conversion function. +func Convert_api_ClientIPConfig_To_v1_ClientIPConfig(in *api.ClientIPConfig, out *v1.ClientIPConfig, s conversion.Scope) error { + return autoConvert_api_ClientIPConfig_To_v1_ClientIPConfig(in, out, s) +} + func autoConvert_v1_ComponentCondition_To_api_ComponentCondition(in *v1.ComponentCondition, out *api.ComponentCondition, s conversion.Scope) error { out.Type = api.ComponentConditionType(in.Type) out.Status = api.ConditionStatus(in.Status) @@ -698,11 +780,7 @@ func Convert_v1_ComponentStatusList_To_api_ComponentStatusList(in *v1.ComponentS func autoConvert_api_ComponentStatusList_To_v1_ComponentStatusList(in *api.ComponentStatusList, out *v1.ComponentStatusList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.ComponentStatus, 0) - } else { - out.Items = *(*[]v1.ComponentStatus)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.ComponentStatus)(unsafe.Pointer(&in.Items)) return nil } @@ -800,11 +878,7 @@ func Convert_v1_ConfigMapList_To_api_ConfigMapList(in *v1.ConfigMapList, out *ap func autoConvert_api_ConfigMapList_To_v1_ConfigMapList(in *api.ConfigMapList, out *v1.ConfigMapList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.ConfigMap, 0) - } else { - out.Items = *(*[]v1.ConfigMap)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.ConfigMap)(unsafe.Pointer(&in.Items)) return nil } @@ -961,11 +1035,7 @@ func Convert_v1_ContainerImage_To_api_ContainerImage(in *v1.ContainerImage, out } func autoConvert_api_ContainerImage_To_v1_ContainerImage(in *api.ContainerImage, out *v1.ContainerImage, s conversion.Scope) error { - if in.Names == nil { - out.Names = make([]string, 0) - } else { - out.Names = *(*[]string)(unsafe.Pointer(&in.Names)) - } + out.Names = *(*[]string)(unsafe.Pointer(&in.Names)) out.SizeBytes = in.SizeBytes return nil } @@ -1366,11 +1436,7 @@ func Convert_v1_Endpoints_To_api_Endpoints(in *v1.Endpoints, out *api.Endpoints, func autoConvert_api_Endpoints_To_v1_Endpoints(in *api.Endpoints, out *v1.Endpoints, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Subsets == nil { - out.Subsets = make([]v1.EndpointSubset, 0) - } else { - out.Subsets = *(*[]v1.EndpointSubset)(unsafe.Pointer(&in.Subsets)) - } + out.Subsets = *(*[]v1.EndpointSubset)(unsafe.Pointer(&in.Subsets)) return nil } @@ -1392,11 +1458,7 @@ func Convert_v1_EndpointsList_To_api_EndpointsList(in *v1.EndpointsList, out *ap func autoConvert_api_EndpointsList_To_v1_EndpointsList(in *api.EndpointsList, out *v1.EndpointsList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.Endpoints, 0) - } else { - out.Items = *(*[]v1.Endpoints)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.Endpoints)(unsafe.Pointer(&in.Items)) return nil } @@ -1536,11 +1598,7 @@ func Convert_v1_EventList_To_api_EventList(in *v1.EventList, out *api.EventList, func autoConvert_api_EventList_To_v1_EventList(in *api.EventList, out *v1.EventList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.Event, 0) - } else { - out.Items = *(*[]v1.Event)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.Event)(unsafe.Pointer(&in.Items)) return nil } @@ -1841,6 +1899,7 @@ func Convert_api_HostAlias_To_v1_HostAlias(in *api.HostAlias, out *v1.HostAlias, func autoConvert_v1_HostPathVolumeSource_To_api_HostPathVolumeSource(in *v1.HostPathVolumeSource, out *api.HostPathVolumeSource, s conversion.Scope) error { out.Path = in.Path + out.Type = (*api.HostPathType)(unsafe.Pointer(in.Type)) return nil } @@ -1851,6 +1910,7 @@ func Convert_v1_HostPathVolumeSource_To_api_HostPathVolumeSource(in *v1.HostPath func autoConvert_api_HostPathVolumeSource_To_v1_HostPathVolumeSource(in *api.HostPathVolumeSource, out *v1.HostPathVolumeSource, s conversion.Scope) error { out.Path = in.Path + out.Type = (*v1.HostPathType)(unsafe.Pointer(in.Type)) return nil } @@ -1870,6 +1930,7 @@ func autoConvert_v1_ISCSIVolumeSource_To_api_ISCSIVolumeSource(in *v1.ISCSIVolum out.DiscoveryCHAPAuth = in.DiscoveryCHAPAuth out.SessionCHAPAuth = in.SessionCHAPAuth out.SecretRef = (*api.LocalObjectReference)(unsafe.Pointer(in.SecretRef)) + out.InitiatorName = (*string)(unsafe.Pointer(in.InitiatorName)) return nil } @@ -1889,6 +1950,7 @@ func autoConvert_api_ISCSIVolumeSource_To_v1_ISCSIVolumeSource(in *api.ISCSIVolu out.DiscoveryCHAPAuth = in.DiscoveryCHAPAuth out.SessionCHAPAuth = in.SessionCHAPAuth out.SecretRef = (*v1.LocalObjectReference)(unsafe.Pointer(in.SecretRef)) + out.InitiatorName = (*string)(unsafe.Pointer(in.InitiatorName)) return nil } @@ -2012,11 +2074,7 @@ func Convert_v1_LimitRangeList_To_api_LimitRangeList(in *v1.LimitRangeList, out func autoConvert_api_LimitRangeList_To_v1_LimitRangeList(in *api.LimitRangeList, out *v1.LimitRangeList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.LimitRange, 0) - } else { - out.Items = *(*[]v1.LimitRange)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.LimitRange)(unsafe.Pointer(&in.Items)) return nil } @@ -2036,11 +2094,7 @@ func Convert_v1_LimitRangeSpec_To_api_LimitRangeSpec(in *v1.LimitRangeSpec, out } func autoConvert_api_LimitRangeSpec_To_v1_LimitRangeSpec(in *api.LimitRangeSpec, out *v1.LimitRangeSpec, s conversion.Scope) error { - if in.Limits == nil { - out.Limits = make([]v1.LimitRangeItem, 0) - } else { - out.Limits = *(*[]v1.LimitRangeItem)(unsafe.Pointer(&in.Limits)) - } + out.Limits = *(*[]v1.LimitRangeItem)(unsafe.Pointer(&in.Limits)) return nil } @@ -2081,7 +2135,7 @@ func autoConvert_api_List_To_v1_List(in *api.List, out *v1.List, s conversion.Sc } } } else { - out.Items = make([]runtime.RawExtension, 0) + out.Items = nil } return nil } @@ -2280,11 +2334,7 @@ func Convert_v1_NamespaceList_To_api_NamespaceList(in *v1.NamespaceList, out *ap func autoConvert_api_NamespaceList_To_v1_NamespaceList(in *api.NamespaceList, out *v1.NamespaceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.Namespace, 0) - } else { - out.Items = *(*[]v1.Namespace)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.Namespace)(unsafe.Pointer(&in.Items)) return nil } @@ -2496,11 +2546,7 @@ func Convert_v1_NodeList_To_api_NodeList(in *v1.NodeList, out *api.NodeList, s c func autoConvert_api_NodeList_To_v1_NodeList(in *api.NodeList, out *v1.NodeList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.Node, 0) - } else { - out.Items = *(*[]v1.Node)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.Node)(unsafe.Pointer(&in.Items)) return nil } @@ -2560,11 +2606,7 @@ func Convert_v1_NodeSelector_To_api_NodeSelector(in *v1.NodeSelector, out *api.N } func autoConvert_api_NodeSelector_To_v1_NodeSelector(in *api.NodeSelector, out *v1.NodeSelector, s conversion.Scope) error { - if in.NodeSelectorTerms == nil { - out.NodeSelectorTerms = make([]v1.NodeSelectorTerm, 0) - } else { - out.NodeSelectorTerms = *(*[]v1.NodeSelectorTerm)(unsafe.Pointer(&in.NodeSelectorTerms)) - } + out.NodeSelectorTerms = *(*[]v1.NodeSelectorTerm)(unsafe.Pointer(&in.NodeSelectorTerms)) return nil } @@ -2608,11 +2650,7 @@ func Convert_v1_NodeSelectorTerm_To_api_NodeSelectorTerm(in *v1.NodeSelectorTerm } func autoConvert_api_NodeSelectorTerm_To_v1_NodeSelectorTerm(in *api.NodeSelectorTerm, out *v1.NodeSelectorTerm, s conversion.Scope) error { - if in.MatchExpressions == nil { - out.MatchExpressions = make([]v1.NodeSelectorRequirement, 0) - } else { - out.MatchExpressions = *(*[]v1.NodeSelectorRequirement)(unsafe.Pointer(&in.MatchExpressions)) - } + out.MatchExpressions = *(*[]v1.NodeSelectorRequirement)(unsafe.Pointer(&in.MatchExpressions)) return nil } @@ -2916,11 +2954,7 @@ func Convert_v1_PersistentVolumeClaimList_To_api_PersistentVolumeClaimList(in *v func autoConvert_api_PersistentVolumeClaimList_To_v1_PersistentVolumeClaimList(in *api.PersistentVolumeClaimList, out *v1.PersistentVolumeClaimList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.PersistentVolumeClaim, 0) - } else { - out.Items = *(*[]v1.PersistentVolumeClaim)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.PersistentVolumeClaim)(unsafe.Pointer(&in.Items)) return nil } @@ -3039,7 +3073,7 @@ func autoConvert_api_PersistentVolumeList_To_v1_PersistentVolumeList(in *api.Per } } } else { - out.Items = make([]v1.PersistentVolume, 0) + out.Items = nil } return nil } @@ -3058,11 +3092,11 @@ func autoConvert_v1_PersistentVolumeSource_To_api_PersistentVolumeSource(in *v1. out.RBD = (*api.RBDVolumeSource)(unsafe.Pointer(in.RBD)) out.ISCSI = (*api.ISCSIVolumeSource)(unsafe.Pointer(in.ISCSI)) out.Cinder = (*api.CinderVolumeSource)(unsafe.Pointer(in.Cinder)) - out.CephFS = (*api.CephFSVolumeSource)(unsafe.Pointer(in.CephFS)) + out.CephFS = (*api.CephFSPersistentVolumeSource)(unsafe.Pointer(in.CephFS)) out.FC = (*api.FCVolumeSource)(unsafe.Pointer(in.FC)) out.Flocker = (*api.FlockerVolumeSource)(unsafe.Pointer(in.Flocker)) out.FlexVolume = (*api.FlexVolumeSource)(unsafe.Pointer(in.FlexVolume)) - out.AzureFile = (*api.AzureFileVolumeSource)(unsafe.Pointer(in.AzureFile)) + out.AzureFile = (*api.AzureFilePersistentVolumeSource)(unsafe.Pointer(in.AzureFile)) out.VsphereVolume = (*api.VsphereVirtualDiskVolumeSource)(unsafe.Pointer(in.VsphereVolume)) out.Quobyte = (*api.QuobyteVolumeSource)(unsafe.Pointer(in.Quobyte)) out.AzureDisk = (*api.AzureDiskVolumeSource)(unsafe.Pointer(in.AzureDisk)) @@ -3090,10 +3124,10 @@ func autoConvert_api_PersistentVolumeSource_To_v1_PersistentVolumeSource(in *api out.ISCSI = (*v1.ISCSIVolumeSource)(unsafe.Pointer(in.ISCSI)) out.FlexVolume = (*v1.FlexVolumeSource)(unsafe.Pointer(in.FlexVolume)) out.Cinder = (*v1.CinderVolumeSource)(unsafe.Pointer(in.Cinder)) - out.CephFS = (*v1.CephFSVolumeSource)(unsafe.Pointer(in.CephFS)) + out.CephFS = (*v1.CephFSPersistentVolumeSource)(unsafe.Pointer(in.CephFS)) out.FC = (*v1.FCVolumeSource)(unsafe.Pointer(in.FC)) out.Flocker = (*v1.FlockerVolumeSource)(unsafe.Pointer(in.Flocker)) - out.AzureFile = (*v1.AzureFileVolumeSource)(unsafe.Pointer(in.AzureFile)) + out.AzureFile = (*v1.AzureFilePersistentVolumeSource)(unsafe.Pointer(in.AzureFile)) out.VsphereVolume = (*v1.VsphereVirtualDiskVolumeSource)(unsafe.Pointer(in.VsphereVolume)) out.AzureDisk = (*v1.AzureDiskVolumeSource)(unsafe.Pointer(in.AzureDisk)) out.PhotonPersistentDisk = (*v1.PhotonPersistentDiskVolumeSource)(unsafe.Pointer(in.PhotonPersistentDisk)) @@ -3118,6 +3152,7 @@ func autoConvert_v1_PersistentVolumeSpec_To_api_PersistentVolumeSpec(in *v1.Pers out.ClaimRef = (*api.ObjectReference)(unsafe.Pointer(in.ClaimRef)) out.PersistentVolumeReclaimPolicy = api.PersistentVolumeReclaimPolicy(in.PersistentVolumeReclaimPolicy) out.StorageClassName = in.StorageClassName + out.MountOptions = *(*[]string)(unsafe.Pointer(&in.MountOptions)) return nil } @@ -3135,6 +3170,7 @@ func autoConvert_api_PersistentVolumeSpec_To_v1_PersistentVolumeSpec(in *api.Per out.ClaimRef = (*v1.ObjectReference)(unsafe.Pointer(in.ClaimRef)) out.PersistentVolumeReclaimPolicy = v1.PersistentVolumeReclaimPolicy(in.PersistentVolumeReclaimPolicy) out.StorageClassName = in.StorageClassName + out.MountOptions = *(*[]string)(unsafe.Pointer(&in.MountOptions)) return nil } @@ -3358,11 +3394,7 @@ func autoConvert_api_PodExecOptions_To_v1_PodExecOptions(in *api.PodExecOptions, out.Stderr = in.Stderr out.TTY = in.TTY out.Container = in.Container - if in.Command == nil { - out.Command = make([]string, 0) - } else { - out.Command = *(*[]string)(unsafe.Pointer(&in.Command)) - } + out.Command = *(*[]string)(unsafe.Pointer(&in.Command)) return nil } @@ -3403,7 +3435,7 @@ func autoConvert_api_PodList_To_v1_PodList(in *api.PodList, out *v1.PodList, s c } } } else { - out.Items = make([]v1.Pod, 0) + out.Items = nil } return nil } @@ -3627,7 +3659,7 @@ func autoConvert_api_PodSpec_To_v1_PodSpec(in *api.PodSpec, out *v1.PodSpec, s c } } } else { - out.Containers = make([]v1.Container, 0) + out.Containers = nil } out.RestartPolicy = v1.RestartPolicy(in.RestartPolicy) out.TerminationGracePeriodSeconds = (*int64)(unsafe.Pointer(in.TerminationGracePeriodSeconds)) @@ -3770,7 +3802,7 @@ func autoConvert_api_PodTemplateList_To_v1_PodTemplateList(in *api.PodTemplateLi } } } else { - out.Items = make([]v1.PodTemplate, 0) + out.Items = nil } return nil } @@ -3942,11 +3974,7 @@ func Convert_v1_ProjectedVolumeSource_To_api_ProjectedVolumeSource(in *v1.Projec } func autoConvert_api_ProjectedVolumeSource_To_v1_ProjectedVolumeSource(in *api.ProjectedVolumeSource, out *v1.ProjectedVolumeSource, s conversion.Scope) error { - if in.Sources == nil { - out.Sources = make([]v1.VolumeProjection, 0) - } else { - out.Sources = *(*[]v1.VolumeProjection)(unsafe.Pointer(&in.Sources)) - } + out.Sources = *(*[]v1.VolumeProjection)(unsafe.Pointer(&in.Sources)) out.DefaultMode = (*int32)(unsafe.Pointer(in.DefaultMode)) return nil } @@ -4002,11 +4030,7 @@ func Convert_v1_RBDVolumeSource_To_api_RBDVolumeSource(in *v1.RBDVolumeSource, o } func autoConvert_api_RBDVolumeSource_To_v1_RBDVolumeSource(in *api.RBDVolumeSource, out *v1.RBDVolumeSource, s conversion.Scope) error { - if in.CephMonitors == nil { - out.CephMonitors = make([]string, 0) - } else { - out.CephMonitors = *(*[]string)(unsafe.Pointer(&in.CephMonitors)) - } + out.CephMonitors = *(*[]string)(unsafe.Pointer(&in.CephMonitors)) out.RBDImage = in.RBDImage out.FSType = in.FSType out.RBDPool = in.RBDPool @@ -4037,11 +4061,7 @@ func Convert_v1_RangeAllocation_To_api_RangeAllocation(in *v1.RangeAllocation, o func autoConvert_api_RangeAllocation_To_v1_RangeAllocation(in *api.RangeAllocation, out *v1.RangeAllocation, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta out.Range = in.Range - if in.Data == nil { - out.Data = make([]byte, 0) - } else { - out.Data = *(*[]byte)(unsafe.Pointer(&in.Data)) - } + out.Data = *(*[]byte)(unsafe.Pointer(&in.Data)) return nil } @@ -4142,7 +4162,7 @@ func autoConvert_api_ReplicationControllerList_To_v1_ReplicationControllerList(i } } } else { - out.Items = make([]v1.ReplicationController, 0) + out.Items = nil } return nil } @@ -4287,11 +4307,7 @@ func Convert_v1_ResourceQuotaList_To_api_ResourceQuotaList(in *v1.ResourceQuotaL func autoConvert_api_ResourceQuotaList_To_v1_ResourceQuotaList(in *api.ResourceQuotaList, out *v1.ResourceQuotaList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.ResourceQuota, 0) - } else { - out.Items = *(*[]v1.ResourceQuota)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.ResourceQuota)(unsafe.Pointer(&in.Items)) return nil } @@ -4536,7 +4552,7 @@ func autoConvert_api_SecretList_To_v1_SecretList(in *api.SecretList, out *v1.Sec } } } else { - out.Items = make([]v1.Secret, 0) + out.Items = nil } return nil } @@ -4574,6 +4590,28 @@ func Convert_api_SecretProjection_To_v1_SecretProjection(in *api.SecretProjectio return autoConvert_api_SecretProjection_To_v1_SecretProjection(in, out, s) } +func autoConvert_v1_SecretReference_To_api_SecretReference(in *v1.SecretReference, out *api.SecretReference, s conversion.Scope) error { + out.Name = in.Name + out.Namespace = in.Namespace + return nil +} + +// Convert_v1_SecretReference_To_api_SecretReference is an autogenerated conversion function. +func Convert_v1_SecretReference_To_api_SecretReference(in *v1.SecretReference, out *api.SecretReference, s conversion.Scope) error { + return autoConvert_v1_SecretReference_To_api_SecretReference(in, out, s) +} + +func autoConvert_api_SecretReference_To_v1_SecretReference(in *api.SecretReference, out *v1.SecretReference, s conversion.Scope) error { + out.Name = in.Name + out.Namespace = in.Namespace + return nil +} + +// Convert_api_SecretReference_To_v1_SecretReference is an autogenerated conversion function. +func Convert_api_SecretReference_To_v1_SecretReference(in *api.SecretReference, out *v1.SecretReference, s conversion.Scope) error { + return autoConvert_api_SecretReference_To_v1_SecretReference(in, out, s) +} + func autoConvert_v1_SecretVolumeSource_To_api_SecretVolumeSource(in *v1.SecretVolumeSource, out *api.SecretVolumeSource, s conversion.Scope) error { out.SecretName = in.SecretName out.Items = *(*[]api.KeyToPath)(unsafe.Pointer(&in.Items)) @@ -4722,11 +4760,7 @@ func Convert_v1_ServiceAccountList_To_api_ServiceAccountList(in *v1.ServiceAccou func autoConvert_api_ServiceAccountList_To_v1_ServiceAccountList(in *api.ServiceAccountList, out *v1.ServiceAccountList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.ServiceAccount, 0) - } else { - out.Items = *(*[]v1.ServiceAccount)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.ServiceAccount)(unsafe.Pointer(&in.Items)) return nil } @@ -4767,7 +4801,7 @@ func autoConvert_api_ServiceList_To_v1_ServiceList(in *api.ServiceList, out *v1. } } } else { - out.Items = make([]v1.Service, 0) + out.Items = nil } return nil } @@ -4838,6 +4872,7 @@ func autoConvert_v1_ServiceSpec_To_api_ServiceSpec(in *v1.ServiceSpec, out *api. out.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyType(in.ExternalTrafficPolicy) out.HealthCheckNodePort = in.HealthCheckNodePort out.PublishNotReadyAddresses = in.PublishNotReadyAddresses + out.SessionAffinityConfig = (*api.SessionAffinityConfig)(unsafe.Pointer(in.SessionAffinityConfig)) return nil } @@ -4855,6 +4890,7 @@ func autoConvert_api_ServiceSpec_To_v1_ServiceSpec(in *api.ServiceSpec, out *v1. out.ExternalIPs = *(*[]string)(unsafe.Pointer(&in.ExternalIPs)) out.LoadBalancerIP = in.LoadBalancerIP out.SessionAffinity = v1.ServiceAffinity(in.SessionAffinity) + out.SessionAffinityConfig = (*v1.SessionAffinityConfig)(unsafe.Pointer(in.SessionAffinityConfig)) out.LoadBalancerSourceRanges = *(*[]string)(unsafe.Pointer(&in.LoadBalancerSourceRanges)) out.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(in.ExternalTrafficPolicy) out.HealthCheckNodePort = in.HealthCheckNodePort @@ -4891,6 +4927,26 @@ func Convert_api_ServiceStatus_To_v1_ServiceStatus(in *api.ServiceStatus, out *v return autoConvert_api_ServiceStatus_To_v1_ServiceStatus(in, out, s) } +func autoConvert_v1_SessionAffinityConfig_To_api_SessionAffinityConfig(in *v1.SessionAffinityConfig, out *api.SessionAffinityConfig, s conversion.Scope) error { + out.ClientIP = (*api.ClientIPConfig)(unsafe.Pointer(in.ClientIP)) + return nil +} + +// Convert_v1_SessionAffinityConfig_To_api_SessionAffinityConfig is an autogenerated conversion function. +func Convert_v1_SessionAffinityConfig_To_api_SessionAffinityConfig(in *v1.SessionAffinityConfig, out *api.SessionAffinityConfig, s conversion.Scope) error { + return autoConvert_v1_SessionAffinityConfig_To_api_SessionAffinityConfig(in, out, s) +} + +func autoConvert_api_SessionAffinityConfig_To_v1_SessionAffinityConfig(in *api.SessionAffinityConfig, out *v1.SessionAffinityConfig, s conversion.Scope) error { + out.ClientIP = (*v1.ClientIPConfig)(unsafe.Pointer(in.ClientIP)) + return nil +} + +// Convert_api_SessionAffinityConfig_To_v1_SessionAffinityConfig is an autogenerated conversion function. +func Convert_api_SessionAffinityConfig_To_v1_SessionAffinityConfig(in *api.SessionAffinityConfig, out *v1.SessionAffinityConfig, s conversion.Scope) error { + return autoConvert_api_SessionAffinityConfig_To_v1_SessionAffinityConfig(in, out, s) +} + func autoConvert_v1_StorageOSPersistentVolumeSource_To_api_StorageOSPersistentVolumeSource(in *v1.StorageOSPersistentVolumeSource, out *api.StorageOSPersistentVolumeSource, s conversion.Scope) error { out.VolumeName = in.VolumeName out.VolumeNamespace = in.VolumeNamespace diff --git a/pkg/api/v1/zz_generated.defaults.go b/pkg/api/v1/zz_generated.defaults.go index b0b0c6ece9e43..c41d6ae3e37ba 100644 --- a/pkg/api/v1/zz_generated.defaults.go +++ b/pkg/api/v1/zz_generated.defaults.go @@ -133,6 +133,9 @@ func SetObjectDefaults_NodeList(in *v1.NodeList) { func SetObjectDefaults_PersistentVolume(in *v1.PersistentVolume) { SetDefaults_PersistentVolume(in) SetDefaults_ResourceList(&in.Spec.Capacity) + if in.Spec.PersistentVolumeSource.HostPath != nil { + SetDefaults_HostPathVolumeSource(in.Spec.PersistentVolumeSource.HostPath) + } if in.Spec.PersistentVolumeSource.RBD != nil { SetDefaults_RBDVolumeSource(in.Spec.PersistentVolumeSource.RBD) } @@ -174,6 +177,9 @@ func SetObjectDefaults_Pod(in *v1.Pod) { for i := range in.Spec.Volumes { a := &in.Spec.Volumes[i] SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -322,6 +328,9 @@ func SetObjectDefaults_PodTemplate(in *v1.PodTemplate) { for i := range in.Template.Spec.Volumes { a := &in.Template.Spec.Volumes[i] SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -464,6 +473,9 @@ func SetObjectDefaults_ReplicationController(in *v1.ReplicationController) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } diff --git a/pkg/api/validation/BUILD b/pkg/api/validation/BUILD index c0b1cbd948503..8107e7f45d183 100644 --- a/pkg/api/validation/BUILD +++ b/pkg/api/validation/BUILD @@ -32,6 +32,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/validation:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library", diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index d40d5a048ef92..5e07a4587291a 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -19,6 +19,7 @@ package validation import ( "encoding/json" "fmt" + "math" "net" "path" "path/filepath" @@ -28,8 +29,6 @@ import ( "github.com/golang/glog" - "math" - "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource" @@ -38,6 +37,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" @@ -68,6 +68,10 @@ var volumeModeErrorMsg string = "must be a number between 0 and 0777 (octal), bo // BannedOwners is a black list of object that are not allowed to be owners. var BannedOwners = genericvalidation.BannedOwners +var iscsiInitiatorIqnRegex = regexp.MustCompile(`iqn\.\d{4}-\d{2}\.([[:alnum:]-.]+)(:[^,;*&$|\s]+)$`) +var iscsiInitiatorEuiRegex = regexp.MustCompile(`^eui.[[:alnum:]]{16}$`) +var iscsiInitiatorNaaRegex = regexp.MustCompile(`^naa.[[:alnum:]]{32}$`) + // ValidateHasLabel requires that metav1.ObjectMeta has a Label with key and expectedValue func ValidateHasLabel(meta metav1.ObjectMeta, fldPath *field.Path, key, expectedValue string) field.ErrorList { allErrs := field.ErrorList{} @@ -357,7 +361,7 @@ func ValidateVolumes(volumes []api.Volume, fldPath *field.Path) (sets.String, fi for i, vol := range volumes { idxPath := fldPath.Index(i) namePath := idxPath.Child("name") - el := validateVolumeSource(&vol.VolumeSource, idxPath) + el := validateVolumeSource(&vol.VolumeSource, idxPath, vol.Name) if len(vol.Name) == 0 { el = append(el, field.Required(namePath, "")) } else { @@ -376,7 +380,7 @@ func ValidateVolumes(volumes []api.Volume, fldPath *field.Path) (sets.String, fi return allNames, allErrs } -func validateVolumeSource(source *api.VolumeSource, fldPath *field.Path) field.ErrorList { +func validateVolumeSource(source *api.VolumeSource, fldPath *field.Path, volName string) field.ErrorList { numVolumes := 0 allErrs := field.ErrorList{} if source.EmptyDir != nil { @@ -443,13 +447,17 @@ func validateVolumeSource(source *api.VolumeSource, fldPath *field.Path) field.E numVolumes++ allErrs = append(allErrs, validateISCSIVolumeSource(source.ISCSI, fldPath.Child("iscsi"))...) } + if source.ISCSI.InitiatorName != nil && len(volName+":"+source.ISCSI.TargetPortal) > 64 { + tooLongErr := "Total length of : must be under 64 characters if iscsi.initiatorName is specified." + allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), volName, tooLongErr)) + } } if source.Glusterfs != nil { if numVolumes > 0 { allErrs = append(allErrs, field.Forbidden(fldPath.Child("glusterfs"), "may not specify more than 1 volume type")) } else { numVolumes++ - allErrs = append(allErrs, validateGlusterfs(source.Glusterfs, fldPath.Child("glusterfs"))...) + allErrs = append(allErrs, validateGlusterfsVolumeSource(source.Glusterfs, fldPath.Child("glusterfs"))...) } } if source.Flocker != nil { @@ -614,6 +622,7 @@ func validateHostPathVolumeSource(hostPath *api.HostPathVolumeSource, fldPath *f } allErrs = append(allErrs, validatePathNoBacksteps(hostPath.Path, fldPath.Child("path"))...) + allErrs = append(allErrs, validateHostPathType(hostPath.Type, fldPath.Child("type"))...) return allErrs } @@ -635,6 +644,16 @@ func validateISCSIVolumeSource(iscsi *api.ISCSIVolumeSource, fldPath *field.Path } if len(iscsi.IQN) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("iqn"), "")) + } else { + if !strings.HasPrefix(iscsi.IQN, "iqn") && !strings.HasPrefix(iscsi.IQN, "eui") && !strings.HasPrefix(iscsi.IQN, "naa") { + allErrs = append(allErrs, field.Invalid(fldPath.Child("iqn"), iscsi.IQN, "must be valid format")) + } else if strings.HasPrefix(iscsi.IQN, "iqn") && !iscsiInitiatorIqnRegex.MatchString(iscsi.IQN) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("iqn"), iscsi.IQN, "must be valid format")) + } else if strings.HasPrefix(iscsi.IQN, "eui") && !iscsiInitiatorEuiRegex.MatchString(iscsi.IQN) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("iqn"), iscsi.IQN, "must be valid format")) + } else if strings.HasPrefix(iscsi.IQN, "naa") && !iscsiInitiatorNaaRegex.MatchString(iscsi.IQN) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("iqn"), iscsi.IQN, "must be valid format")) + } } if iscsi.Lun < 0 || iscsi.Lun > 255 { allErrs = append(allErrs, field.Invalid(fldPath.Child("lun"), iscsi.Lun, validation.InclusiveRangeError(0, 255))) @@ -642,6 +661,19 @@ func validateISCSIVolumeSource(iscsi *api.ISCSIVolumeSource, fldPath *field.Path if (iscsi.DiscoveryCHAPAuth || iscsi.SessionCHAPAuth) && iscsi.SecretRef == nil { allErrs = append(allErrs, field.Required(fldPath.Child("secretRef"), "")) } + if iscsi.InitiatorName != nil { + initiator := *iscsi.InitiatorName + if !strings.HasPrefix(initiator, "iqn") && !strings.HasPrefix(initiator, "eui") && !strings.HasPrefix(initiator, "naa") { + allErrs = append(allErrs, field.Invalid(fldPath.Child("initiatorname"), initiator, "must be valid format")) + } + if strings.HasPrefix(initiator, "iqn") && !iscsiInitiatorIqnRegex.MatchString(initiator) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("initiatorname"), initiator, "must be valid format")) + } else if strings.HasPrefix(initiator, "eui") && !iscsiInitiatorEuiRegex.MatchString(initiator) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("initiatorname"), initiator, "must be valid format")) + } else if strings.HasPrefix(initiator, "naa") && !iscsiInitiatorNaaRegex.MatchString(initiator) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("initiatorname"), initiator, "must be valid format")) + } + } return allErrs } @@ -783,7 +815,7 @@ func validateQuobyteVolumeSource(quobyte *api.QuobyteVolumeSource, fldPath *fiel return allErrs } -func validateGlusterfs(glusterfs *api.GlusterfsVolumeSource, fldPath *field.Path) field.ErrorList { +func validateGlusterfsVolumeSource(glusterfs *api.GlusterfsVolumeSource, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if len(glusterfs.EndpointsName) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("endpoints"), "")) @@ -942,6 +974,26 @@ func validateProjectedVolumeSource(projection *api.ProjectedVolumeSource, fldPat return allErrs } +var supportedHostPathTypes = sets.NewString( + string(api.HostPathUnset), + string(api.HostPathDirectoryOrCreate), + string(api.HostPathDirectory), + string(api.HostPathFileOrCreate), + string(api.HostPathFile), + string(api.HostPathSocket), + string(api.HostPathCharDev), + string(api.HostPathBlockDev)) + +func validateHostPathType(hostPathType *api.HostPathType, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if !supportedHostPathTypes.Has(string(*hostPathType)) { + allErrs = append(allErrs, field.NotSupported(fldPath, hostPathType, supportedHostPathTypes.List())) + } + + return allErrs +} + // This validate will make sure targetPath: // 1. is not abs path // 2. does not have any element which is ".." @@ -1013,6 +1065,14 @@ func validateCephFSVolumeSource(cephfs *api.CephFSVolumeSource, fldPath *field.P return allErrs } +func validateCephFSPersistentVolumeSource(cephfs *api.CephFSPersistentVolumeSource, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if len(cephfs.Monitors) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("monitors"), "")) + } + return allErrs +} + func validateFlexVolumeSource(fv *api.FlexVolumeSource, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if len(fv.Driver) == 0 { @@ -1045,6 +1105,22 @@ func validateAzureFile(azure *api.AzureFileVolumeSource, fldPath *field.Path) fi return allErrs } +func validateAzureFilePV(azure *api.AzureFilePersistentVolumeSource, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if azure.SecretName == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("secretName"), "")) + } + if azure.ShareName == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("shareName"), "")) + } + if azure.SecretNamespace != nil { + if len(*azure.SecretNamespace) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("secretNamespace"), "")) + } + } + return allErrs +} + func validateAzureDisk(azure *api.AzureDiskVolumeSource, fldPath *field.Path) field.ErrorList { var supportedCachingModes = sets.NewString(string(api.AzureDataDiskCachingNone), string(api.AzureDataDiskCachingReadOnly), string(api.AzureDataDiskCachingReadWrite)) var supportedDiskKinds = sets.NewString(string(api.AzureSharedBlobDisk), string(api.AzureDedicatedBlobDisk), string(api.AzureManagedDisk)) @@ -1241,7 +1317,7 @@ func ValidatePersistentVolume(pv *api.PersistentVolume) field.ErrorList { allErrs = append(allErrs, field.Forbidden(specPath.Child("glusterfs"), "may not specify more than 1 volume type")) } else { numVolumes++ - allErrs = append(allErrs, validateGlusterfs(pv.Spec.Glusterfs, specPath.Child("glusterfs"))...) + allErrs = append(allErrs, validateGlusterfsVolumeSource(pv.Spec.Glusterfs, specPath.Child("glusterfs"))...) } } if pv.Spec.Flocker != nil { @@ -1281,7 +1357,7 @@ func ValidatePersistentVolume(pv *api.PersistentVolume) field.ErrorList { allErrs = append(allErrs, field.Forbidden(specPath.Child("cephFS"), "may not specify more than 1 volume type")) } else { numVolumes++ - allErrs = append(allErrs, validateCephFSVolumeSource(pv.Spec.CephFS, specPath.Child("cephfs"))...) + allErrs = append(allErrs, validateCephFSPersistentVolumeSource(pv.Spec.CephFS, specPath.Child("cephfs"))...) } } if pv.Spec.ISCSI != nil { @@ -1291,6 +1367,10 @@ func ValidatePersistentVolume(pv *api.PersistentVolume) field.ErrorList { numVolumes++ allErrs = append(allErrs, validateISCSIVolumeSource(pv.Spec.ISCSI, specPath.Child("iscsi"))...) } + if pv.Spec.ISCSI.InitiatorName != nil && len(pv.ObjectMeta.Name+":"+pv.Spec.ISCSI.TargetPortal) > 64 { + tooLongErr := "Total length of : must be under 64 characters if iscsi.initiatorName is specified." + allErrs = append(allErrs, field.Invalid(metaPath.Child("name"), pv.ObjectMeta.Name, tooLongErr)) + } } if pv.Spec.Cinder != nil { if numVolumes > 0 { @@ -1318,7 +1398,7 @@ func ValidatePersistentVolume(pv *api.PersistentVolume) field.ErrorList { } else { numVolumes++ - allErrs = append(allErrs, validateAzureFile(pv.Spec.AzureFile, specPath.Child("azureFile"))...) + allErrs = append(allErrs, validateAzureFilePV(pv.Spec.AzureFile, specPath.Child("azureFile"))...) } } @@ -1563,7 +1643,7 @@ func ValidateEnv(vars []api.EnvVar, fldPath *field.Path) field.ErrorList { } var validFieldPathExpressionsEnv = sets.NewString("metadata.name", "metadata.namespace", "metadata.uid", "spec.nodeName", "spec.serviceAccountName", "status.hostIP", "status.podIP") -var validContainerResourceFieldPathExpressions = sets.NewString("limits.cpu", "limits.memory", "requests.cpu", "requests.memory") +var validContainerResourceFieldPathExpressions = sets.NewString("limits.cpu", "limits.memory", "limits.ephemeral-storage", "requests.cpu", "requests.memory", "requests.ephemeral-storage") func validateEnvVarValueFrom(ev api.EnvVar, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} @@ -1623,6 +1703,13 @@ func validateObjectFieldSelector(fs *api.ObjectFieldSelector, expressions *sets. return allErrs } +func fsResourceIsEphemeralStorage(resource string) bool { + if resource == "limits.ephemeral-storage" || resource == "requests.ephemeral-storage" { + return true + } + return false +} + func validateContainerResourceFieldSelector(fs *api.ResourceFieldSelector, expressions *sets.String, fldPath *field.Path, volume bool) field.ErrorList { allErrs := field.ErrorList{} @@ -1632,6 +1719,8 @@ func validateContainerResourceFieldSelector(fs *api.ResourceFieldSelector, expre allErrs = append(allErrs, field.Required(fldPath.Child("resource"), "")) } else if !expressions.Has(fs.Resource) { allErrs = append(allErrs, field.NotSupported(fldPath.Child("resource"), fs.Resource, expressions.List())) + } else if fsResourceIsEphemeralStorage(fs.Resource) && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) { + allErrs = append(allErrs, field.Forbidden(fldPath, "Containers' ephemeral storage requests/limits disabled by feature-gate for Downward API")) } allErrs = append(allErrs, validateContainerResourceDivisor(fs.Resource, fs.Divisor, fldPath)...) return allErrs @@ -1692,6 +1781,7 @@ func validateSecretEnvSource(secretSource *api.SecretEnvSource, fldPath *field.P var validContainerResourceDivisorForCPU = sets.NewString("1m", "1") var validContainerResourceDivisorForMemory = sets.NewString("1", "1k", "1M", "1G", "1T", "1P", "1E", "1Ki", "1Mi", "1Gi", "1Ti", "1Pi", "1Ei") +var validContainerResourceDivisorForEphemeralStorage = sets.NewString("1", "1k", "1M", "1G", "1T", "1P", "1E", "1Ki", "1Mi", "1Gi", "1Ti", "1Pi", "1Ei") func validateContainerResourceDivisor(rName string, divisor resource.Quantity, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} @@ -1708,6 +1798,10 @@ func validateContainerResourceDivisor(rName string, divisor resource.Quantity, f if !validContainerResourceDivisorForMemory.Has(divisor.String()) { allErrs = append(allErrs, field.Invalid(fldPath.Child("divisor"), rName, "only divisor's values 1, 1k, 1M, 1G, 1T, 1P, 1E, 1Ki, 1Mi, 1Gi, 1Ti, 1Pi, 1Ei are supported with the memory resource")) } + case "limits.ephemeral-storage", "requests.ephemeral-storage": + if !validContainerResourceDivisorForEphemeralStorage.Has(divisor.String()) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("divisor"), rName, "only divisor's values 1, 1k, 1M, 1G, 1T, 1P, 1E, 1Ki, 1Mi, 1Gi, 1Ti, 1Pi, 1Ei are supported with the local ephemeral storage resource")) + } } return allErrs } @@ -1792,6 +1886,33 @@ func validateProbe(probe *api.Probe, fldPath *field.Path) field.ErrorList { return allErrs } +func validateClientIPAffinityConfig(config *api.SessionAffinityConfig, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if config == nil { + allErrs = append(allErrs, field.Required(fldPath, fmt.Sprintf("when session affinity type is %s", api.ServiceAffinityClientIP))) + return allErrs + } + if config.ClientIP == nil { + allErrs = append(allErrs, field.Required(fldPath.Child("clientIP"), fmt.Sprintf("when session affinity type is %s", api.ServiceAffinityClientIP))) + return allErrs + } + if config.ClientIP.TimeoutSeconds == nil { + allErrs = append(allErrs, field.Required(fldPath.Child("clientIP").Child("timeoutSeconds"), fmt.Sprintf("when session affinity type is %s", api.ServiceAffinityClientIP))) + return allErrs + } + allErrs = append(allErrs, validateAffinityTimeout(config.ClientIP.TimeoutSeconds, fldPath.Child("clientIP").Child("timeoutSeconds"))...) + + return allErrs +} + +func validateAffinityTimeout(timeout *int32, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if *timeout <= 0 || *timeout > api.MaxClientIPServiceAffinitySeconds { + allErrs = append(allErrs, field.Invalid(fldPath, timeout, fmt.Sprintf("must be greater than 0 and less than %d", api.MaxClientIPServiceAffinitySeconds))) + } + return allErrs +} + // AccumulateUniqueHostPorts extracts each HostPort of each Container, // accumulating the results and returning an error if any ports conflict. func AccumulateUniqueHostPorts(containers []api.Container, accumulator *sets.String, fldPath *field.Path) field.ErrorList { @@ -2067,16 +2188,6 @@ func validateHostNetwork(hostNetwork bool, containers []api.Container, fldPath * return allErrors } -func validateHostNetworkNoHostAliases(hostNetwork bool, hostAliases []api.HostAlias, fldPath *field.Path) field.ErrorList { - allErrors := field.ErrorList{} - if hostNetwork { - if len(hostAliases) > 0 { - allErrors = append(allErrors, field.Forbidden(fldPath, "may not be set when `hostNetwork` is true")) - } - } - return allErrors -} - // validateImagePullSecrets checks to make sure the pull secrets are well // formed. Right now, we only expect name to be set (it's the only field). If // this ever changes and someone decides to set those fields, we'd like to @@ -2434,7 +2545,7 @@ func ValidatePreferredSchedulingTerms(terms []api.PreferredSchedulingTerm, fldPa } // validatePodAffinityTerm tests that the specified podAffinityTerm fields have valid data -func validatePodAffinityTerm(podAffinityTerm api.PodAffinityTerm, allowEmptyTopologyKey bool, fldPath *field.Path) field.ErrorList { +func validatePodAffinityTerm(podAffinityTerm api.PodAffinityTerm, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(podAffinityTerm.LabelSelector, fldPath.Child("matchExpressions"))...) @@ -2443,32 +2554,29 @@ func validatePodAffinityTerm(podAffinityTerm api.PodAffinityTerm, allowEmptyTopo allErrs = append(allErrs, field.Invalid(fldPath.Child("namespace"), name, msg)) } } - if !allowEmptyTopologyKey && len(podAffinityTerm.TopologyKey) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("topologyKey"), "can only be empty for PreferredDuringScheduling pod anti affinity")) - } - if len(podAffinityTerm.TopologyKey) != 0 { - allErrs = append(allErrs, unversionedvalidation.ValidateLabelName(podAffinityTerm.TopologyKey, fldPath.Child("topologyKey"))...) + if len(podAffinityTerm.TopologyKey) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("topologyKey"), "can not be empty")) } - return allErrs + return append(allErrs, unversionedvalidation.ValidateLabelName(podAffinityTerm.TopologyKey, fldPath.Child("topologyKey"))...) } // validatePodAffinityTerms tests that the specified podAffinityTerms fields have valid data -func validatePodAffinityTerms(podAffinityTerms []api.PodAffinityTerm, allowEmptyTopologyKey bool, fldPath *field.Path) field.ErrorList { +func validatePodAffinityTerms(podAffinityTerms []api.PodAffinityTerm, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} for i, podAffinityTerm := range podAffinityTerms { - allErrs = append(allErrs, validatePodAffinityTerm(podAffinityTerm, allowEmptyTopologyKey, fldPath.Index(i))...) + allErrs = append(allErrs, validatePodAffinityTerm(podAffinityTerm, fldPath.Index(i))...) } return allErrs } // validateWeightedPodAffinityTerms tests that the specified weightedPodAffinityTerms fields have valid data -func validateWeightedPodAffinityTerms(weightedPodAffinityTerms []api.WeightedPodAffinityTerm, allowEmptyTopologyKey bool, fldPath *field.Path) field.ErrorList { +func validateWeightedPodAffinityTerms(weightedPodAffinityTerms []api.WeightedPodAffinityTerm, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} for j, weightedTerm := range weightedPodAffinityTerms { if weightedTerm.Weight <= 0 || weightedTerm.Weight > 100 { allErrs = append(allErrs, field.Invalid(fldPath.Index(j).Child("weight"), weightedTerm.Weight, "must be in the range 1-100")) } - allErrs = append(allErrs, validatePodAffinityTerm(weightedTerm.PodAffinityTerm, allowEmptyTopologyKey, fldPath.Index(j).Child("podAffinityTerm"))...) + allErrs = append(allErrs, validatePodAffinityTerm(weightedTerm.PodAffinityTerm, fldPath.Index(j).Child("podAffinityTerm"))...) } return allErrs } @@ -2482,13 +2590,11 @@ func validatePodAntiAffinity(podAntiAffinity *api.PodAntiAffinity, fldPath *fiel // fldPath.Child("requiredDuringSchedulingRequiredDuringExecution"))...) //} if podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { - // empty topologyKey is not allowed for hard pod anti-affinity - allErrs = append(allErrs, validatePodAffinityTerms(podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, false, + allErrs = append(allErrs, validatePodAffinityTerms(podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, fldPath.Child("requiredDuringSchedulingIgnoredDuringExecution"))...) } if podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution != nil { - // empty topologyKey is allowed for soft pod anti-affinity - allErrs = append(allErrs, validateWeightedPodAffinityTerms(podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, true, + allErrs = append(allErrs, validateWeightedPodAffinityTerms(podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, fldPath.Child("preferredDuringSchedulingIgnoredDuringExecution"))...) } return allErrs @@ -2503,13 +2609,11 @@ func validatePodAffinity(podAffinity *api.PodAffinity, fldPath *field.Path) fiel // fldPath.Child("requiredDuringSchedulingRequiredDuringExecution"))...) //} if podAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { - // empty topologyKey is not allowed for hard pod affinity - allErrs = append(allErrs, validatePodAffinityTerms(podAffinity.RequiredDuringSchedulingIgnoredDuringExecution, false, + allErrs = append(allErrs, validatePodAffinityTerms(podAffinity.RequiredDuringSchedulingIgnoredDuringExecution, fldPath.Child("requiredDuringSchedulingIgnoredDuringExecution"))...) } if podAffinity.PreferredDuringSchedulingIgnoredDuringExecution != nil { - // empty topologyKey is not allowed for soft pod affinity - allErrs = append(allErrs, validateWeightedPodAffinityTerms(podAffinity.PreferredDuringSchedulingIgnoredDuringExecution, false, + allErrs = append(allErrs, validateWeightedPodAffinityTerms(podAffinity.PreferredDuringSchedulingIgnoredDuringExecution, fldPath.Child("preferredDuringSchedulingIgnoredDuringExecution"))...) } return allErrs @@ -2620,7 +2724,6 @@ func ValidatePodSecurityContext(securityContext *api.PodSecurityContext, spec *a if securityContext != nil { allErrs = append(allErrs, validateHostNetwork(securityContext.HostNetwork, spec.Containers, specPath.Child("containers"))...) - allErrs = append(allErrs, validateHostNetworkNoHostAliases(securityContext.HostNetwork, spec.HostAliases, specPath)...) if securityContext.FSGroup != nil { for _, msg := range validation.IsValidGroupID(*securityContext.FSGroup) { allErrs = append(allErrs, field.Invalid(fldPath.Child("fsGroup"), *(securityContext.FSGroup), msg)) @@ -2734,8 +2837,10 @@ func ValidatePodUpdate(newPod, oldPod *api.Pod) field.ErrorList { allErrs = append(allErrs, validateOnlyAddedTolerations(newPod.Spec.Tolerations, oldPod.Spec.Tolerations, specPath.Child("tolerations"))...) if !apiequality.Semantic.DeepEqual(mungedPod.Spec, oldPod.Spec) { + // This diff isn't perfect, but it's a helluva lot better an "I'm not going to tell you what the difference is". //TODO: Pinpoint the specific field that causes the invalid error after we have strategic merge diff - allErrs = append(allErrs, field.Forbidden(specPath, "pod updates may not change fields other than `spec.containers[*].image`, `spec.initContainers[*].image`, `spec.activeDeadlineSeconds` or `spec.tolerations` (only additions to existing tolerations)")) + specDiff := diff.ObjectDiff(mungedPod.Spec, oldPod.Spec) + allErrs = append(allErrs, field.Forbidden(specPath, fmt.Sprintf("pod updates may not change fields other than `spec.containers[*].image`, `spec.initContainers[*].image`, `spec.activeDeadlineSeconds` or `spec.tolerations` (only additions to existing tolerations)\n%v", specDiff))) } return allErrs @@ -2849,6 +2954,14 @@ func ValidateService(service *api.Service) field.ErrorList { allErrs = append(allErrs, field.NotSupported(specPath.Child("sessionAffinity"), service.Spec.SessionAffinity, supportedSessionAffinityType.List())) } + if service.Spec.SessionAffinity == api.ServiceAffinityClientIP { + allErrs = append(allErrs, validateClientIPAffinityConfig(service.Spec.SessionAffinityConfig, specPath.Child("sessionAffinityConfig"))...) + } else if service.Spec.SessionAffinity == api.ServiceAffinityNone { + if service.Spec.SessionAffinityConfig != nil { + allErrs = append(allErrs, field.Forbidden(specPath.Child("sessionAffinityConfig"), fmt.Sprintf("must not be set when session affinity is %s", string(api.ServiceAffinityNone)))) + } + } + if helper.IsServiceIPSet(service) { if ip := net.ParseIP(service.Spec.ClusterIP); ip == nil { allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIP"), service.Spec.ClusterIP, "must be empty, 'None', or a valid IP address")) @@ -3378,7 +3491,7 @@ func validateResourceName(value string, fldPath *field.Path) field.ErrorList { } } - return field.ErrorList{} + return allErrs } // Validate container resource name @@ -3391,19 +3504,32 @@ func validateContainerResourceName(value string, fldPath *field.Path) field.Erro return append(allErrs, field.Invalid(fldPath, value, "must be a standard resource for containers")) } } - return field.ErrorList{} + return allErrs +} + +// isLocalStorageResource checks whether the resource is local ephemeral storage +func isLocalStorageResource(name string) bool { + if name == string(api.ResourceEphemeralStorage) || name == string(api.ResourceRequestsEphemeralStorage) || + name == string(api.ResourceLimitsEphemeralStorage) { + return true + } else { + return false + } } // Validate resource names that can go in a resource quota // Refer to docs/design/resources.md for more details. func ValidateResourceQuotaResourceName(value string, fldPath *field.Path) field.ErrorList { allErrs := validateResourceName(value, fldPath) + if isLocalStorageResource(value) && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) { + return append(allErrs, field.Forbidden(fldPath, "ResourceEphemeralStorage field disabled by feature-gate for ResourceQuota")) + } if len(strings.Split(value, "/")) == 1 { if !helper.IsStandardQuotaResourceName(value) { return append(allErrs, field.Invalid(fldPath, value, isInvalidQuotaResource)) } } - return field.ErrorList{} + return allErrs } // Validate limit range types @@ -3718,18 +3844,8 @@ func ValidateResourceRequirements(requirements *api.ResourceRequirements, fldPat // Validate resource quantity. allErrs = append(allErrs, ValidateResourceQuantityValue(string(resourceName), quantity, fldPath)...) - // Check that request <= limit. - requestQuantity, exists := requirements.Requests[resourceName] - if exists { - // Ensure overcommit is allowed for the resource if request != limit - if quantity.Cmp(requestQuantity) != 0 && !helper.IsOvercommitAllowed(resourceName) { - allErrs = append(allErrs, field.Invalid(reqPath, requestQuantity.String(), fmt.Sprintf("must be equal to %s limit", resourceName))) - } else if quantity.Cmp(requestQuantity) < 0 { - allErrs = append(allErrs, field.Invalid(limPath, quantity.String(), fmt.Sprintf("must be greater than or equal to %s request", resourceName))) - } - } - if resourceName == api.ResourceStorageOverlay && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) { - allErrs = append(allErrs, field.Forbidden(limPath, "ResourceStorageOverlay field disabled by feature-gate for ResourceRequirements")) + if resourceName == api.ResourceEphemeralStorage && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) { + allErrs = append(allErrs, field.Forbidden(limPath, "ResourceEphemeralStorage field disabled by feature-gate for ResourceRequirements")) } } for resourceName, quantity := range requirements.Requests { @@ -3738,6 +3854,19 @@ func ValidateResourceRequirements(requirements *api.ResourceRequirements, fldPat allErrs = append(allErrs, validateContainerResourceName(string(resourceName), fldPath)...) // Validate resource quantity. allErrs = append(allErrs, ValidateResourceQuantityValue(string(resourceName), quantity, fldPath)...) + + // Check that request <= limit. + limitQuantity, exists := requirements.Limits[resourceName] + if exists { + // For GPUs, not only requests can't exceed limits, they also can't be lower, i.e. must be equal. + if quantity.Cmp(limitQuantity) != 0 && !helper.IsOvercommitAllowed(resourceName) { + allErrs = append(allErrs, field.Invalid(reqPath, quantity.String(), fmt.Sprintf("must be equal to %s limit", api.ResourceNvidiaGPU))) + } else if quantity.Cmp(limitQuantity) > 0 { + allErrs = append(allErrs, field.Invalid(reqPath, quantity.String(), fmt.Sprintf("must be less than or equal to %s limit", resourceName))) + } + } else if resourceName == api.ResourceNvidiaGPU { + allErrs = append(allErrs, field.Invalid(reqPath, quantity.String(), fmt.Sprintf("must be equal to %s request", api.ResourceNvidiaGPU))) + } } return allErrs diff --git a/pkg/api/validation/validation_test.go b/pkg/api/validation/validation_test.go index 99b6a68874b47..d7038753f3040 100644 --- a/pkg/api/validation/validation_test.go +++ b/pkg/api/validation/validation_test.go @@ -43,6 +43,12 @@ const ( envVarNameErrMsg = "a valid environment variable name must consist of" ) +func newHostPathType(pathType string) *api.HostPathType { + hostPathType := new(api.HostPathType) + *hostPathType = api.HostPathType(pathType) + return hostPathType +} + func testVolume(name string, namespace string, spec api.PersistentVolumeSpec) *api.PersistentVolume { objMeta := metav1.ObjectMeta{Name: name} if namespace != "" { @@ -86,7 +92,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, StorageClassName: "valid", }), @@ -99,7 +108,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimRetain, }), @@ -112,7 +124,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{"fakemode"}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, }), }, @@ -124,7 +139,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, PersistentVolumeReclaimPolicy: "fakeReclaimPolicy", }), @@ -137,7 +155,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, }), }, @@ -149,7 +170,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, }), }, @@ -173,7 +197,10 @@ func TestValidatePersistentVolumes(t *testing.T) { api.ResourceName(api.ResourceStorage): resource.MustParse("10G"), }, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, }), }, @@ -184,7 +211,10 @@ func TestValidatePersistentVolumes(t *testing.T) { api.ResourceName(api.ResourceStorage): resource.MustParse("5G"), }, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(api.HostPathDirectory)), + }, GCEPersistentDisk: &api.GCEPersistentDiskVolumeSource{PDName: "foo", FSType: "ext4"}, }, }), @@ -197,7 +227,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimRecycle, }), @@ -210,7 +243,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/a/.."}, + HostPath: &api.HostPathVolumeSource{ + Path: "/a/..", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimRecycle, }), @@ -223,7 +259,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, StorageClassName: "-invalid-", }), @@ -272,7 +311,10 @@ func TestValidatePersistentVolumes(t *testing.T) { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo/.."}, + HostPath: &api.HostPathVolumeSource{ + Path: "/foo/..", + Type: newHostPathType(string(api.HostPathDirectory)), + }, }, StorageClassName: "backstep-hostpath", }), @@ -998,7 +1040,7 @@ func TestValidateGlusterfs(t *testing.T) { } for i, tc := range testCases { - errs := validateGlusterfs(tc.gfs, field.NewPath("field")) + errs := validateGlusterfsVolumeSource(tc.gfs, field.NewPath("field")) if len(errs) > 0 && tc.errtype == "" { t.Errorf("[%d: %q] unexpected error(s): %v", i, tc.name, errs) @@ -1025,6 +1067,8 @@ func newInt32(val int) *int32 { // type on its own, but we want to also make sure that the logic works through // the one-of wrapper, so we just do it all in one place. func TestValidateVolumes(t *testing.T) { + validInitiatorName := "iqn.2015-02.example.com:init" + invalidInitiatorName := "2015-02.example.com:init" testCases := []struct { name string vol api.Volume @@ -1107,6 +1151,7 @@ func TestValidateVolumes(t *testing.T) { EmptyDir: &api.EmptyDirVolumeSource{}, HostPath: &api.HostPathVolumeSource{ Path: "/mnt/path", + Type: newHostPathType(string(api.HostPathDirectory)), }, }, }, @@ -1114,7 +1159,20 @@ func TestValidateVolumes(t *testing.T) { errfield: "hostPath", errdetail: "may not specify more than 1 volume", }, - // HostPath + // HostPath Default + { + name: "default HostPath", + vol: api.Volume{ + Name: "hostpath", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{ + Path: "/mnt/path", + Type: newHostPathType(string(api.HostPathDirectory)), + }, + }, + }, + }, + // HostPath Supported { name: "valid HostPath", vol: api.Volume{ @@ -1122,10 +1180,26 @@ func TestValidateVolumes(t *testing.T) { VolumeSource: api.VolumeSource{ HostPath: &api.HostPathVolumeSource{ Path: "/mnt/path", + Type: newHostPathType(string(api.HostPathSocket)), }, }, }, }, + // HostPath Invalid + { + name: "invalid HostPath", + vol: api.Volume{ + Name: "hostpath", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{ + Path: "/mnt/path", + Type: newHostPathType("invalid"), + }, + }, + }, + errtype: field.ErrorTypeNotSupported, + errfield: "type", + }, { name: "invalid HostPath backsteps", vol: api.Volume{ @@ -1133,6 +1207,7 @@ func TestValidateVolumes(t *testing.T) { VolumeSource: api.VolumeSource{ HostPath: &api.HostPathVolumeSource{ Path: "/mnt/path/..", + Type: newHostPathType(string(api.HostPathDirectory)), }, }, }, @@ -1268,6 +1343,36 @@ func TestValidateVolumes(t *testing.T) { }, }, }, + { + name: "valid IQN: eui format", + vol: api.Volume{ + Name: "iscsi", + VolumeSource: api.VolumeSource{ + ISCSI: &api.ISCSIVolumeSource{ + TargetPortal: "127.0.0.1", + IQN: "eui.0123456789ABCDEF", + Lun: 1, + FSType: "ext4", + ReadOnly: false, + }, + }, + }, + }, + { + name: "valid IQN: naa format", + vol: api.Volume{ + Name: "iscsi", + VolumeSource: api.VolumeSource{ + ISCSI: &api.ISCSIVolumeSource{ + TargetPortal: "127.0.0.1", + IQN: "naa.62004567BA64678D0123456789ABCDEF", + Lun: 1, + FSType: "ext4", + ReadOnly: false, + }, + }, + }, + }, { name: "empty portal", vol: api.Volume{ @@ -1302,6 +1407,91 @@ func TestValidateVolumes(t *testing.T) { errtype: field.ErrorTypeRequired, errfield: "iscsi.iqn", }, + { + name: "invalid IQN: iqn format", + vol: api.Volume{ + Name: "iscsi", + VolumeSource: api.VolumeSource{ + ISCSI: &api.ISCSIVolumeSource{ + TargetPortal: "127.0.0.1", + IQN: "iqn.2015-02.example.com:test;ls;", + Lun: 1, + FSType: "ext4", + ReadOnly: false, + }, + }, + }, + errtype: field.ErrorTypeInvalid, + errfield: "iscsi.iqn", + }, + { + name: "invalid IQN: eui format", + vol: api.Volume{ + Name: "iscsi", + VolumeSource: api.VolumeSource{ + ISCSI: &api.ISCSIVolumeSource{ + TargetPortal: "127.0.0.1", + IQN: "eui.0123456789ABCDEFGHIJ", + Lun: 1, + FSType: "ext4", + ReadOnly: false, + }, + }, + }, + errtype: field.ErrorTypeInvalid, + errfield: "iscsi.iqn", + }, + { + name: "invalid IQN: naa format", + vol: api.Volume{ + Name: "iscsi", + VolumeSource: api.VolumeSource{ + ISCSI: &api.ISCSIVolumeSource{ + TargetPortal: "127.0.0.1", + IQN: "naa.62004567BA_4-78D.123456789ABCDEF", + Lun: 1, + FSType: "ext4", + ReadOnly: false, + }, + }, + }, + errtype: field.ErrorTypeInvalid, + errfield: "iscsi.iqn", + }, + { + name: "valid initiatorName", + vol: api.Volume{ + Name: "iscsi", + VolumeSource: api.VolumeSource{ + ISCSI: &api.ISCSIVolumeSource{ + TargetPortal: "127.0.0.1", + IQN: "iqn.2015-02.example.com:test", + Lun: 1, + InitiatorName: &validInitiatorName, + FSType: "ext4", + ReadOnly: false, + }, + }, + }, + }, + { + name: "invalid initiatorName", + vol: api.Volume{ + Name: "iscsi", + VolumeSource: api.VolumeSource{ + ISCSI: &api.ISCSIVolumeSource{ + TargetPortal: "127.0.0.1", + IQN: "iqn.2015-02.example.com:test", + Lun: 1, + InitiatorName: &invalidInitiatorName, + FSType: "ext4", + ReadOnly: false, + }, + }, + }, + errtype: field.ErrorTypeInvalid, + errfield: "iscsi.initiatorname", + }, { name: "empty secret", vol: api.Volume{ @@ -2475,7 +2665,7 @@ func TestAlphaLocalStorageCapacityIsolation(t *testing.T) { return } for _, tc := range testCases { - if errs := validateVolumeSource(&tc, field.NewPath("spec")); len(errs) != 0 { + if errs := validateVolumeSource(&tc, field.NewPath("spec"), "tmpvol"); len(errs) != 0 { t.Errorf("expected success: %v", errs) } } @@ -2486,14 +2676,14 @@ func TestAlphaLocalStorageCapacityIsolation(t *testing.T) { return } for _, tc := range testCases { - if errs := validateVolumeSource(&tc, field.NewPath("spec")); len(errs) == 0 { + if errs := validateVolumeSource(&tc, field.NewPath("spec"), "tmpvol"); len(errs) == 0 { t.Errorf("expected failure: %v", errs) } } containerLimitCase := api.ResourceRequirements{ Limits: api.ResourceList{ - api.ResourceStorageOverlay: *resource.NewMilliQuantity( + api.ResourceEphemeralStorage: *resource.NewMilliQuantity( int64(40000), resource.BinarySI), }, @@ -2519,6 +2709,62 @@ func TestAlphaLocalStorageCapacityIsolation(t *testing.T) { } +func TestValidateResourceQuotaWithAlphaLocalStorageCapacityIsolation(t *testing.T) { + spec := api.ResourceQuotaSpec{ + Hard: api.ResourceList{ + api.ResourceCPU: resource.MustParse("100"), + api.ResourceMemory: resource.MustParse("10000"), + api.ResourceRequestsCPU: resource.MustParse("100"), + api.ResourceRequestsMemory: resource.MustParse("10000"), + api.ResourceLimitsCPU: resource.MustParse("100"), + api.ResourceLimitsMemory: resource.MustParse("10000"), + api.ResourcePods: resource.MustParse("10"), + api.ResourceServices: resource.MustParse("0"), + api.ResourceReplicationControllers: resource.MustParse("10"), + api.ResourceQuotas: resource.MustParse("10"), + api.ResourceConfigMaps: resource.MustParse("10"), + api.ResourceSecrets: resource.MustParse("10"), + api.ResourceEphemeralStorage: resource.MustParse("10000"), + api.ResourceRequestsEphemeralStorage: resource.MustParse("10000"), + api.ResourceLimitsEphemeralStorage: resource.MustParse("10000"), + }, + } + resourceQuota := &api.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "abc", + Namespace: "foo", + }, + Spec: spec, + } + + // Enable alpha feature LocalStorageCapacityIsolation + err := utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=true") + if err != nil { + t.Errorf("Failed to enable feature gate for LocalStorageCapacityIsolation: %v", err) + return + } + if errs := ValidateResourceQuota(resourceQuota); len(errs) != 0 { + t.Errorf("expected success: %v", errs) + } + + // Disable alpha feature LocalStorageCapacityIsolation + err = utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=false") + if err != nil { + t.Errorf("Failed to disable feature gate for LocalStorageCapacityIsolation: %v", err) + return + } + errs := ValidateResourceQuota(resourceQuota) + if len(errs) == 0 { + t.Errorf("expected failure for %s", resourceQuota.Name) + } + expectedErrMes := "ResourceEphemeralStorage field disabled by feature-gate for ResourceQuota" + for i := range errs { + if !strings.Contains(errs[i].Detail, expectedErrMes) { + t.Errorf("[%s]: expected error detail either empty or %s, got %s", resourceQuota.Name, expectedErrMes, errs[i].Detail) + } + } +} + func TestValidatePorts(t *testing.T) { successCase := []api.ContainerPort{ {Name: "abc", ContainerPort: 80, HostPort: 80, Protocol: "TCP"}, @@ -2617,6 +2863,52 @@ func TestValidatePorts(t *testing.T) { } } +func TestLocalStorageEnvWithFeatureGate(t *testing.T) { + testCases := []api.EnvVar{ + { + Name: "ephemeral-storage-limits", + ValueFrom: &api.EnvVarSource{ + ResourceFieldRef: &api.ResourceFieldSelector{ + ContainerName: "test-container", + Resource: "limits.ephemeral-storage", + }, + }, + }, + { + Name: "ephemeral-storage-requests", + ValueFrom: &api.EnvVarSource{ + ResourceFieldRef: &api.ResourceFieldSelector{ + ContainerName: "test-container", + Resource: "requests.ephemeral-storage", + }, + }, + }, + } + // Enable alpha feature LocalStorageCapacityIsolation + err := utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=true") + if err != nil { + t.Errorf("Failed to enable feature gate for LocalStorageCapacityIsolation: %v", err) + return + } + for _, testCase := range testCases { + if errs := validateEnvVarValueFrom(testCase, field.NewPath("field")); len(errs) != 0 { + t.Errorf("expected success, got: %v", errs) + } + } + + // Disable alpha feature LocalStorageCapacityIsolation + err = utilfeature.DefaultFeatureGate.Set("LocalStorageCapacityIsolation=false") + if err != nil { + t.Errorf("Failed to disable feature gate for LocalStorageCapacityIsolation: %v", err) + return + } + for _, testCase := range testCases { + if errs := validateEnvVarValueFrom(testCase, field.NewPath("field")); len(errs) == 0 { + t.Errorf("expected failure for %v", testCase.Name) + } + } +} + func TestValidateEnv(t *testing.T) { successCase := []api.EnvVar{ {Name: "abc", Value: "value"}, @@ -3631,6 +3923,21 @@ func TestValidateContainers(t *testing.T) { ImagePullPolicy: "IfNotPresent", }, }, + "Resource GPU invalid setting only request": { + { + Name: "gpu-resource-request-limit", + Image: "image", + Resources: api.ResourceRequirements{ + Requests: api.ResourceList{ + api.ResourceName(api.ResourceCPU): resource.MustParse("10"), + api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), + api.ResourceName(api.ResourceNvidiaGPU): resource.MustParse("1"), + }, + }, + TerminationMessagePolicy: "File", + ImagePullPolicy: "IfNotPresent", + }, + }, "Request limit simple invalid": { { Name: "abc-123", @@ -3845,13 +4152,22 @@ func TestValidatePodSpec(t *testing.T) { RestartPolicy: api.RestartPolicyAlways, DNSPolicy: api.DNSClusterFirst, }, - { // Populate HostAliases with `foo.bar` hostnames . + { // Populate HostAliases with `foo.bar` hostnames. HostAliases: []api.HostAlias{{IP: "12.34.56.78", Hostnames: []string{"host1.foo", "host2.bar"}}}, Volumes: []api.Volume{{Name: "vol", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}}, Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}, RestartPolicy: api.RestartPolicyAlways, DNSPolicy: api.DNSClusterFirst, }, + { // Populate HostAliases with HostNetwork. + HostAliases: []api.HostAlias{{IP: "12.34.56.78", Hostnames: []string{"host1.foo", "host2.bar"}}}, + Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}, + SecurityContext: &api.PodSecurityContext{ + HostNetwork: true, + }, + RestartPolicy: api.RestartPolicyAlways, + DNSPolicy: api.DNSClusterFirst, + }, { // Populate PriorityClassName. Volumes: []api.Volume{{Name: "vol", VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}}, Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File"}}, @@ -3924,12 +4240,6 @@ func TestValidatePodSpec(t *testing.T) { RestartPolicy: api.RestartPolicyAlways, DNSPolicy: api.DNSClusterFirst, }, - "with hostNetwork and hostAliases": { - SecurityContext: &api.PodSecurityContext{ - HostNetwork: true, - }, - HostAliases: []api.HostAlias{{IP: "12.34.56.78", Hostnames: []string{"host1", "host2"}}}, - }, "with hostAliases with invalid IP": { SecurityContext: &api.PodSecurityContext{ HostNetwork: false, @@ -4732,8 +5042,8 @@ func TestValidatePod(t *testing.T) { }), }, }, - "invalid pod affinity, empty topologyKey is not allowed for hard pod affinity": { - expectedError: "can only be empty for PreferredDuringScheduling pod anti affinity", + "invalid hard pod affinity, empty topologyKey is not allowed for hard pod affinity": { + expectedError: "can not be empty", spec: api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "123", @@ -4759,8 +5069,8 @@ func TestValidatePod(t *testing.T) { }), }, }, - "invalid pod anti-affinity, empty topologyKey is not allowed for hard pod anti-affinity": { - expectedError: "can only be empty for PreferredDuringScheduling pod anti affinity", + "invalid hard pod anti-affinity, empty topologyKey is not allowed for hard pod anti-affinity": { + expectedError: "can not be empty", spec: api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "123", @@ -4786,8 +5096,8 @@ func TestValidatePod(t *testing.T) { }), }, }, - "invalid pod anti-affinity, empty topologyKey is not allowed for soft pod affinity": { - expectedError: "can only be empty for PreferredDuringScheduling pod anti affinity", + "invalid soft pod affinity, empty topologyKey is not allowed for soft pod affinity": { + expectedError: "can not be empty", spec: api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "123", @@ -4816,6 +5126,36 @@ func TestValidatePod(t *testing.T) { }), }, }, + "invalid soft pod anti-affinity, empty topologyKey is not allowed for soft pod anti-affinity": { + expectedError: "can not be empty", + spec: api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "123", + Namespace: "ns", + }, + Spec: validPodSpec(&api.Affinity{ + PodAntiAffinity: &api.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []api.WeightedPodAffinityTerm{ + { + Weight: 10, + PodAffinityTerm: api.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key2", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"value1", "value2"}, + }, + }, + }, + Namespaces: []string{"ns"}, + }, + }, + }, + }, + }), + }, + }, "invalid toleration key": { expectedError: "spec.tolerations[0].key", spec: api.Pod{ @@ -5045,7 +5385,7 @@ func TestValidatePod(t *testing.T) { }, }, "invalid opaque integer resource requirement: request must be <= limit": { - expectedError: "must be greater than or equal to pod.alpha.kubernetes.io/opaque-int-resource-A", + expectedError: "must be less than or equal to pod.alpha.kubernetes.io/opaque-int-resource-A", spec: api.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "123", Namespace: "ns"}, Spec: api.PodSpec{ @@ -6554,6 +6894,32 @@ func TestValidateService(t *testing.T) { numErrs: 0, }, // ESIPP section ends. + { + name: "invalid timeoutSeconds field", + tweakSvc: func(s *api.Service) { + s.Spec.Type = api.ServiceTypeClusterIP + s.Spec.SessionAffinity = api.ServiceAffinityClientIP + s.Spec.SessionAffinityConfig = &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: newInt32(-1), + }, + } + }, + numErrs: 1, + }, + { + name: "sessionAffinityConfig can't be set when session affinity is None", + tweakSvc: func(s *api.Service) { + s.Spec.Type = api.ServiceTypeLoadBalancer + s.Spec.SessionAffinity = api.ServiceAffinityNone + s.Spec.SessionAffinityConfig = &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: newInt32(90), + }, + } + }, + numErrs: 1, + }, } for _, tc := range testCases { @@ -7955,6 +8321,11 @@ func TestValidateServiceUpdate(t *testing.T) { name: "change affinity", tweakSvc: func(oldSvc, newSvc *api.Service) { newSvc.Spec.SessionAffinity = "ClientIP" + newSvc.Spec.SessionAffinityConfig = &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: newInt32(90), + }, + } }, numErrs: 0, }, @@ -10076,3 +10447,62 @@ func TestValidateFlexVolumeSource(t *testing.T) { } } } + +func TestValidateOrSetClientIPAffinityConfig(t *testing.T) { + successCases := map[string]*api.SessionAffinityConfig{ + "non-empty config, valid timeout: 1": { + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: newInt32(1), + }, + }, + "non-empty config, valid timeout: api.MaxClientIPServiceAffinitySeconds-1": { + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: newInt32(int(api.MaxClientIPServiceAffinitySeconds - 1)), + }, + }, + "non-empty config, valid timeout: api.MaxClientIPServiceAffinitySeconds": { + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: newInt32(int(api.MaxClientIPServiceAffinitySeconds)), + }, + }, + } + + for name, test := range successCases { + if errs := validateClientIPAffinityConfig(test, field.NewPath("field")); len(errs) != 0 { + t.Errorf("case: %s, expected success: %v", name, errs) + } + } + + errorCases := map[string]*api.SessionAffinityConfig{ + "empty session affinity config": nil, + "empty client IP config": { + ClientIP: nil, + }, + "empty timeoutSeconds": { + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: nil, + }, + }, + "non-empty config, invalid timeout: api.MaxClientIPServiceAffinitySeconds+1": { + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: newInt32(int(api.MaxClientIPServiceAffinitySeconds + 1)), + }, + }, + "non-empty config, invalid timeout: -1": { + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: newInt32(-1), + }, + }, + "non-empty config, invalid timeout: 0": { + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: newInt32(0), + }, + }, + } + + for name, test := range errorCases { + if errs := validateClientIPAffinityConfig(test, field.NewPath("field")); len(errs) == 0 { + t.Errorf("case: %v, expected failures: %v", name, errs) + } + } +} diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index efba5e24a5007..3a3d100284913 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -58,6 +58,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*AzureDiskVolumeSource).DeepCopyInto(out.(*AzureDiskVolumeSource)) return nil }, InType: reflect.TypeOf(&AzureDiskVolumeSource{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*AzureFilePersistentVolumeSource).DeepCopyInto(out.(*AzureFilePersistentVolumeSource)) + return nil + }, InType: reflect.TypeOf(&AzureFilePersistentVolumeSource{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*AzureFileVolumeSource).DeepCopyInto(out.(*AzureFileVolumeSource)) return nil @@ -70,6 +74,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*Capabilities).DeepCopyInto(out.(*Capabilities)) return nil }, InType: reflect.TypeOf(&Capabilities{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*CephFSPersistentVolumeSource).DeepCopyInto(out.(*CephFSPersistentVolumeSource)) + return nil + }, InType: reflect.TypeOf(&CephFSPersistentVolumeSource{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*CephFSVolumeSource).DeepCopyInto(out.(*CephFSVolumeSource)) return nil @@ -78,6 +86,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*CinderVolumeSource).DeepCopyInto(out.(*CinderVolumeSource)) return nil }, InType: reflect.TypeOf(&CinderVolumeSource{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*ClientIPConfig).DeepCopyInto(out.(*ClientIPConfig)) + return nil + }, InType: reflect.TypeOf(&ClientIPConfig{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*ComponentCondition).DeepCopyInto(out.(*ComponentCondition)) return nil @@ -634,6 +646,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*SecretProjection).DeepCopyInto(out.(*SecretProjection)) return nil }, InType: reflect.TypeOf(&SecretProjection{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*SecretReference).DeepCopyInto(out.(*SecretReference)) + return nil + }, InType: reflect.TypeOf(&SecretReference{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*SecretVolumeSource).DeepCopyInto(out.(*SecretVolumeSource)) return nil @@ -678,6 +694,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*ServiceStatus).DeepCopyInto(out.(*ServiceStatus)) return nil }, InType: reflect.TypeOf(&ServiceStatus{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*SessionAffinityConfig).DeepCopyInto(out.(*SessionAffinityConfig)) + return nil + }, InType: reflect.TypeOf(&SessionAffinityConfig{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*StorageOSPersistentVolumeSource).DeepCopyInto(out.(*StorageOSPersistentVolumeSource)) return nil @@ -879,6 +899,31 @@ func (in *AzureDiskVolumeSource) DeepCopy() *AzureDiskVolumeSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureFilePersistentVolumeSource) DeepCopyInto(out *AzureFilePersistentVolumeSource) { + *out = *in + if in.SecretNamespace != nil { + in, out := &in.SecretNamespace, &out.SecretNamespace + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureFilePersistentVolumeSource. +func (in *AzureFilePersistentVolumeSource) DeepCopy() *AzureFilePersistentVolumeSource { + if in == nil { + return nil + } + out := new(AzureFilePersistentVolumeSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureFileVolumeSource) DeepCopyInto(out *AzureFileVolumeSource) { *out = *in @@ -949,6 +994,36 @@ func (in *Capabilities) DeepCopy() *Capabilities { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CephFSPersistentVolumeSource) DeepCopyInto(out *CephFSPersistentVolumeSource) { + *out = *in + if in.Monitors != nil { + in, out := &in.Monitors, &out.Monitors + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + if *in == nil { + *out = nil + } else { + *out = new(SecretReference) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephFSPersistentVolumeSource. +func (in *CephFSPersistentVolumeSource) DeepCopy() *CephFSPersistentVolumeSource { + if in == nil { + return nil + } + out := new(CephFSPersistentVolumeSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CephFSVolumeSource) DeepCopyInto(out *CephFSVolumeSource) { *out = *in @@ -995,6 +1070,31 @@ func (in *CinderVolumeSource) DeepCopy() *CinderVolumeSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientIPConfig) DeepCopyInto(out *ClientIPConfig) { + *out = *in + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientIPConfig. +func (in *ClientIPConfig) DeepCopy() *ClientIPConfig { + if in == nil { + return nil + } + out := new(ClientIPConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ComponentCondition) DeepCopyInto(out *ComponentCondition) { *out = *in @@ -2303,6 +2403,15 @@ func (in *HostAlias) DeepCopy() *HostAlias { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HostPathVolumeSource) DeepCopyInto(out *HostPathVolumeSource) { *out = *in + if in.Type != nil { + in, out := &in.Type, &out.Type + if *in == nil { + *out = nil + } else { + *out = new(HostPathType) + **out = **in + } + } return } @@ -2333,6 +2442,15 @@ func (in *ISCSIVolumeSource) DeepCopyInto(out *ISCSIVolumeSource) { **out = **in } } + if in.InitiatorName != nil { + in, out := &in.InitiatorName, &out.InitiatorName + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } return } @@ -3563,7 +3681,7 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) { *out = nil } else { *out = new(HostPathVolumeSource) - **out = **in + (*in).DeepCopyInto(*out) } } if in.Glusterfs != nil { @@ -3634,7 +3752,7 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) { if *in == nil { *out = nil } else { - *out = new(CephFSVolumeSource) + *out = new(CephFSPersistentVolumeSource) (*in).DeepCopyInto(*out) } } @@ -3661,8 +3779,8 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) { if *in == nil { *out = nil } else { - *out = new(AzureFileVolumeSource) - **out = **in + *out = new(AzureFilePersistentVolumeSource) + (*in).DeepCopyInto(*out) } } if in.VsphereVolume != nil { @@ -3766,6 +3884,11 @@ func (in *PersistentVolumeSpec) DeepCopyInto(out *PersistentVolumeSpec) { **out = **in } } + if in.MountOptions != nil { + in, out := &in.MountOptions, &out.MountOptions + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -5216,6 +5339,22 @@ func (in *SecretProjection) DeepCopy() *SecretProjection { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretReference) DeepCopyInto(out *SecretReference) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference. +func (in *SecretReference) DeepCopy() *SecretReference { + if in == nil { + return nil + } + out := new(SecretReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretVolumeSource) DeepCopyInto(out *SecretVolumeSource) { *out = *in @@ -5569,6 +5708,15 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.SessionAffinityConfig != nil { + in, out := &in.SessionAffinityConfig, &out.SessionAffinityConfig + if *in == nil { + *out = nil + } else { + *out = new(SessionAffinityConfig) + (*in).DeepCopyInto(*out) + } + } if in.LoadBalancerSourceRanges != nil { in, out := &in.LoadBalancerSourceRanges, &out.LoadBalancerSourceRanges *out = make([]string, len(*in)) @@ -5604,6 +5752,31 @@ func (in *ServiceStatus) DeepCopy() *ServiceStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SessionAffinityConfig) DeepCopyInto(out *SessionAffinityConfig) { + *out = *in + if in.ClientIP != nil { + in, out := &in.ClientIP, &out.ClientIP + if *in == nil { + *out = nil + } else { + *out = new(ClientIPConfig) + (*in).DeepCopyInto(*out) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SessionAffinityConfig. +func (in *SessionAffinityConfig) DeepCopy() *SessionAffinityConfig { + if in == nil { + return nil + } + out := new(SessionAffinityConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageOSPersistentVolumeSource) DeepCopyInto(out *StorageOSPersistentVolumeSource) { *out = *in @@ -5814,7 +5987,7 @@ func (in *VolumeSource) DeepCopyInto(out *VolumeSource) { *out = nil } else { *out = new(HostPathVolumeSource) - **out = **in + (*in).DeepCopyInto(*out) } } if in.EmptyDir != nil { diff --git a/pkg/apis/OWNERS b/pkg/apis/OWNERS index 646f794bcaf8b..c8cd82a728e83 100644 --- a/pkg/apis/OWNERS +++ b/pkg/apis/OWNERS @@ -3,6 +3,8 @@ approvers: - lavalamp - smarterclayton - thockin +- liggitt +# - bgrant0607 # manual escalations only reviewers: - lavalamp - smarterclayton diff --git a/pkg/apis/admission/BUILD b/pkg/apis/admission/BUILD index ea672dbcacc6f..2756df077cdaf 100644 --- a/pkg/apis/admission/BUILD +++ b/pkg/apis/admission/BUILD @@ -19,7 +19,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", ], ) diff --git a/pkg/apis/admission/types.go b/pkg/apis/admission/types.go index 86efca6d7f53e..a816fa8d3faa4 100644 --- a/pkg/apis/admission/types.go +++ b/pkg/apis/admission/types.go @@ -19,7 +19,6 @@ package admission import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apiserver/pkg/admission" "k8s.io/kubernetes/pkg/apis/authentication" ) @@ -51,7 +50,7 @@ type AdmissionReviewSpec struct { // OldObject is the existing object. Only populated for UPDATE requests. OldObject runtime.Object // Operation is the operation being performed - Operation admission.Operation + Operation Operation // Resource is the name of the resource being requested. This is not the kind. For example: pods Resource metav1.GroupVersionResource // SubResource is the name of the subresource being requested. This is a different resource, scoped to the parent @@ -73,3 +72,14 @@ type AdmissionReviewStatus struct { // +optional Result *metav1.Status } + +// Operation is the type of resource operation being checked for admission control +type Operation string + +// Operation constants +const ( + Create Operation = "CREATE" + Update Operation = "UPDATE" + Delete Operation = "DELETE" + Connect Operation = "CONNECT" +) diff --git a/pkg/apis/admission/v1alpha1/helpers.go b/pkg/apis/admission/v1alpha1/helpers.go index c7a3960b6555c..ad1e543744cd3 100644 --- a/pkg/apis/admission/v1alpha1/helpers.go +++ b/pkg/apis/admission/v1alpha1/helpers.go @@ -51,7 +51,7 @@ func NewAdmissionReview(attr admission.Attributes) admissionv1alpha1.AdmissionRe Version: gvr.Version, }, SubResource: attr.GetSubresource(), - Operation: attr.GetOperation(), + Operation: admissionv1alpha1.Operation(attr.GetOperation()), Object: runtime.RawExtension{ Object: attr.GetObject(), }, diff --git a/pkg/apis/admission/v1alpha1/zz_generated.conversion.go b/pkg/apis/admission/v1alpha1/zz_generated.conversion.go index e30473017571f..c96eb913882c8 100644 --- a/pkg/apis/admission/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/admission/v1alpha1/zz_generated.conversion.go @@ -25,7 +25,6 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" - pkg_admission "k8s.io/apiserver/pkg/admission" admission "k8s.io/kubernetes/pkg/apis/admission" unsafe "unsafe" ) @@ -85,7 +84,7 @@ func autoConvert_v1alpha1_AdmissionReviewSpec_To_admission_AdmissionReviewSpec(i if err := runtime.Convert_runtime_RawExtension_To_runtime_Object(&in.OldObject, &out.OldObject, s); err != nil { return err } - out.Operation = pkg_admission.Operation(in.Operation) + out.Operation = admission.Operation(in.Operation) out.Name = in.Name out.Namespace = in.Namespace out.Resource = in.Resource @@ -112,7 +111,7 @@ func autoConvert_admission_AdmissionReviewSpec_To_v1alpha1_AdmissionReviewSpec(i if err := runtime.Convert_runtime_Object_To_runtime_RawExtension(&in.OldObject, &out.OldObject, s); err != nil { return err } - out.Operation = pkg_admission.Operation(in.Operation) + out.Operation = v1alpha1.Operation(in.Operation) out.Resource = in.Resource out.SubResource = in.SubResource // TODO: Inefficient conversion - can we improve it? diff --git a/pkg/apis/admissionregistration/fuzzer/fuzzer.go b/pkg/apis/admissionregistration/fuzzer/fuzzer.go index 4029e02298c7d..7c0287f3cc347 100644 --- a/pkg/apis/admissionregistration/fuzzer/fuzzer.go +++ b/pkg/apis/admissionregistration/fuzzer/fuzzer.go @@ -31,10 +31,5 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { p := admissionregistration.FailurePolicyType("Fail") obj.FailurePolicy = &p }, - func(obj *admissionregistration.Initializer, c fuzz.Continue) { - c.FuzzNoCustom(obj) // fuzz self without calling this function again - p := admissionregistration.FailurePolicyType("Fail") - obj.FailurePolicy = &p - }, } } diff --git a/pkg/apis/admissionregistration/types.go b/pkg/apis/admissionregistration/types.go index 888a3ebffae23..b467656003709 100644 --- a/pkg/apis/admissionregistration/types.go +++ b/pkg/apis/admissionregistration/types.go @@ -70,13 +70,6 @@ type Initializer struct { // The initializer cares about an operation if it matches _any_ Rule. // Rule.Resources must not include subresources. Rules []Rule - - // FailurePolicy defines what happens if the responsible initializer controller - // fails to takes action. Allowed values are Ignore, or Fail. If "Ignore" is - // set, initializer is removed from the initializers list of an object if - // the timeout is reached; If "Fail" is set, admissionregistration returns timeout error - // if the timeout is reached. - FailurePolicy *FailurePolicyType } // Rule is a tuple of APIGroups, APIVersion, and Resources.It is recommended diff --git a/pkg/apis/admissionregistration/v1alpha1/defaults.go b/pkg/apis/admissionregistration/v1alpha1/defaults.go index 0757d1bfc76a2..92a37ad1ccd73 100644 --- a/pkg/apis/admissionregistration/v1alpha1/defaults.go +++ b/pkg/apis/admissionregistration/v1alpha1/defaults.go @@ -25,13 +25,6 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error { return RegisterDefaults(scheme) } -func SetDefaults_Initializer(obj *admissionregistrationv1alpha1.Initializer) { - if obj.FailurePolicy == nil { - policy := admissionregistrationv1alpha1.Ignore - obj.FailurePolicy = &policy - } -} - func SetDefaults_ExternalAdmissionHook(obj *admissionregistrationv1alpha1.ExternalAdmissionHook) { if obj.FailurePolicy == nil { policy := admissionregistrationv1alpha1.Ignore diff --git a/pkg/apis/admissionregistration/v1alpha1/zz_generated.conversion.go b/pkg/apis/admissionregistration/v1alpha1/zz_generated.conversion.go index 9e278de1bc520..86ef143f0ae73 100644 --- a/pkg/apis/admissionregistration/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/admissionregistration/v1alpha1/zz_generated.conversion.go @@ -76,11 +76,7 @@ func autoConvert_admissionregistration_AdmissionHookClientConfig_To_v1alpha1_Adm if err := Convert_admissionregistration_ServiceReference_To_v1alpha1_ServiceReference(&in.Service, &out.Service, s); err != nil { return err } - if in.CABundle == nil { - out.CABundle = make([]byte, 0) - } else { - out.CABundle = *(*[]byte)(unsafe.Pointer(&in.CABundle)) - } + out.CABundle = *(*[]byte)(unsafe.Pointer(&in.CABundle)) return nil } @@ -154,11 +150,7 @@ func Convert_v1alpha1_ExternalAdmissionHookConfigurationList_To_admissionregistr func autoConvert_admissionregistration_ExternalAdmissionHookConfigurationList_To_v1alpha1_ExternalAdmissionHookConfigurationList(in *admissionregistration.ExternalAdmissionHookConfigurationList, out *v1alpha1.ExternalAdmissionHookConfigurationList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1alpha1.ExternalAdmissionHookConfiguration, 0) - } else { - out.Items = *(*[]v1alpha1.ExternalAdmissionHookConfiguration)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1alpha1.ExternalAdmissionHookConfiguration)(unsafe.Pointer(&in.Items)) return nil } @@ -170,7 +162,6 @@ func Convert_admissionregistration_ExternalAdmissionHookConfigurationList_To_v1a func autoConvert_v1alpha1_Initializer_To_admissionregistration_Initializer(in *v1alpha1.Initializer, out *admissionregistration.Initializer, s conversion.Scope) error { out.Name = in.Name out.Rules = *(*[]admissionregistration.Rule)(unsafe.Pointer(&in.Rules)) - out.FailurePolicy = (*admissionregistration.FailurePolicyType)(unsafe.Pointer(in.FailurePolicy)) return nil } @@ -182,7 +173,6 @@ func Convert_v1alpha1_Initializer_To_admissionregistration_Initializer(in *v1alp func autoConvert_admissionregistration_Initializer_To_v1alpha1_Initializer(in *admissionregistration.Initializer, out *v1alpha1.Initializer, s conversion.Scope) error { out.Name = in.Name out.Rules = *(*[]v1alpha1.Rule)(unsafe.Pointer(&in.Rules)) - out.FailurePolicy = (*v1alpha1.FailurePolicyType)(unsafe.Pointer(in.FailurePolicy)) return nil } @@ -226,11 +216,7 @@ func Convert_v1alpha1_InitializerConfigurationList_To_admissionregistration_Init func autoConvert_admissionregistration_InitializerConfigurationList_To_v1alpha1_InitializerConfigurationList(in *admissionregistration.InitializerConfigurationList, out *v1alpha1.InitializerConfigurationList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1alpha1.InitializerConfiguration, 0) - } else { - out.Items = *(*[]v1alpha1.InitializerConfiguration)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1alpha1.InitializerConfiguration)(unsafe.Pointer(&in.Items)) return nil } diff --git a/pkg/apis/admissionregistration/v1alpha1/zz_generated.defaults.go b/pkg/apis/admissionregistration/v1alpha1/zz_generated.defaults.go index b21fc9896dfb3..da214c5d95f38 100644 --- a/pkg/apis/admissionregistration/v1alpha1/zz_generated.defaults.go +++ b/pkg/apis/admissionregistration/v1alpha1/zz_generated.defaults.go @@ -35,12 +35,6 @@ func RegisterDefaults(scheme *runtime.Scheme) error { scheme.AddTypeDefaultingFunc(&v1alpha1.ExternalAdmissionHookConfigurationList{}, func(obj interface{}) { SetObjectDefaults_ExternalAdmissionHookConfigurationList(obj.(*v1alpha1.ExternalAdmissionHookConfigurationList)) }) - scheme.AddTypeDefaultingFunc(&v1alpha1.InitializerConfiguration{}, func(obj interface{}) { - SetObjectDefaults_InitializerConfiguration(obj.(*v1alpha1.InitializerConfiguration)) - }) - scheme.AddTypeDefaultingFunc(&v1alpha1.InitializerConfigurationList{}, func(obj interface{}) { - SetObjectDefaults_InitializerConfigurationList(obj.(*v1alpha1.InitializerConfigurationList)) - }) return nil } @@ -57,17 +51,3 @@ func SetObjectDefaults_ExternalAdmissionHookConfigurationList(in *v1alpha1.Exter SetObjectDefaults_ExternalAdmissionHookConfiguration(a) } } - -func SetObjectDefaults_InitializerConfiguration(in *v1alpha1.InitializerConfiguration) { - for i := range in.Initializers { - a := &in.Initializers[i] - SetDefaults_Initializer(a) - } -} - -func SetObjectDefaults_InitializerConfigurationList(in *v1alpha1.InitializerConfigurationList) { - for i := range in.Items { - a := &in.Items[i] - SetObjectDefaults_InitializerConfiguration(a) - } -} diff --git a/pkg/apis/admissionregistration/validation/validation.go b/pkg/apis/admissionregistration/validation/validation.go index 4c478f457e041..f161c98dee8b9 100644 --- a/pkg/apis/admissionregistration/validation/validation.go +++ b/pkg/apis/admissionregistration/validation/validation.go @@ -52,10 +52,6 @@ func validateInitializer(initializer *admissionregistration.Initializer, fldPath notAllowSubresources := false allErrors = append(allErrors, validateRule(&rule, fldPath.Child("rules").Index(i), notAllowSubresources)...) } - // TODO: relax the validation rule when admissionregistration is beta. - if initializer.FailurePolicy != nil && *initializer.FailurePolicy != admissionregistration.Ignore { - allErrors = append(allErrors, field.NotSupported(fldPath.Child("failurePolicy"), *initializer.FailurePolicy, []string{string(admissionregistration.Ignore)})) - } return allErrors } diff --git a/pkg/apis/admissionregistration/validation/validation_test.go b/pkg/apis/admissionregistration/validation/validation_test.go index a864ac3a86633..eca9376a67743 100644 --- a/pkg/apis/admissionregistration/validation/validation_test.go +++ b/pkg/apis/admissionregistration/validation/validation_test.go @@ -214,20 +214,6 @@ func TestValidateInitializerConfiguration(t *testing.T) { }), expectedError: ` "a/b": must not specify subresources`, }, - { - name: "FailurePolicy can only be \"Ignore\"", - config: getInitializerConfiguration( - []admissionregistration.Initializer{ - { - Name: "initializer.k8s.io", - FailurePolicy: func() *admissionregistration.FailurePolicyType { - r := admissionregistration.Fail - return &r - }(), - }, - }), - expectedError: `failurePolicy: Unsupported value: "Fail": supported values: Ignore`, - }, } for _, test := range tests { diff --git a/pkg/apis/admissionregistration/zz_generated.deepcopy.go b/pkg/apis/admissionregistration/zz_generated.deepcopy.go index edb368a311f05..4952a75f79153 100644 --- a/pkg/apis/admissionregistration/zz_generated.deepcopy.go +++ b/pkg/apis/admissionregistration/zz_generated.deepcopy.go @@ -212,15 +212,6 @@ func (in *Initializer) DeepCopyInto(out *Initializer) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.FailurePolicy != nil { - in, out := &in.FailurePolicy, &out.FailurePolicy - if *in == nil { - *out = nil - } else { - *out = new(FailurePolicyType) - **out = **in - } - } return } diff --git a/pkg/apis/apps/fuzzer/fuzzer.go b/pkg/apis/apps/fuzzer/fuzzer.go index beb97dc168474..8013a273b015a 100644 --- a/pkg/apis/apps/fuzzer/fuzzer.go +++ b/pkg/apis/apps/fuzzer/fuzzer.go @@ -44,7 +44,7 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { s.Status.ObservedGeneration = new(int64) } if s.Status.CollisionCount == nil { - s.Status.CollisionCount = new(int64) + s.Status.CollisionCount = new(int32) } }, } diff --git a/pkg/apis/apps/types.go b/pkg/apis/apps/types.go index 9841c2ab4f6fc..5e2b010c73012 100644 --- a/pkg/apis/apps/types.go +++ b/pkg/apis/apps/types.go @@ -192,7 +192,7 @@ type StatefulSetStatus struct { // uses this field as a collision avoidance mechanism when it needs to create the name for the // newest ControllerRevision. // +optional - CollisionCount *int64 + CollisionCount *int32 } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/apps/v1beta1/conversion.go b/pkg/apis/apps/v1beta1/conversion.go index 600773d34c98e..42d26b86d3152 100644 --- a/pkg/apis/apps/v1beta1/conversion.go +++ b/pkg/apis/apps/v1beta1/conversion.go @@ -70,18 +70,6 @@ func addConversionFuncs(scheme *runtime.Scheme) error { if err != nil { return err } - err = scheme.AddFieldLabelConversionFunc("apps/v1beta1", "Deployment", - func(label, value string) (string, string, error) { - switch label { - case "metadata.name", "metadata.namespace": - return label, value, nil - default: - return "", "", fmt.Errorf("field label %q not supported for appsv1beta1.Deployment", label) - } - }) - if err != nil { - return err - } return nil } diff --git a/pkg/apis/apps/v1beta1/zz_generated.conversion.go b/pkg/apis/apps/v1beta1/zz_generated.conversion.go index 972e5a70a2556..d3ddaac0333fa 100644 --- a/pkg/apis/apps/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/apps/v1beta1/zz_generated.conversion.go @@ -144,7 +144,7 @@ func autoConvert_apps_ControllerRevisionList_To_v1beta1_ControllerRevisionList(i } } } else { - out.Items = make([]v1beta1.ControllerRevision, 0) + out.Items = nil } return nil } @@ -248,7 +248,7 @@ func autoConvert_extensions_DeploymentList_To_v1beta1_DeploymentList(in *extensi } } } else { - out.Items = make([]v1beta1.Deployment, 0) + out.Items = nil } return nil } @@ -332,7 +332,7 @@ func autoConvert_v1beta1_DeploymentStatus_To_extensions_DeploymentStatus(in *v1b out.AvailableReplicas = in.AvailableReplicas out.UnavailableReplicas = in.UnavailableReplicas out.Conditions = *(*[]extensions.DeploymentCondition)(unsafe.Pointer(&in.Conditions)) - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -349,7 +349,7 @@ func autoConvert_extensions_DeploymentStatus_To_v1beta1_DeploymentStatus(in *ext out.AvailableReplicas = in.AvailableReplicas out.UnavailableReplicas = in.UnavailableReplicas out.Conditions = *(*[]v1beta1.DeploymentCondition)(unsafe.Pointer(&in.Conditions)) - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -571,7 +571,7 @@ func autoConvert_apps_StatefulSetList_To_v1beta1_StatefulSetList(in *apps.Statef } } } else { - out.Items = make([]v1beta1.StatefulSet, 0) + out.Items = nil } return nil } @@ -625,7 +625,7 @@ func autoConvert_v1beta1_StatefulSetStatus_To_apps_StatefulSetStatus(in *v1beta1 out.UpdatedReplicas = in.UpdatedReplicas out.CurrentRevision = in.CurrentRevision out.UpdateRevision = in.UpdateRevision - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -642,7 +642,7 @@ func autoConvert_apps_StatefulSetStatus_To_v1beta1_StatefulSetStatus(in *apps.St out.UpdatedReplicas = in.UpdatedReplicas out.CurrentRevision = in.CurrentRevision out.UpdateRevision = in.UpdateRevision - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } diff --git a/pkg/apis/apps/v1beta1/zz_generated.defaults.go b/pkg/apis/apps/v1beta1/zz_generated.defaults.go index ed9daa94b5dcb..26fde73fe2d05 100644 --- a/pkg/apis/apps/v1beta1/zz_generated.defaults.go +++ b/pkg/apis/apps/v1beta1/zz_generated.defaults.go @@ -43,6 +43,9 @@ func SetObjectDefaults_Deployment(in *v1beta1.Deployment) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -184,6 +187,9 @@ func SetObjectDefaults_StatefulSet(in *v1beta1.StatefulSet) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } diff --git a/pkg/apis/apps/v1beta2/conversion.go b/pkg/apis/apps/v1beta2/conversion.go index 01eb4528920b4..191c2eb56b42a 100644 --- a/pkg/apis/apps/v1beta2/conversion.go +++ b/pkg/apis/apps/v1beta2/conversion.go @@ -85,30 +85,6 @@ func addConversionFuncs(scheme *runtime.Scheme) error { if err != nil { return err } - err = scheme.AddFieldLabelConversionFunc("apps/v1beta2", "Deployment", - func(label, value string) (string, string, error) { - switch label { - case "metadata.name", "metadata.namespace": - return label, value, nil - default: - return "", "", fmt.Errorf("field label %q not supported for appsv1beta2.Deployment", label) - } - }) - if err != nil { - return err - } - err = scheme.AddFieldLabelConversionFunc("apps/v1beta2", "ReplicaSet", - func(label, value string) (string, string, error) { - switch label { - case "metadata.name", "metadata.namespace": - return label, value, nil - default: - return "", "", fmt.Errorf("field label %q not supported for appsv1beta2.ReplicaSet", label) - } - }) - if err != nil { - return err - } return nil } @@ -244,7 +220,7 @@ func Convert_v1beta2_StatefulSetStatus_To_apps_StatefulSetStatus(in *appsv1beta2 out.CurrentRevision = in.CurrentRevision out.UpdateRevision = in.UpdateRevision if in.CollisionCount != nil { - out.CollisionCount = new(int64) + out.CollisionCount = new(int32) *out.CollisionCount = *in.CollisionCount } return nil @@ -261,7 +237,7 @@ func Convert_apps_StatefulSetStatus_To_v1beta2_StatefulSetStatus(in *apps.Statef out.CurrentRevision = in.CurrentRevision out.UpdateRevision = in.UpdateRevision if in.CollisionCount != nil { - out.CollisionCount = new(int64) + out.CollisionCount = new(int32) *out.CollisionCount = *in.CollisionCount } return nil diff --git a/pkg/apis/apps/v1beta2/zz_generated.conversion.go b/pkg/apis/apps/v1beta2/zz_generated.conversion.go index 6dabfaa441768..4624fa42bf09a 100644 --- a/pkg/apis/apps/v1beta2/zz_generated.conversion.go +++ b/pkg/apis/apps/v1beta2/zz_generated.conversion.go @@ -162,7 +162,7 @@ func autoConvert_apps_ControllerRevisionList_To_v1beta2_ControllerRevisionList(i } } } else { - out.Items = make([]v1beta2.ControllerRevision, 0) + out.Items = nil } return nil } @@ -226,7 +226,7 @@ func autoConvert_extensions_DaemonSetList_To_v1beta2_DaemonSetList(in *extension } } } else { - out.Items = make([]v1beta2.DaemonSet, 0) + out.Items = nil } return nil } @@ -272,7 +272,7 @@ func autoConvert_v1beta2_DaemonSetStatus_To_extensions_DaemonSetStatus(in *v1bet out.UpdatedNumberScheduled = in.UpdatedNumberScheduled out.NumberAvailable = in.NumberAvailable out.NumberUnavailable = in.NumberUnavailable - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -290,7 +290,7 @@ func autoConvert_extensions_DaemonSetStatus_To_v1beta2_DaemonSetStatus(in *exten out.UpdatedNumberScheduled = in.UpdatedNumberScheduled out.NumberAvailable = in.NumberAvailable out.NumberUnavailable = in.NumberUnavailable - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -411,7 +411,7 @@ func autoConvert_extensions_DeploymentList_To_v1beta2_DeploymentList(in *extensi } } } else { - out.Items = make([]v1beta2.Deployment, 0) + out.Items = nil } return nil } @@ -466,7 +466,7 @@ func autoConvert_v1beta2_DeploymentStatus_To_extensions_DeploymentStatus(in *v1b out.AvailableReplicas = in.AvailableReplicas out.UnavailableReplicas = in.UnavailableReplicas out.Conditions = *(*[]extensions.DeploymentCondition)(unsafe.Pointer(&in.Conditions)) - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -483,7 +483,7 @@ func autoConvert_extensions_DeploymentStatus_To_v1beta2_DeploymentStatus(in *ext out.AvailableReplicas = in.AvailableReplicas out.UnavailableReplicas = in.UnavailableReplicas out.Conditions = *(*[]v1beta2.DeploymentCondition)(unsafe.Pointer(&in.Conditions)) - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -612,7 +612,7 @@ func autoConvert_extensions_ReplicaSetList_To_v1beta2_ReplicaSetList(in *extensi } } } else { - out.Items = make([]v1beta2.ReplicaSet, 0) + out.Items = nil } return nil } @@ -851,7 +851,7 @@ func autoConvert_apps_StatefulSetList_To_v1beta2_StatefulSetList(in *apps.Statef } } } else { - out.Items = make([]v1beta2.StatefulSet, 0) + out.Items = nil } return nil } @@ -905,7 +905,7 @@ func autoConvert_v1beta2_StatefulSetStatus_To_apps_StatefulSetStatus(in *v1beta2 out.UpdatedReplicas = in.UpdatedReplicas out.CurrentRevision = in.CurrentRevision out.UpdateRevision = in.UpdateRevision - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -917,7 +917,7 @@ func autoConvert_apps_StatefulSetStatus_To_v1beta2_StatefulSetStatus(in *apps.St out.UpdatedReplicas = in.UpdatedReplicas out.CurrentRevision = in.CurrentRevision out.UpdateRevision = in.UpdateRevision - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } diff --git a/pkg/apis/apps/v1beta2/zz_generated.defaults.go b/pkg/apis/apps/v1beta2/zz_generated.defaults.go index 02ced2d8e3998..29eabcaf89b2a 100644 --- a/pkg/apis/apps/v1beta2/zz_generated.defaults.go +++ b/pkg/apis/apps/v1beta2/zz_generated.defaults.go @@ -47,6 +47,9 @@ func SetObjectDefaults_DaemonSet(in *v1beta2.DaemonSet) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -188,6 +191,9 @@ func SetObjectDefaults_Deployment(in *v1beta2.Deployment) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -329,6 +335,9 @@ func SetObjectDefaults_ReplicaSet(in *v1beta2.ReplicaSet) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -470,6 +479,9 @@ func SetObjectDefaults_StatefulSet(in *v1beta2.StatefulSet) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } diff --git a/pkg/apis/apps/validation/validation.go b/pkg/apis/apps/validation/validation.go index 6b0e7d985343a..85bc564ff468b 100644 --- a/pkg/apis/apps/validation/validation.go +++ b/pkg/apis/apps/validation/validation.go @@ -158,8 +158,6 @@ func ValidateStatefulSetUpdate(statefulSet, oldStatefulSet *apps.StatefulSet) fi statefulSet.Spec.UpdateStrategy = restoreStrategy allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(statefulSet.Spec.Replicas), field.NewPath("spec", "replicas"))...) - containerErrs, _ := apivalidation.ValidateContainerUpdates(statefulSet.Spec.Template.Spec.Containers, oldStatefulSet.Spec.Template.Spec.Containers, field.NewPath("spec").Child("template").Child("containers")) - allErrs = append(allErrs, containerErrs...) return allErrs } diff --git a/pkg/apis/apps/validation/validation_test.go b/pkg/apis/apps/validation/validation_test.go index 9071fa189f0a7..76c65bd92778f 100644 --- a/pkg/apis/apps/validation/validation_test.go +++ b/pkg/apis/apps/validation/validation_test.go @@ -302,7 +302,8 @@ func TestValidateStatefulSet(t *testing.T) { } func TestValidateStatefulSetStatus(t *testing.T) { - minusOne := int64(-1) + observedGenerationMinusOne := int64(-1) + collisionCountMinusOne := int32(-1) tests := []struct { name string replicas int32 @@ -310,7 +311,7 @@ func TestValidateStatefulSetStatus(t *testing.T) { currentReplicas int32 updatedReplicas int32 observedGeneration *int64 - collisionCount *int64 + collisionCount *int32 expectedErr bool }{ { @@ -359,7 +360,7 @@ func TestValidateStatefulSetStatus(t *testing.T) { readyReplicas: 3, currentReplicas: 2, updatedReplicas: 1, - observedGeneration: &minusOne, + observedGeneration: &observedGenerationMinusOne, expectedErr: true, }, { @@ -368,7 +369,7 @@ func TestValidateStatefulSetStatus(t *testing.T) { readyReplicas: 3, currentReplicas: 2, updatedReplicas: 1, - collisionCount: &minusOne, + collisionCount: &collisionCountMinusOne, expectedErr: true, }, { @@ -428,6 +429,21 @@ func TestValidateStatefulSetUpdate(t *testing.T) { }, }, } + + obj, err := api.Scheme.DeepCopy(validPodTemplate) + if err != nil { + t.Errorf("failure during test setup when copying PodTemplate: %v", err) + } + addContainersValidTemplate, ok := obj.(api.PodTemplate) + if !ok { + t.Errorf("failure during test setup, copied pod template is not a pod template") + } + addContainersValidTemplate.Template.Spec.Containers = append(addContainersValidTemplate.Template.Spec.Containers, + api.Container{Name: "def", Image: "image2", ImagePullPolicy: "IfNotPresent"}) + if len(addContainersValidTemplate.Template.Spec.Containers) != len(validPodTemplate.Template.Spec.Containers)+1 { + t.Errorf("failure during test setup: template %v should have more containers than template %v", addContainersValidTemplate, validPodTemplate) + } + readWriteVolumePodTemplate := api.PodTemplate{ Template: api.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -479,6 +495,46 @@ func TestValidateStatefulSetUpdate(t *testing.T) { }, }, }, + { + old: apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + }, + }, + update: apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: addContainersValidTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + }, + }, + }, + { + old: apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: addContainersValidTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + }, + }, + update: apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + Spec: apps.StatefulSetSpec{ + PodManagementPolicy: apps.OrderedReadyPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: validLabels}, + Template: validPodTemplate.Template, + UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + }, + }, + }, } for _, successCase := range successCases { successCase.old.ObjectMeta.ResourceVersion = "1" diff --git a/pkg/apis/apps/zz_generated.deepcopy.go b/pkg/apis/apps/zz_generated.deepcopy.go index 0a4d363861484..983dabf46af75 100644 --- a/pkg/apis/apps/zz_generated.deepcopy.go +++ b/pkg/apis/apps/zz_generated.deepcopy.go @@ -278,7 +278,7 @@ func (in *StatefulSetStatus) DeepCopyInto(out *StatefulSetStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } diff --git a/pkg/apis/authentication/OWNERS b/pkg/apis/authentication/OWNERS index 4135522b21d5f..95235234107c9 100755 --- a/pkg/apis/authentication/OWNERS +++ b/pkg/apis/authentication/OWNERS @@ -7,3 +7,4 @@ reviewers: - timothysc - mbohlool - jianhuiz +- enj diff --git a/pkg/apis/authorization/OWNERS b/pkg/apis/authorization/OWNERS index 2fef50443b652..a68d7eef5775a 100755 --- a/pkg/apis/authorization/OWNERS +++ b/pkg/apis/authorization/OWNERS @@ -15,3 +15,4 @@ reviewers: - mbohlool - david-mcmahon - jianhuiz +- enj diff --git a/pkg/apis/autoscaling/v1/zz_generated.conversion.go b/pkg/apis/autoscaling/v1/zz_generated.conversion.go index 5baee8823543c..e1f48bdce5356 100644 --- a/pkg/apis/autoscaling/v1/zz_generated.conversion.go +++ b/pkg/apis/autoscaling/v1/zz_generated.conversion.go @@ -183,7 +183,7 @@ func autoConvert_autoscaling_HorizontalPodAutoscalerList_To_v1_HorizontalPodAuto } } } else { - out.Items = make([]v1.HorizontalPodAutoscaler, 0) + out.Items = nil } return nil } diff --git a/pkg/apis/autoscaling/v2alpha1/BUILD b/pkg/apis/autoscaling/v2alpha1/BUILD index 3cbdae37c32a1..290a577572088 100644 --- a/pkg/apis/autoscaling/v2alpha1/BUILD +++ b/pkg/apis/autoscaling/v2alpha1/BUILD @@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( @@ -39,3 +40,19 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_xtest", + srcs = ["defaults_test.go"], + deps = [ + ":go_default_library", + "//pkg/api:go_default_library", + "//pkg/api/install:go_default_library", + "//pkg/apis/autoscaling:go_default_library", + "//pkg/apis/autoscaling/install:go_default_library", + "//vendor/k8s.io/api/autoscaling/v2alpha1:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + ], +) diff --git a/pkg/apis/autoscaling/v2alpha1/defaults_test.go b/pkg/apis/autoscaling/v2alpha1/defaults_test.go new file mode 100644 index 0000000000000..2d91214a535b0 --- /dev/null +++ b/pkg/apis/autoscaling/v2alpha1/defaults_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2alpha1_test + +import ( + "reflect" + "testing" + + autoscalingv2alpha1 "k8s.io/api/autoscaling/v2alpha1" + "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/api" + _ "k8s.io/kubernetes/pkg/api/install" + "k8s.io/kubernetes/pkg/apis/autoscaling" + _ "k8s.io/kubernetes/pkg/apis/autoscaling/install" + . "k8s.io/kubernetes/pkg/apis/autoscaling/v2alpha1" +) + +func TestSetDefaultHPA(t *testing.T) { + utilizationDefaultVal := int32(autoscaling.DefaultCPUUtilization) + defaultReplicas := newInt32(1) + defaultTemplate := []autoscalingv2alpha1.MetricSpec{ + { + Type: autoscalingv2alpha1.ResourceMetricSourceType, + Resource: &autoscalingv2alpha1.ResourceMetricSource{ + Name: v1.ResourceCPU, + TargetAverageUtilization: &utilizationDefaultVal, + }, + }, + } + + tests := []struct { + original *autoscalingv2alpha1.HorizontalPodAutoscaler + expected *autoscalingv2alpha1.HorizontalPodAutoscaler + }{ + { // MinReplicas default value + original: &autoscalingv2alpha1.HorizontalPodAutoscaler{ + Spec: autoscalingv2alpha1.HorizontalPodAutoscalerSpec{ + Metrics: defaultTemplate, + }, + }, + expected: &autoscalingv2alpha1.HorizontalPodAutoscaler{ + Spec: autoscalingv2alpha1.HorizontalPodAutoscalerSpec{ + MinReplicas: defaultReplicas, + Metrics: defaultTemplate, + }, + }, + }, + { // MinReplicas update + original: &autoscalingv2alpha1.HorizontalPodAutoscaler{ + Spec: autoscalingv2alpha1.HorizontalPodAutoscalerSpec{ + MinReplicas: newInt32(3), + Metrics: defaultTemplate, + }, + }, + expected: &autoscalingv2alpha1.HorizontalPodAutoscaler{ + Spec: autoscalingv2alpha1.HorizontalPodAutoscalerSpec{ + MinReplicas: newInt32(3), + Metrics: defaultTemplate, + }, + }, + }, + { // Metrics default value + original: &autoscalingv2alpha1.HorizontalPodAutoscaler{ + Spec: autoscalingv2alpha1.HorizontalPodAutoscalerSpec{ + MinReplicas: defaultReplicas, + }, + }, + expected: &autoscalingv2alpha1.HorizontalPodAutoscaler{ + Spec: autoscalingv2alpha1.HorizontalPodAutoscalerSpec{ + MinReplicas: defaultReplicas, + Metrics: defaultTemplate, + }, + }, + }, + } + + for i, test := range tests { + original := test.original + expected := test.expected + obj2 := roundTrip(t, runtime.Object(original)) + got, ok := obj2.(*autoscalingv2alpha1.HorizontalPodAutoscaler) + if !ok { + t.Fatalf("(%d) unexpected object: %v", i, obj2) + } + if !apiequality.Semantic.DeepEqual(got.Spec, expected.Spec) { + t.Errorf("(%d) got different than expected\ngot:\n\t%+v\nexpected:\n\t%+v", i, got.Spec, expected.Spec) + } + } +} + +func roundTrip(t *testing.T, obj runtime.Object) runtime.Object { + data, err := runtime.Encode(api.Codecs.LegacyCodec(SchemeGroupVersion), obj) + if err != nil { + t.Errorf("%v\n %#v", err, obj) + return nil + } + obj2, err := runtime.Decode(api.Codecs.UniversalDecoder(), data) + if err != nil { + t.Errorf("%v\nData: %s\nSource: %#v", err, string(data), obj) + return nil + } + obj3 := reflect.New(reflect.TypeOf(obj).Elem()).Interface().(runtime.Object) + err = api.Scheme.Convert(obj2, obj3, nil) + if err != nil { + t.Errorf("%v\nSource: %#v", err, obj2) + return nil + } + return obj3 +} + +func newInt32(val int32) *int32 { + p := new(int32) + *p = val + return p +} diff --git a/pkg/apis/autoscaling/v2alpha1/zz_generated.conversion.go b/pkg/apis/autoscaling/v2alpha1/zz_generated.conversion.go index e0c5eed6828ad..9ceea56ef4bfb 100644 --- a/pkg/apis/autoscaling/v2alpha1/zz_generated.conversion.go +++ b/pkg/apis/autoscaling/v2alpha1/zz_generated.conversion.go @@ -168,11 +168,7 @@ func Convert_v2alpha1_HorizontalPodAutoscalerList_To_autoscaling_HorizontalPodAu func autoConvert_autoscaling_HorizontalPodAutoscalerList_To_v2alpha1_HorizontalPodAutoscalerList(in *autoscaling.HorizontalPodAutoscalerList, out *v2alpha1.HorizontalPodAutoscalerList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v2alpha1.HorizontalPodAutoscaler, 0) - } else { - out.Items = *(*[]v2alpha1.HorizontalPodAutoscaler)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v2alpha1.HorizontalPodAutoscaler)(unsafe.Pointer(&in.Items)) return nil } @@ -231,16 +227,8 @@ func autoConvert_autoscaling_HorizontalPodAutoscalerStatus_To_v2alpha1_Horizonta out.LastScaleTime = (*meta_v1.Time)(unsafe.Pointer(in.LastScaleTime)) out.CurrentReplicas = in.CurrentReplicas out.DesiredReplicas = in.DesiredReplicas - if in.CurrentMetrics == nil { - out.CurrentMetrics = make([]v2alpha1.MetricStatus, 0) - } else { - out.CurrentMetrics = *(*[]v2alpha1.MetricStatus)(unsafe.Pointer(&in.CurrentMetrics)) - } - if in.Conditions == nil { - out.Conditions = make([]v2alpha1.HorizontalPodAutoscalerCondition, 0) - } else { - out.Conditions = *(*[]v2alpha1.HorizontalPodAutoscalerCondition)(unsafe.Pointer(&in.Conditions)) - } + out.CurrentMetrics = *(*[]v2alpha1.MetricStatus)(unsafe.Pointer(&in.CurrentMetrics)) + out.Conditions = *(*[]v2alpha1.HorizontalPodAutoscalerCondition)(unsafe.Pointer(&in.Conditions)) return nil } diff --git a/pkg/apis/batch/v1/zz_generated.conversion.go b/pkg/apis/batch/v1/zz_generated.conversion.go index c1b8843fcba47..015a0f76a4155 100644 --- a/pkg/apis/batch/v1/zz_generated.conversion.go +++ b/pkg/apis/batch/v1/zz_generated.conversion.go @@ -147,7 +147,7 @@ func autoConvert_batch_JobList_To_v1_JobList(in *batch.JobList, out *v1.JobList, } } } else { - out.Items = make([]v1.Job, 0) + out.Items = nil } return nil } diff --git a/pkg/apis/batch/v1/zz_generated.defaults.go b/pkg/apis/batch/v1/zz_generated.defaults.go index 6936429fb3879..18c0cc6d82ae8 100644 --- a/pkg/apis/batch/v1/zz_generated.defaults.go +++ b/pkg/apis/batch/v1/zz_generated.defaults.go @@ -41,6 +41,9 @@ func SetObjectDefaults_Job(in *v1.Job) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] api_v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + api_v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { api_v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } diff --git a/pkg/apis/batch/v1beta1/zz_generated.conversion.go b/pkg/apis/batch/v1beta1/zz_generated.conversion.go index d335dbadba32d..f6ae62bd03912 100644 --- a/pkg/apis/batch/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/batch/v1beta1/zz_generated.conversion.go @@ -119,7 +119,7 @@ func autoConvert_batch_CronJobList_To_v1beta1_CronJobList(in *batch.CronJobList, } } } else { - out.Items = make([]v1beta1.CronJob, 0) + out.Items = nil } return nil } diff --git a/pkg/apis/batch/v1beta1/zz_generated.defaults.go b/pkg/apis/batch/v1beta1/zz_generated.defaults.go index a780cfe3d46ab..e9256b91d078e 100644 --- a/pkg/apis/batch/v1beta1/zz_generated.defaults.go +++ b/pkg/apis/batch/v1beta1/zz_generated.defaults.go @@ -42,6 +42,9 @@ func SetObjectDefaults_CronJob(in *v1beta1.CronJob) { for i := range in.Spec.JobTemplate.Spec.Template.Spec.Volumes { a := &in.Spec.JobTemplate.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -182,6 +185,9 @@ func SetObjectDefaults_JobTemplate(in *v1beta1.JobTemplate) { for i := range in.Template.Spec.Template.Spec.Volumes { a := &in.Template.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } diff --git a/pkg/apis/batch/v2alpha1/zz_generated.conversion.go b/pkg/apis/batch/v2alpha1/zz_generated.conversion.go index a492fe79bd52e..92c8c8ceb6aa5 100644 --- a/pkg/apis/batch/v2alpha1/zz_generated.conversion.go +++ b/pkg/apis/batch/v2alpha1/zz_generated.conversion.go @@ -119,7 +119,7 @@ func autoConvert_batch_CronJobList_To_v2alpha1_CronJobList(in *batch.CronJobList } } } else { - out.Items = make([]v2alpha1.CronJob, 0) + out.Items = nil } return nil } diff --git a/pkg/apis/batch/v2alpha1/zz_generated.defaults.go b/pkg/apis/batch/v2alpha1/zz_generated.defaults.go index b76e4407fad26..46e8ca6121d00 100644 --- a/pkg/apis/batch/v2alpha1/zz_generated.defaults.go +++ b/pkg/apis/batch/v2alpha1/zz_generated.defaults.go @@ -42,6 +42,9 @@ func SetObjectDefaults_CronJob(in *v2alpha1.CronJob) { for i := range in.Spec.JobTemplate.Spec.Template.Spec.Volumes { a := &in.Spec.JobTemplate.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -182,6 +185,9 @@ func SetObjectDefaults_JobTemplate(in *v2alpha1.JobTemplate) { for i := range in.Template.Spec.Template.Spec.Volumes { a := &in.Template.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } diff --git a/pkg/apis/certificates/OWNERS b/pkg/apis/certificates/OWNERS index c67bd1172a1b3..6066d2c1218f7 100755 --- a/pkg/apis/certificates/OWNERS +++ b/pkg/apis/certificates/OWNERS @@ -12,3 +12,4 @@ reviewers: - mbohlool - david-mcmahon - jianhuiz +- enj diff --git a/pkg/apis/certificates/v1beta1/BUILD b/pkg/apis/certificates/v1beta1/BUILD index aa2d4eed06853..879a2bc71827a 100644 --- a/pkg/apis/certificates/v1beta1/BUILD +++ b/pkg/apis/certificates/v1beta1/BUILD @@ -8,7 +8,6 @@ load( go_library( name = "go_default_library", srcs = [ - "conversion.go", "defaults.go", "doc.go", "helpers.go", diff --git a/pkg/apis/certificates/v1beta1/conversion.go b/pkg/apis/certificates/v1beta1/conversion.go deleted file mode 100644 index b9cf0b0163a30..0000000000000 --- a/pkg/apis/certificates/v1beta1/conversion.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "fmt" - - "k8s.io/apimachinery/pkg/runtime" -) - -func addConversionFuncs(scheme *runtime.Scheme) error { - // Add non-generated conversion functions here. Currently there are none. - - return scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.String(), "CertificateSigningRequest", - func(label, value string) (string, string, error) { - switch label { - case "metadata.name": - return label, value, nil - default: - return "", "", fmt.Errorf("field label not supported: %s", label) - } - }, - ) -} diff --git a/pkg/apis/certificates/v1beta1/register.go b/pkg/apis/certificates/v1beta1/register.go index e20ef9a3106d7..dbb0e801660d4 100644 --- a/pkg/apis/certificates/v1beta1/register.go +++ b/pkg/apis/certificates/v1beta1/register.go @@ -46,5 +46,5 @@ func init() { // We only register manually written functions here. The registration of the // generated functions takes place in the generated files. The separation // makes the code compile even when the generated files are missing. - localSchemeBuilder.Register(addConversionFuncs, addDefaultingFuncs) + localSchemeBuilder.Register(addDefaultingFuncs) } diff --git a/pkg/apis/certificates/v1beta1/zz_generated.conversion.go b/pkg/apis/certificates/v1beta1/zz_generated.conversion.go index 9253dbed8ad2f..96780e6730d87 100644 --- a/pkg/apis/certificates/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/certificates/v1beta1/zz_generated.conversion.go @@ -120,11 +120,7 @@ func Convert_v1beta1_CertificateSigningRequestList_To_certificates_CertificateSi func autoConvert_certificates_CertificateSigningRequestList_To_v1beta1_CertificateSigningRequestList(in *certificates.CertificateSigningRequestList, out *v1beta1.CertificateSigningRequestList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.CertificateSigningRequest, 0) - } else { - out.Items = *(*[]v1beta1.CertificateSigningRequest)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.CertificateSigningRequest)(unsafe.Pointer(&in.Items)) return nil } @@ -149,11 +145,7 @@ func Convert_v1beta1_CertificateSigningRequestSpec_To_certificates_CertificateSi } func autoConvert_certificates_CertificateSigningRequestSpec_To_v1beta1_CertificateSigningRequestSpec(in *certificates.CertificateSigningRequestSpec, out *v1beta1.CertificateSigningRequestSpec, s conversion.Scope) error { - if in.Request == nil { - out.Request = make([]byte, 0) - } else { - out.Request = *(*[]byte)(unsafe.Pointer(&in.Request)) - } + out.Request = *(*[]byte)(unsafe.Pointer(&in.Request)) out.Usages = *(*[]v1beta1.KeyUsage)(unsafe.Pointer(&in.Usages)) out.Username = in.Username out.UID = in.UID diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index bd99e466313f0..1cf5cb20c6efd 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -299,19 +299,6 @@ type KubeControllerManagerConfiguration struct { ConcurrentSATokenSyncs int32 // lookupCacheSizeForRC is the size of lookup cache for replication controllers. // Larger number = more responsive replica management, but more MEM load. - // TODO(#43388): Remove the following flag 6 months after v1.6.0 is released. - // DEPRECATED: This is no longer used. - LookupCacheSizeForRC int32 - // lookupCacheSizeForRS is the size of lookup cache for replicatsets. - // Larger number = more responsive replica management, but more MEM load. - // TODO(#43388): Remove the following flag 6 months after v1.6.0 is released. - // DEPRECATED: This is no longer used. - LookupCacheSizeForRS int32 - // lookupCacheSizeForDaemonSet is the size of lookup cache for daemonsets. - // Larger number = more responsive daemonset, but more MEM load. - // TODO(#43388): Remove the following flag 6 months after v1.6.0 is released. - // DEPRECATED: This is no longer used. - LookupCacheSizeForDaemonSet int32 // serviceSyncPeriod is the period for syncing services with their external // load balancers. ServiceSyncPeriod metav1.Duration diff --git a/pkg/apis/extensions/types.go b/pkg/apis/extensions/types.go index 8e60c6f9ed949..55511f96f1349 100644 --- a/pkg/apis/extensions/types.go +++ b/pkg/apis/extensions/types.go @@ -345,7 +345,7 @@ type DeploymentStatus struct { // field as a collision avoidance mechanism when it needs to create the name for the // newest ReplicaSet. // +optional - CollisionCount *int64 + CollisionCount *int32 } type DeploymentConditionType string @@ -519,7 +519,7 @@ type DaemonSetStatus struct { // uses this field as a collision avoidance mechanism when it needs to // create the name for the newest ControllerRevision. // +optional - CollisionCount *int64 + CollisionCount *int32 } // +genclient diff --git a/pkg/apis/extensions/v1beta1/conversion.go b/pkg/apis/extensions/v1beta1/conversion.go index 8e8c4c08cd9fc..62167b6f201a6 100644 --- a/pkg/apis/extensions/v1beta1/conversion.go +++ b/pkg/apis/extensions/v1beta1/conversion.go @@ -60,29 +60,13 @@ func addConversionFuncs(scheme *runtime.Scheme) error { Convert_v1beta1_NetworkPolicySpec_To_networking_NetworkPolicySpec, Convert_networking_NetworkPolicySpec_To_v1beta1_NetworkPolicySpec, Convert_extensions_PodSecurityPolicySpec_To_v1beta1_PodSecurityPolicySpec, + Convert_v1beta1_IPBlock_To_networking_IPBlock, + Convert_networking_IPBlock_To_v1beta1_IPBlock, ) if err != nil { return err } - // Add field label conversions for kinds having selectable nothing but ObjectMeta fields. - for _, k := range []string{"DaemonSet", "Deployment", "Ingress"} { - kind := k // don't close over range variables - err = scheme.AddFieldLabelConversionFunc("extensions/v1beta1", kind, - func(label, value string) (string, string, error) { - switch label { - case "metadata.name", "metadata.namespace": - return label, value, nil - default: - return "", "", fmt.Errorf("field label %q not supported for %q", label, kind) - } - }, - ) - if err != nil { - return err - } - } - return nil } @@ -364,6 +348,14 @@ func Convert_v1beta1_NetworkPolicyPeer_To_networking_NetworkPolicyPeer(in *exten } else { out.NamespaceSelector = nil } + if in.IPBlock != nil { + out.IPBlock = new(networking.IPBlock) + if err := s.Convert(in.IPBlock, out.IPBlock, 0); err != nil { + return err + } + } else { + out.IPBlock = nil + } return nil } @@ -384,6 +376,30 @@ func Convert_networking_NetworkPolicyPeer_To_v1beta1_NetworkPolicyPeer(in *netwo } else { out.NamespaceSelector = nil } + if in.IPBlock != nil { + out.IPBlock = new(extensionsv1beta1.IPBlock) + if err := s.Convert(in.IPBlock, out.IPBlock, 0); err != nil { + return err + } + } else { + out.IPBlock = nil + } + return nil +} + +func Convert_v1beta1_IPBlock_To_networking_IPBlock(in *extensionsv1beta1.IPBlock, out *networking.IPBlock, s conversion.Scope) error { + out.CIDR = in.CIDR + + out.Except = make([]string, len(in.Except)) + copy(out.Except, in.Except) + return nil +} + +func Convert_networking_IPBlock_To_v1beta1_IPBlock(in *networking.IPBlock, out *extensionsv1beta1.IPBlock, s conversion.Scope) error { + out.CIDR = in.CIDR + + out.Except = make([]string, len(in.Except)) + copy(out.Except, in.Except) return nil } diff --git a/pkg/apis/extensions/v1beta1/zz_generated.conversion.go b/pkg/apis/extensions/v1beta1/zz_generated.conversion.go index e9b2aa64a3f03..8b7a7b100739a 100644 --- a/pkg/apis/extensions/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/extensions/v1beta1/zz_generated.conversion.go @@ -198,11 +198,7 @@ func Convert_v1beta1_CustomMetricCurrentStatusList_To_extensions_CustomMetricCur } func autoConvert_extensions_CustomMetricCurrentStatusList_To_v1beta1_CustomMetricCurrentStatusList(in *extensions.CustomMetricCurrentStatusList, out *v1beta1.CustomMetricCurrentStatusList, s conversion.Scope) error { - if in.Items == nil { - out.Items = make([]v1beta1.CustomMetricCurrentStatus, 0) - } else { - out.Items = *(*[]v1beta1.CustomMetricCurrentStatus)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.CustomMetricCurrentStatus)(unsafe.Pointer(&in.Items)) return nil } @@ -244,11 +240,7 @@ func Convert_v1beta1_CustomMetricTargetList_To_extensions_CustomMetricTargetList } func autoConvert_extensions_CustomMetricTargetList_To_v1beta1_CustomMetricTargetList(in *extensions.CustomMetricTargetList, out *v1beta1.CustomMetricTargetList, s conversion.Scope) error { - if in.Items == nil { - out.Items = make([]v1beta1.CustomMetricTarget, 0) - } else { - out.Items = *(*[]v1beta1.CustomMetricTarget)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.CustomMetricTarget)(unsafe.Pointer(&in.Items)) return nil } @@ -321,7 +313,7 @@ func autoConvert_extensions_DaemonSetList_To_v1beta1_DaemonSetList(in *extension } } } else { - out.Items = make([]v1beta1.DaemonSet, 0) + out.Items = nil } return nil } @@ -378,7 +370,7 @@ func autoConvert_v1beta1_DaemonSetStatus_To_extensions_DaemonSetStatus(in *v1bet out.UpdatedNumberScheduled = in.UpdatedNumberScheduled out.NumberAvailable = in.NumberAvailable out.NumberUnavailable = in.NumberUnavailable - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -396,7 +388,7 @@ func autoConvert_extensions_DaemonSetStatus_To_v1beta1_DaemonSetStatus(in *exten out.UpdatedNumberScheduled = in.UpdatedNumberScheduled out.NumberAvailable = in.NumberAvailable out.NumberUnavailable = in.NumberUnavailable - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -537,7 +529,7 @@ func autoConvert_extensions_DeploymentList_To_v1beta1_DeploymentList(in *extensi } } } else { - out.Items = make([]v1beta1.Deployment, 0) + out.Items = nil } return nil } @@ -621,7 +613,7 @@ func autoConvert_v1beta1_DeploymentStatus_To_extensions_DeploymentStatus(in *v1b out.AvailableReplicas = in.AvailableReplicas out.UnavailableReplicas = in.UnavailableReplicas out.Conditions = *(*[]extensions.DeploymentCondition)(unsafe.Pointer(&in.Conditions)) - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -638,7 +630,7 @@ func autoConvert_extensions_DeploymentStatus_To_v1beta1_DeploymentStatus(in *ext out.AvailableReplicas = in.AvailableReplicas out.UnavailableReplicas = in.UnavailableReplicas out.Conditions = *(*[]v1beta1.DeploymentCondition)(unsafe.Pointer(&in.Conditions)) - out.CollisionCount = (*int64)(unsafe.Pointer(in.CollisionCount)) + out.CollisionCount = (*int32)(unsafe.Pointer(in.CollisionCount)) return nil } @@ -734,11 +726,7 @@ func Convert_v1beta1_HTTPIngressRuleValue_To_extensions_HTTPIngressRuleValue(in } func autoConvert_extensions_HTTPIngressRuleValue_To_v1beta1_HTTPIngressRuleValue(in *extensions.HTTPIngressRuleValue, out *v1beta1.HTTPIngressRuleValue, s conversion.Scope) error { - if in.Paths == nil { - out.Paths = make([]v1beta1.HTTPIngressPath, 0) - } else { - out.Paths = *(*[]v1beta1.HTTPIngressPath)(unsafe.Pointer(&in.Paths)) - } + out.Paths = *(*[]v1beta1.HTTPIngressPath)(unsafe.Pointer(&in.Paths)) return nil } @@ -836,11 +824,7 @@ func Convert_v1beta1_IngressList_To_extensions_IngressList(in *v1beta1.IngressLi func autoConvert_extensions_IngressList_To_v1beta1_IngressList(in *extensions.IngressList, out *v1beta1.IngressList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.Ingress, 0) - } else { - out.Items = *(*[]v1beta1.Ingress)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.Ingress)(unsafe.Pointer(&in.Items)) return nil } @@ -1025,7 +1009,7 @@ func autoConvert_extensions_PodSecurityPolicyList_To_v1beta1_PodSecurityPolicyLi } } } else { - out.Items = make([]v1beta1.PodSecurityPolicy, 0) + out.Items = nil } return nil } @@ -1208,7 +1192,7 @@ func autoConvert_extensions_ReplicaSetList_To_v1beta1_ReplicaSetList(in *extensi } } } else { - out.Items = make([]v1beta1.ReplicaSet, 0) + out.Items = nil } return nil } @@ -1522,11 +1506,7 @@ func Convert_v1beta1_ThirdPartyResourceDataList_To_extensions_ThirdPartyResource func autoConvert_extensions_ThirdPartyResourceDataList_To_v1beta1_ThirdPartyResourceDataList(in *extensions.ThirdPartyResourceDataList, out *v1beta1.ThirdPartyResourceDataList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.ThirdPartyResourceData, 0) - } else { - out.Items = *(*[]v1beta1.ThirdPartyResourceData)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.ThirdPartyResourceData)(unsafe.Pointer(&in.Items)) return nil } @@ -1548,11 +1528,7 @@ func Convert_v1beta1_ThirdPartyResourceList_To_extensions_ThirdPartyResourceList func autoConvert_extensions_ThirdPartyResourceList_To_v1beta1_ThirdPartyResourceList(in *extensions.ThirdPartyResourceList, out *v1beta1.ThirdPartyResourceList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.ThirdPartyResource, 0) - } else { - out.Items = *(*[]v1beta1.ThirdPartyResource)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.ThirdPartyResource)(unsafe.Pointer(&in.Items)) return nil } diff --git a/pkg/apis/extensions/v1beta1/zz_generated.defaults.go b/pkg/apis/extensions/v1beta1/zz_generated.defaults.go index 7c4b382fedb03..ba0c3716a2354 100644 --- a/pkg/apis/extensions/v1beta1/zz_generated.defaults.go +++ b/pkg/apis/extensions/v1beta1/zz_generated.defaults.go @@ -47,6 +47,9 @@ func SetObjectDefaults_DaemonSet(in *v1beta1.DaemonSet) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -188,6 +191,9 @@ func SetObjectDefaults_Deployment(in *v1beta1.Deployment) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } @@ -340,6 +346,9 @@ func SetObjectDefaults_ReplicaSet(in *v1beta1.ReplicaSet) { for i := range in.Spec.Template.Spec.Volumes { a := &in.Spec.Template.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } diff --git a/pkg/apis/extensions/validation/validation.go b/pkg/apis/extensions/validation/validation.go index d145fd421fb50..6696d0b195e0b 100644 --- a/pkg/apis/extensions/validation/validation.go +++ b/pkg/apis/extensions/validation/validation.go @@ -95,7 +95,7 @@ func ValidateDaemonSetStatusUpdate(ds, oldDS *extensions.DaemonSet) field.ErrorL allErrs := apivalidation.ValidateObjectMetaUpdate(&ds.ObjectMeta, &oldDS.ObjectMeta, field.NewPath("metadata")) allErrs = append(allErrs, validateDaemonSetStatus(&ds.Status, field.NewPath("status"))...) if isDecremented(ds.Status.CollisionCount, oldDS.Status.CollisionCount) { - value := int64(0) + value := int32(0) if ds.Status.CollisionCount != nil { value = *ds.Status.CollisionCount } @@ -311,7 +311,7 @@ func ValidateDeploymentStatus(status *extensions.DeploymentStatus, fldPath *fiel allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(status.AvailableReplicas), fldPath.Child("availableReplicas"))...) allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(status.UnavailableReplicas), fldPath.Child("unavailableReplicas"))...) if status.CollisionCount != nil { - allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(*status.CollisionCount, fldPath.Child("collisionCount"))...) + allErrs = append(allErrs, apivalidation.ValidateNonnegativeField(int64(*status.CollisionCount), fldPath.Child("collisionCount"))...) } msg := "cannot be greater than status.replicas" if status.UpdatedReplicas > status.Replicas { @@ -342,7 +342,7 @@ func ValidateDeploymentStatusUpdate(update, old *extensions.Deployment) field.Er fldPath := field.NewPath("status") allErrs = append(allErrs, ValidateDeploymentStatus(&update.Status, fldPath)...) if isDecremented(update.Status.CollisionCount, old.Status.CollisionCount) { - value := int64(0) + value := int32(0) if update.Status.CollisionCount != nil { value = *update.Status.CollisionCount } @@ -352,7 +352,7 @@ func ValidateDeploymentStatusUpdate(update, old *extensions.Deployment) field.Er } // TODO: Move in "k8s.io/kubernetes/pkg/api/validation" -func isDecremented(update, old *int64) bool { +func isDecremented(update, old *int32) bool { if update == nil && old != nil { return true } diff --git a/pkg/apis/extensions/validation/validation_test.go b/pkg/apis/extensions/validation/validation_test.go index 5e6c6df8ee219..0afd286dcca0e 100644 --- a/pkg/apis/extensions/validation/validation_test.go +++ b/pkg/apis/extensions/validation/validation_test.go @@ -1238,6 +1238,7 @@ func int64p(i int) *int64 { } func TestValidateDeploymentStatus(t *testing.T) { + collisionCount := int32(-3) tests := []struct { name string @@ -1246,7 +1247,7 @@ func TestValidateDeploymentStatus(t *testing.T) { readyReplicas int32 availableReplicas int32 observedGeneration int64 - collisionCount *int64 + collisionCount *int32 expectedErr bool }{ @@ -1347,7 +1348,7 @@ func TestValidateDeploymentStatus(t *testing.T) { name: "invalid collisionCount", replicas: 3, observedGeneration: 1, - collisionCount: int64p(-3), + collisionCount: &collisionCount, expectedErr: true, }, } @@ -1371,6 +1372,8 @@ func TestValidateDeploymentStatus(t *testing.T) { } func TestValidateDeploymentStatusUpdate(t *testing.T) { + collisionCount := int32(1) + otherCollisionCount := int32(2) tests := []struct { name string @@ -1384,24 +1387,24 @@ func TestValidateDeploymentStatusUpdate(t *testing.T) { CollisionCount: nil, }, to: extensions.DeploymentStatus{ - CollisionCount: int64p(1), + CollisionCount: &collisionCount, }, expectedErr: false, }, { name: "stable: valid update", from: extensions.DeploymentStatus{ - CollisionCount: int64p(1), + CollisionCount: &collisionCount, }, to: extensions.DeploymentStatus{ - CollisionCount: int64p(1), + CollisionCount: &collisionCount, }, expectedErr: false, }, { name: "unset: invalid update", from: extensions.DeploymentStatus{ - CollisionCount: int64p(1), + CollisionCount: &collisionCount, }, to: extensions.DeploymentStatus{ CollisionCount: nil, @@ -1411,10 +1414,10 @@ func TestValidateDeploymentStatusUpdate(t *testing.T) { { name: "decrease: invalid update", from: extensions.DeploymentStatus{ - CollisionCount: int64p(2), + CollisionCount: &otherCollisionCount, }, to: extensions.DeploymentStatus{ - CollisionCount: int64p(1), + CollisionCount: &collisionCount, }, expectedErr: true, }, diff --git a/pkg/apis/extensions/zz_generated.deepcopy.go b/pkg/apis/extensions/zz_generated.deepcopy.go index d5af9371dff80..69414fa715642 100644 --- a/pkg/apis/extensions/zz_generated.deepcopy.go +++ b/pkg/apis/extensions/zz_generated.deepcopy.go @@ -456,7 +456,7 @@ func (in *DaemonSetStatus) DeepCopyInto(out *DaemonSetStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } @@ -682,7 +682,7 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } diff --git a/pkg/apis/networking/fuzzer/BUILD b/pkg/apis/networking/fuzzer/BUILD index dd0561eec8f8f..b5819de095d1e 100644 --- a/pkg/apis/networking/fuzzer/BUILD +++ b/pkg/apis/networking/fuzzer/BUILD @@ -8,7 +8,11 @@ load( go_library( name = "go_default_library", srcs = ["fuzzer.go"], - deps = ["//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library"], + deps = [ + "//pkg/apis/networking:go_default_library", + "//vendor/github.com/google/gofuzz:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", + ], ) filegroup( diff --git a/pkg/apis/networking/fuzzer/fuzzer.go b/pkg/apis/networking/fuzzer/fuzzer.go index 6e142826d3be5..1325b9d336cef 100644 --- a/pkg/apis/networking/fuzzer/fuzzer.go +++ b/pkg/apis/networking/fuzzer/fuzzer.go @@ -17,10 +17,24 @@ limitations under the License. package fuzzer import ( + fuzz "github.com/google/gofuzz" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/kubernetes/pkg/apis/networking" ) // Funcs returns the fuzzer functions for the networking api group. var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { - return []interface{}{} + return []interface{}{ + func(np *networking.NetworkPolicyPeer, c fuzz.Continue) { + c.FuzzNoCustom(np) // fuzz self without calling this function again + // TODO: Implement a fuzzer to generate valid keys, values and operators for + // selector requirements. + if np.IPBlock != nil { + np.IPBlock = &networking.IPBlock{ + CIDR: "192.168.1.0/24", + Except: []string{"192.168.1.1/24", "192.168.1.2/24"}, + } + } + }, + } } diff --git a/pkg/apis/networking/types.go b/pkg/apis/networking/types.go index a422b05d5c010..74d818cd77f4d 100644 --- a/pkg/apis/networking/types.go +++ b/pkg/apis/networking/types.go @@ -90,6 +90,20 @@ type NetworkPolicyPort struct { Port *intstr.IntOrString } +// IPBlock describes a particular CIDR (Ex. "192.168.1.1/24") that is allowed to the pods +// matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should +// not be included within this rule. +type IPBlock struct { + // CIDR is a string representing the IP Block + // Valid examples are "192.168.1.1/24" + CIDR string + // Except is a slice of CIDRs that should not be included within an IP Block + // Valid examples are "192.168.1.1/24" + // Except values will be rejected if they are outside the CIDR range + // +optional + Except []string +} + // NetworkPolicyPeer describes a peer to allow traffic from. Exactly one of its fields // must be specified. type NetworkPolicyPeer struct { @@ -104,6 +118,10 @@ type NetworkPolicyPeer struct { // selector semantics. If present but empty, this selector selects all namespaces. // +optional NamespaceSelector *metav1.LabelSelector + + // IPBlock defines policy on a particular IPBlock + // +optional + IPBlock *IPBlock } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/networking/v1/zz_generated.conversion.go b/pkg/apis/networking/v1/zz_generated.conversion.go index d746e7ffaef40..bc0f0fb2aaf5f 100644 --- a/pkg/apis/networking/v1/zz_generated.conversion.go +++ b/pkg/apis/networking/v1/zz_generated.conversion.go @@ -40,6 +40,8 @@ func init() { // Public to allow building arbitrary schemes. func RegisterConversions(scheme *runtime.Scheme) error { return scheme.AddGeneratedConversionFuncs( + Convert_v1_IPBlock_To_networking_IPBlock, + Convert_networking_IPBlock_To_v1_IPBlock, Convert_v1_NetworkPolicy_To_networking_NetworkPolicy, Convert_networking_NetworkPolicy_To_v1_NetworkPolicy, Convert_v1_NetworkPolicyIngressRule_To_networking_NetworkPolicyIngressRule, @@ -55,6 +57,28 @@ func RegisterConversions(scheme *runtime.Scheme) error { ) } +func autoConvert_v1_IPBlock_To_networking_IPBlock(in *v1.IPBlock, out *networking.IPBlock, s conversion.Scope) error { + out.CIDR = in.CIDR + out.Except = *(*[]string)(unsafe.Pointer(&in.Except)) + return nil +} + +// Convert_v1_IPBlock_To_networking_IPBlock is an autogenerated conversion function. +func Convert_v1_IPBlock_To_networking_IPBlock(in *v1.IPBlock, out *networking.IPBlock, s conversion.Scope) error { + return autoConvert_v1_IPBlock_To_networking_IPBlock(in, out, s) +} + +func autoConvert_networking_IPBlock_To_v1_IPBlock(in *networking.IPBlock, out *v1.IPBlock, s conversion.Scope) error { + out.CIDR = in.CIDR + out.Except = *(*[]string)(unsafe.Pointer(&in.Except)) + return nil +} + +// Convert_networking_IPBlock_To_v1_IPBlock is an autogenerated conversion function. +func Convert_networking_IPBlock_To_v1_IPBlock(in *networking.IPBlock, out *v1.IPBlock, s conversion.Scope) error { + return autoConvert_networking_IPBlock_To_v1_IPBlock(in, out, s) +} + func autoConvert_v1_NetworkPolicy_To_networking_NetworkPolicy(in *v1.NetworkPolicy, out *networking.NetworkPolicy, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1_NetworkPolicySpec_To_networking_NetworkPolicySpec(&in.Spec, &out.Spec, s); err != nil { @@ -116,11 +140,7 @@ func Convert_v1_NetworkPolicyList_To_networking_NetworkPolicyList(in *v1.Network func autoConvert_networking_NetworkPolicyList_To_v1_NetworkPolicyList(in *networking.NetworkPolicyList, out *v1.NetworkPolicyList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.NetworkPolicy, 0) - } else { - out.Items = *(*[]v1.NetworkPolicy)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.NetworkPolicy)(unsafe.Pointer(&in.Items)) return nil } @@ -132,6 +152,7 @@ func Convert_networking_NetworkPolicyList_To_v1_NetworkPolicyList(in *networking func autoConvert_v1_NetworkPolicyPeer_To_networking_NetworkPolicyPeer(in *v1.NetworkPolicyPeer, out *networking.NetworkPolicyPeer, s conversion.Scope) error { out.PodSelector = (*meta_v1.LabelSelector)(unsafe.Pointer(in.PodSelector)) out.NamespaceSelector = (*meta_v1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) + out.IPBlock = (*networking.IPBlock)(unsafe.Pointer(in.IPBlock)) return nil } @@ -143,6 +164,7 @@ func Convert_v1_NetworkPolicyPeer_To_networking_NetworkPolicyPeer(in *v1.Network func autoConvert_networking_NetworkPolicyPeer_To_v1_NetworkPolicyPeer(in *networking.NetworkPolicyPeer, out *v1.NetworkPolicyPeer, s conversion.Scope) error { out.PodSelector = (*meta_v1.LabelSelector)(unsafe.Pointer(in.PodSelector)) out.NamespaceSelector = (*meta_v1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) + out.IPBlock = (*v1.IPBlock)(unsafe.Pointer(in.IPBlock)) return nil } diff --git a/pkg/apis/networking/validation/validation.go b/pkg/apis/networking/validation/validation.go index 12272d5028936..780cc652905e4 100644 --- a/pkg/apis/networking/validation/validation.go +++ b/pkg/apis/networking/validation/validation.go @@ -17,6 +17,8 @@ limitations under the License. package validation import ( + "net" + unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/validation" @@ -68,7 +70,10 @@ func ValidateNetworkPolicySpec(spec *networking.NetworkPolicySpec, fldPath *fiel numFroms++ allErrs = append(allErrs, unversionedvalidation.ValidateLabelSelector(from.NamespaceSelector, fromPath.Child("namespaceSelector"))...) } - + if from.IPBlock != nil { + numFroms++ + allErrs = append(allErrs, ValidateIPBlock(from.IPBlock, fromPath.Child("ipBlock"))...) + } if numFroms == 0 { allErrs = append(allErrs, field.Required(fromPath, "must specify a from type")) } else if numFroms > 1 { @@ -93,3 +98,39 @@ func ValidateNetworkPolicyUpdate(update, old *networking.NetworkPolicy) field.Er allErrs = append(allErrs, ValidateNetworkPolicySpec(&update.Spec, field.NewPath("spec"))...) return allErrs } + +// ValidateIPBlock validates a cidr and the except fields of an IpBlock NetworkPolicyPeer +func ValidateIPBlock(ipb *networking.IPBlock, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if len(ipb.CIDR) == 0 || ipb.CIDR == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("cidr"), "")) + return allErrs + } + cidrIPNet, err := validateCIDR(ipb.CIDR) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("cidr"), ipb.CIDR, "not a valid CIDR")) + return allErrs + } + exceptCIDR := ipb.Except + for i, exceptIP := range exceptCIDR { + exceptPath := fldPath.Child("except").Index(i) + exceptCIDR, err := validateCIDR(exceptIP) + if err != nil { + allErrs = append(allErrs, field.Invalid(exceptPath, exceptIP, "not a valid CIDR")) + return allErrs + } + if !cidrIPNet.Contains(exceptCIDR.IP) { + allErrs = append(allErrs, field.Invalid(exceptPath, exceptCIDR.IP, "not within CIDR range")) + } + } + return allErrs +} + +// validateCIDR validates whether a CIDR matches the conventions expected by net.ParseCIDR +func validateCIDR(cidr string) (*net.IPNet, error) { + _, net, err := net.ParseCIDR(cidr) + if err != nil { + return nil, err + } + return net, nil +} diff --git a/pkg/apis/networking/validation/validation_test.go b/pkg/apis/networking/validation/validation_test.go index 154d783bf153c..0be1f0998c576 100644 --- a/pkg/apis/networking/validation/validation_test.go +++ b/pkg/apis/networking/validation/validation_test.go @@ -122,6 +122,26 @@ func TestValidateNetworkPolicy(t *testing.T) { }, }, }, + { + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, + Spec: networking.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + Ingress: []networking.NetworkPolicyIngressRule{ + { + From: []networking.NetworkPolicyPeer{ + { + IPBlock: &networking.IPBlock{ + CIDR: "192.168.0.0/16", + Except: []string{"192.168.3.0/24", "192.168.4.0/24"}, + }, + }, + }, + }, + }, + }, + }, } // Success cases are expected to pass validation. @@ -256,6 +276,83 @@ func TestValidateNetworkPolicy(t *testing.T) { }, }, }, + "missing cidr field": { + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, + Spec: networking.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{}, + Ingress: []networking.NetworkPolicyIngressRule{ + { + From: []networking.NetworkPolicyPeer{ + { + IPBlock: &networking.IPBlock{ + Except: []string{"192.168.8.0/24", "192.168.9.0/24"}, + }, + }, + }, + }, + }, + }, + }, + "invalid cidr format": { + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, + Spec: networking.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + Ingress: []networking.NetworkPolicyIngressRule{ + { + From: []networking.NetworkPolicyPeer{ + { + IPBlock: &networking.IPBlock{ + CIDR: "192.168.5.6", + Except: []string{"192.168.1.0/24", "192.168.2.0/24"}, + }, + }, + }, + }, + }, + }, + }, + "except field is an empty string": { + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, + Spec: networking.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + Ingress: []networking.NetworkPolicyIngressRule{ + { + From: []networking.NetworkPolicyPeer{ + { + IPBlock: &networking.IPBlock{ + CIDR: "192.168.8.0/24", + Except: []string{"", " "}, + }, + }, + }, + }, + }, + }, + }, + "except IP is outside of CIDR range": { + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, + Spec: networking.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + Ingress: []networking.NetworkPolicyIngressRule{ + { + From: []networking.NetworkPolicyPeer{ + { + IPBlock: &networking.IPBlock{ + CIDR: "192.168.8.0/24", + Except: []string{"192.168.9.1/24"}, + }, + }, + }, + }, + }, + }, + }, } // Error cases are not expected to pass validation. diff --git a/pkg/apis/networking/zz_generated.deepcopy.go b/pkg/apis/networking/zz_generated.deepcopy.go index d68f23467bcf4..6cc1fa4d3d60d 100644 --- a/pkg/apis/networking/zz_generated.deepcopy.go +++ b/pkg/apis/networking/zz_generated.deepcopy.go @@ -39,6 +39,10 @@ func init() { // Deprecated: deepcopy registration will go away when static deepcopy is fully implemented. func RegisterDeepCopies(scheme *runtime.Scheme) error { return scheme.AddGeneratedDeepCopyFuncs( + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*IPBlock).DeepCopyInto(out.(*IPBlock)) + return nil + }, InType: reflect.TypeOf(&IPBlock{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*NetworkPolicy).DeepCopyInto(out.(*NetworkPolicy)) return nil @@ -66,6 +70,27 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { ) } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPBlock) DeepCopyInto(out *IPBlock) { + *out = *in + if in.Except != nil { + in, out := &in.Except, &out.Except + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPBlock. +func (in *IPBlock) DeepCopy() *IPBlock { + if in == nil { + return nil + } + out := new(IPBlock) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkPolicy) DeepCopyInto(out *NetworkPolicy) { *out = *in @@ -179,6 +204,15 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { (*in).DeepCopyInto(*out) } } + if in.IPBlock != nil { + in, out := &in.IPBlock, &out.IPBlock + if *in == nil { + *out = nil + } else { + *out = new(IPBlock) + (*in).DeepCopyInto(*out) + } + } return } diff --git a/pkg/apis/policy/v1beta1/zz_generated.conversion.go b/pkg/apis/policy/v1beta1/zz_generated.conversion.go index 29b2bef6ea00d..aa07a7bd7d5a1 100644 --- a/pkg/apis/policy/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/policy/v1beta1/zz_generated.conversion.go @@ -118,11 +118,7 @@ func Convert_v1beta1_PodDisruptionBudgetList_To_policy_PodDisruptionBudgetList(i func autoConvert_policy_PodDisruptionBudgetList_To_v1beta1_PodDisruptionBudgetList(in *policy.PodDisruptionBudgetList, out *v1beta1.PodDisruptionBudgetList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.PodDisruptionBudget, 0) - } else { - out.Items = *(*[]v1beta1.PodDisruptionBudget)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.PodDisruptionBudget)(unsafe.Pointer(&in.Items)) return nil } diff --git a/pkg/apis/rbac/OWNERS b/pkg/apis/rbac/OWNERS index a35477b920d78..53710111cde18 100755 --- a/pkg/apis/rbac/OWNERS +++ b/pkg/apis/rbac/OWNERS @@ -15,3 +15,4 @@ reviewers: - lixiaobing10051267 - jianhuiz - liggitt +- enj diff --git a/pkg/apis/rbac/v1/zz_generated.conversion.go b/pkg/apis/rbac/v1/zz_generated.conversion.go index dab304a212784..e5e668811e21a 100644 --- a/pkg/apis/rbac/v1/zz_generated.conversion.go +++ b/pkg/apis/rbac/v1/zz_generated.conversion.go @@ -74,11 +74,7 @@ func Convert_v1_ClusterRole_To_rbac_ClusterRole(in *v1.ClusterRole, out *rbac.Cl func autoConvert_rbac_ClusterRole_To_v1_ClusterRole(in *rbac.ClusterRole, out *v1.ClusterRole, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Rules == nil { - out.Rules = make([]v1.PolicyRule, 0) - } else { - out.Rules = *(*[]v1.PolicyRule)(unsafe.Pointer(&in.Rules)) - } + out.Rules = *(*[]v1.PolicyRule)(unsafe.Pointer(&in.Rules)) return nil } @@ -103,11 +99,7 @@ func Convert_v1_ClusterRoleBinding_To_rbac_ClusterRoleBinding(in *v1.ClusterRole func autoConvert_rbac_ClusterRoleBinding_To_v1_ClusterRoleBinding(in *rbac.ClusterRoleBinding, out *v1.ClusterRoleBinding, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Subjects == nil { - out.Subjects = make([]v1.Subject, 0) - } else { - out.Subjects = *(*[]v1.Subject)(unsafe.Pointer(&in.Subjects)) - } + out.Subjects = *(*[]v1.Subject)(unsafe.Pointer(&in.Subjects)) if err := Convert_rbac_RoleRef_To_v1_RoleRef(&in.RoleRef, &out.RoleRef, s); err != nil { return err } @@ -132,11 +124,7 @@ func Convert_v1_ClusterRoleBindingList_To_rbac_ClusterRoleBindingList(in *v1.Clu func autoConvert_rbac_ClusterRoleBindingList_To_v1_ClusterRoleBindingList(in *rbac.ClusterRoleBindingList, out *v1.ClusterRoleBindingList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.ClusterRoleBinding, 0) - } else { - out.Items = *(*[]v1.ClusterRoleBinding)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.ClusterRoleBinding)(unsafe.Pointer(&in.Items)) return nil } @@ -158,11 +146,7 @@ func Convert_v1_ClusterRoleList_To_rbac_ClusterRoleList(in *v1.ClusterRoleList, func autoConvert_rbac_ClusterRoleList_To_v1_ClusterRoleList(in *rbac.ClusterRoleList, out *v1.ClusterRoleList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.ClusterRole, 0) - } else { - out.Items = *(*[]v1.ClusterRole)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.ClusterRole)(unsafe.Pointer(&in.Items)) return nil } @@ -186,11 +170,7 @@ func Convert_v1_PolicyRule_To_rbac_PolicyRule(in *v1.PolicyRule, out *rbac.Polic } func autoConvert_rbac_PolicyRule_To_v1_PolicyRule(in *rbac.PolicyRule, out *v1.PolicyRule, s conversion.Scope) error { - if in.Verbs == nil { - out.Verbs = make([]string, 0) - } else { - out.Verbs = *(*[]string)(unsafe.Pointer(&in.Verbs)) - } + out.Verbs = *(*[]string)(unsafe.Pointer(&in.Verbs)) out.APIGroups = *(*[]string)(unsafe.Pointer(&in.APIGroups)) out.Resources = *(*[]string)(unsafe.Pointer(&in.Resources)) out.ResourceNames = *(*[]string)(unsafe.Pointer(&in.ResourceNames)) @@ -216,11 +196,7 @@ func Convert_v1_Role_To_rbac_Role(in *v1.Role, out *rbac.Role, s conversion.Scop func autoConvert_rbac_Role_To_v1_Role(in *rbac.Role, out *v1.Role, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Rules == nil { - out.Rules = make([]v1.PolicyRule, 0) - } else { - out.Rules = *(*[]v1.PolicyRule)(unsafe.Pointer(&in.Rules)) - } + out.Rules = *(*[]v1.PolicyRule)(unsafe.Pointer(&in.Rules)) return nil } @@ -245,11 +221,7 @@ func Convert_v1_RoleBinding_To_rbac_RoleBinding(in *v1.RoleBinding, out *rbac.Ro func autoConvert_rbac_RoleBinding_To_v1_RoleBinding(in *rbac.RoleBinding, out *v1.RoleBinding, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Subjects == nil { - out.Subjects = make([]v1.Subject, 0) - } else { - out.Subjects = *(*[]v1.Subject)(unsafe.Pointer(&in.Subjects)) - } + out.Subjects = *(*[]v1.Subject)(unsafe.Pointer(&in.Subjects)) if err := Convert_rbac_RoleRef_To_v1_RoleRef(&in.RoleRef, &out.RoleRef, s); err != nil { return err } @@ -274,11 +246,7 @@ func Convert_v1_RoleBindingList_To_rbac_RoleBindingList(in *v1.RoleBindingList, func autoConvert_rbac_RoleBindingList_To_v1_RoleBindingList(in *rbac.RoleBindingList, out *v1.RoleBindingList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.RoleBinding, 0) - } else { - out.Items = *(*[]v1.RoleBinding)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.RoleBinding)(unsafe.Pointer(&in.Items)) return nil } @@ -300,11 +268,7 @@ func Convert_v1_RoleList_To_rbac_RoleList(in *v1.RoleList, out *rbac.RoleList, s func autoConvert_rbac_RoleList_To_v1_RoleList(in *rbac.RoleList, out *v1.RoleList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.Role, 0) - } else { - out.Items = *(*[]v1.Role)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.Role)(unsafe.Pointer(&in.Items)) return nil } diff --git a/pkg/apis/rbac/v1alpha1/zz_generated.conversion.go b/pkg/apis/rbac/v1alpha1/zz_generated.conversion.go index e59d4144527bc..4c52c5a783919 100644 --- a/pkg/apis/rbac/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/rbac/v1alpha1/zz_generated.conversion.go @@ -74,11 +74,7 @@ func Convert_v1alpha1_ClusterRole_To_rbac_ClusterRole(in *v1alpha1.ClusterRole, func autoConvert_rbac_ClusterRole_To_v1alpha1_ClusterRole(in *rbac.ClusterRole, out *v1alpha1.ClusterRole, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Rules == nil { - out.Rules = make([]v1alpha1.PolicyRule, 0) - } else { - out.Rules = *(*[]v1alpha1.PolicyRule)(unsafe.Pointer(&in.Rules)) - } + out.Rules = *(*[]v1alpha1.PolicyRule)(unsafe.Pointer(&in.Rules)) return nil } @@ -122,7 +118,7 @@ func autoConvert_rbac_ClusterRoleBinding_To_v1alpha1_ClusterRoleBinding(in *rbac } } } else { - out.Subjects = make([]v1alpha1.Subject, 0) + out.Subjects = nil } if err := Convert_rbac_RoleRef_To_v1alpha1_RoleRef(&in.RoleRef, &out.RoleRef, s); err != nil { return err @@ -167,7 +163,7 @@ func autoConvert_rbac_ClusterRoleBindingList_To_v1alpha1_ClusterRoleBindingList( } } } else { - out.Items = make([]v1alpha1.ClusterRoleBinding, 0) + out.Items = nil } return nil } @@ -190,11 +186,7 @@ func Convert_v1alpha1_ClusterRoleList_To_rbac_ClusterRoleList(in *v1alpha1.Clust func autoConvert_rbac_ClusterRoleList_To_v1alpha1_ClusterRoleList(in *rbac.ClusterRoleList, out *v1alpha1.ClusterRoleList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1alpha1.ClusterRole, 0) - } else { - out.Items = *(*[]v1alpha1.ClusterRole)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1alpha1.ClusterRole)(unsafe.Pointer(&in.Items)) return nil } @@ -218,11 +210,7 @@ func Convert_v1alpha1_PolicyRule_To_rbac_PolicyRule(in *v1alpha1.PolicyRule, out } func autoConvert_rbac_PolicyRule_To_v1alpha1_PolicyRule(in *rbac.PolicyRule, out *v1alpha1.PolicyRule, s conversion.Scope) error { - if in.Verbs == nil { - out.Verbs = make([]string, 0) - } else { - out.Verbs = *(*[]string)(unsafe.Pointer(&in.Verbs)) - } + out.Verbs = *(*[]string)(unsafe.Pointer(&in.Verbs)) out.APIGroups = *(*[]string)(unsafe.Pointer(&in.APIGroups)) out.Resources = *(*[]string)(unsafe.Pointer(&in.Resources)) out.ResourceNames = *(*[]string)(unsafe.Pointer(&in.ResourceNames)) @@ -248,11 +236,7 @@ func Convert_v1alpha1_Role_To_rbac_Role(in *v1alpha1.Role, out *rbac.Role, s con func autoConvert_rbac_Role_To_v1alpha1_Role(in *rbac.Role, out *v1alpha1.Role, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Rules == nil { - out.Rules = make([]v1alpha1.PolicyRule, 0) - } else { - out.Rules = *(*[]v1alpha1.PolicyRule)(unsafe.Pointer(&in.Rules)) - } + out.Rules = *(*[]v1alpha1.PolicyRule)(unsafe.Pointer(&in.Rules)) return nil } @@ -296,7 +280,7 @@ func autoConvert_rbac_RoleBinding_To_v1alpha1_RoleBinding(in *rbac.RoleBinding, } } } else { - out.Subjects = make([]v1alpha1.Subject, 0) + out.Subjects = nil } if err := Convert_rbac_RoleRef_To_v1alpha1_RoleRef(&in.RoleRef, &out.RoleRef, s); err != nil { return err @@ -341,7 +325,7 @@ func autoConvert_rbac_RoleBindingList_To_v1alpha1_RoleBindingList(in *rbac.RoleB } } } else { - out.Items = make([]v1alpha1.RoleBinding, 0) + out.Items = nil } return nil } @@ -364,11 +348,7 @@ func Convert_v1alpha1_RoleList_To_rbac_RoleList(in *v1alpha1.RoleList, out *rbac func autoConvert_rbac_RoleList_To_v1alpha1_RoleList(in *rbac.RoleList, out *v1alpha1.RoleList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1alpha1.Role, 0) - } else { - out.Items = *(*[]v1alpha1.Role)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1alpha1.Role)(unsafe.Pointer(&in.Items)) return nil } diff --git a/pkg/apis/rbac/v1beta1/zz_generated.conversion.go b/pkg/apis/rbac/v1beta1/zz_generated.conversion.go index ade9a69bf003c..d06b9265aa1a9 100644 --- a/pkg/apis/rbac/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/rbac/v1beta1/zz_generated.conversion.go @@ -74,11 +74,7 @@ func Convert_v1beta1_ClusterRole_To_rbac_ClusterRole(in *v1beta1.ClusterRole, ou func autoConvert_rbac_ClusterRole_To_v1beta1_ClusterRole(in *rbac.ClusterRole, out *v1beta1.ClusterRole, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Rules == nil { - out.Rules = make([]v1beta1.PolicyRule, 0) - } else { - out.Rules = *(*[]v1beta1.PolicyRule)(unsafe.Pointer(&in.Rules)) - } + out.Rules = *(*[]v1beta1.PolicyRule)(unsafe.Pointer(&in.Rules)) return nil } @@ -103,11 +99,7 @@ func Convert_v1beta1_ClusterRoleBinding_To_rbac_ClusterRoleBinding(in *v1beta1.C func autoConvert_rbac_ClusterRoleBinding_To_v1beta1_ClusterRoleBinding(in *rbac.ClusterRoleBinding, out *v1beta1.ClusterRoleBinding, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Subjects == nil { - out.Subjects = make([]v1beta1.Subject, 0) - } else { - out.Subjects = *(*[]v1beta1.Subject)(unsafe.Pointer(&in.Subjects)) - } + out.Subjects = *(*[]v1beta1.Subject)(unsafe.Pointer(&in.Subjects)) if err := Convert_rbac_RoleRef_To_v1beta1_RoleRef(&in.RoleRef, &out.RoleRef, s); err != nil { return err } @@ -132,11 +124,7 @@ func Convert_v1beta1_ClusterRoleBindingList_To_rbac_ClusterRoleBindingList(in *v func autoConvert_rbac_ClusterRoleBindingList_To_v1beta1_ClusterRoleBindingList(in *rbac.ClusterRoleBindingList, out *v1beta1.ClusterRoleBindingList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.ClusterRoleBinding, 0) - } else { - out.Items = *(*[]v1beta1.ClusterRoleBinding)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.ClusterRoleBinding)(unsafe.Pointer(&in.Items)) return nil } @@ -158,11 +146,7 @@ func Convert_v1beta1_ClusterRoleList_To_rbac_ClusterRoleList(in *v1beta1.Cluster func autoConvert_rbac_ClusterRoleList_To_v1beta1_ClusterRoleList(in *rbac.ClusterRoleList, out *v1beta1.ClusterRoleList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.ClusterRole, 0) - } else { - out.Items = *(*[]v1beta1.ClusterRole)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.ClusterRole)(unsafe.Pointer(&in.Items)) return nil } @@ -186,11 +170,7 @@ func Convert_v1beta1_PolicyRule_To_rbac_PolicyRule(in *v1beta1.PolicyRule, out * } func autoConvert_rbac_PolicyRule_To_v1beta1_PolicyRule(in *rbac.PolicyRule, out *v1beta1.PolicyRule, s conversion.Scope) error { - if in.Verbs == nil { - out.Verbs = make([]string, 0) - } else { - out.Verbs = *(*[]string)(unsafe.Pointer(&in.Verbs)) - } + out.Verbs = *(*[]string)(unsafe.Pointer(&in.Verbs)) out.APIGroups = *(*[]string)(unsafe.Pointer(&in.APIGroups)) out.Resources = *(*[]string)(unsafe.Pointer(&in.Resources)) out.ResourceNames = *(*[]string)(unsafe.Pointer(&in.ResourceNames)) @@ -216,11 +196,7 @@ func Convert_v1beta1_Role_To_rbac_Role(in *v1beta1.Role, out *rbac.Role, s conve func autoConvert_rbac_Role_To_v1beta1_Role(in *rbac.Role, out *v1beta1.Role, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Rules == nil { - out.Rules = make([]v1beta1.PolicyRule, 0) - } else { - out.Rules = *(*[]v1beta1.PolicyRule)(unsafe.Pointer(&in.Rules)) - } + out.Rules = *(*[]v1beta1.PolicyRule)(unsafe.Pointer(&in.Rules)) return nil } @@ -245,11 +221,7 @@ func Convert_v1beta1_RoleBinding_To_rbac_RoleBinding(in *v1beta1.RoleBinding, ou func autoConvert_rbac_RoleBinding_To_v1beta1_RoleBinding(in *rbac.RoleBinding, out *v1beta1.RoleBinding, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Subjects == nil { - out.Subjects = make([]v1beta1.Subject, 0) - } else { - out.Subjects = *(*[]v1beta1.Subject)(unsafe.Pointer(&in.Subjects)) - } + out.Subjects = *(*[]v1beta1.Subject)(unsafe.Pointer(&in.Subjects)) if err := Convert_rbac_RoleRef_To_v1beta1_RoleRef(&in.RoleRef, &out.RoleRef, s); err != nil { return err } @@ -274,11 +246,7 @@ func Convert_v1beta1_RoleBindingList_To_rbac_RoleBindingList(in *v1beta1.RoleBin func autoConvert_rbac_RoleBindingList_To_v1beta1_RoleBindingList(in *rbac.RoleBindingList, out *v1beta1.RoleBindingList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.RoleBinding, 0) - } else { - out.Items = *(*[]v1beta1.RoleBinding)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.RoleBinding)(unsafe.Pointer(&in.Items)) return nil } @@ -300,11 +268,7 @@ func Convert_v1beta1_RoleList_To_rbac_RoleList(in *v1beta1.RoleList, out *rbac.R func autoConvert_rbac_RoleList_To_v1beta1_RoleList(in *rbac.RoleList, out *v1beta1.RoleList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.Role, 0) - } else { - out.Items = *(*[]v1beta1.Role)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.Role)(unsafe.Pointer(&in.Items)) return nil } diff --git a/pkg/apis/scheduling/v1alpha1/zz_generated.conversion.go b/pkg/apis/scheduling/v1alpha1/zz_generated.conversion.go index c9be26ed14690..6a99b58a6e5cf 100644 --- a/pkg/apis/scheduling/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/scheduling/v1alpha1/zz_generated.conversion.go @@ -82,11 +82,7 @@ func Convert_v1alpha1_PriorityClassList_To_scheduling_PriorityClassList(in *v1al func autoConvert_scheduling_PriorityClassList_To_v1alpha1_PriorityClassList(in *scheduling.PriorityClassList, out *v1alpha1.PriorityClassList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1alpha1.PriorityClass, 0) - } else { - out.Items = *(*[]v1alpha1.PriorityClass)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1alpha1.PriorityClass)(unsafe.Pointer(&in.Items)) return nil } diff --git a/pkg/apis/settings/v1alpha1/zz_generated.conversion.go b/pkg/apis/settings/v1alpha1/zz_generated.conversion.go index dfdd7d133baee..396102ecd8433 100644 --- a/pkg/apis/settings/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/settings/v1alpha1/zz_generated.conversion.go @@ -105,7 +105,7 @@ func autoConvert_settings_PodPresetList_To_v1alpha1_PodPresetList(in *settings.P } } } else { - out.Items = make([]v1alpha1.PodPreset, 0) + out.Items = nil } return nil } diff --git a/pkg/apis/settings/v1alpha1/zz_generated.defaults.go b/pkg/apis/settings/v1alpha1/zz_generated.defaults.go index 0469b4aaa7f9f..b2fa4af7d3e8f 100644 --- a/pkg/apis/settings/v1alpha1/zz_generated.defaults.go +++ b/pkg/apis/settings/v1alpha1/zz_generated.defaults.go @@ -47,6 +47,9 @@ func SetObjectDefaults_PodPreset(in *v1alpha1.PodPreset) { for i := range in.Spec.Volumes { a := &in.Spec.Volumes[i] v1.SetDefaults_Volume(a) + if a.VolumeSource.HostPath != nil { + v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath) + } if a.VolumeSource.Secret != nil { v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret) } diff --git a/pkg/apis/storage/v1/zz_generated.conversion.go b/pkg/apis/storage/v1/zz_generated.conversion.go index 6041cff96fae2..70472b1e9e8ce 100644 --- a/pkg/apis/storage/v1/zz_generated.conversion.go +++ b/pkg/apis/storage/v1/zz_generated.conversion.go @@ -84,11 +84,7 @@ func Convert_v1_StorageClassList_To_storage_StorageClassList(in *v1.StorageClass func autoConvert_storage_StorageClassList_To_v1_StorageClassList(in *storage.StorageClassList, out *v1.StorageClassList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1.StorageClass, 0) - } else { - out.Items = *(*[]v1.StorageClass)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1.StorageClass)(unsafe.Pointer(&in.Items)) return nil } diff --git a/pkg/apis/storage/v1beta1/zz_generated.conversion.go b/pkg/apis/storage/v1beta1/zz_generated.conversion.go index 80bab19cd8942..7d259d1e32479 100644 --- a/pkg/apis/storage/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/storage/v1beta1/zz_generated.conversion.go @@ -84,11 +84,7 @@ func Convert_v1beta1_StorageClassList_To_storage_StorageClassList(in *v1beta1.St func autoConvert_storage_StorageClassList_To_v1beta1_StorageClassList(in *storage.StorageClassList, out *v1beta1.StorageClassList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]v1beta1.StorageClass, 0) - } else { - out.Items = *(*[]v1beta1.StorageClass)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]v1beta1.StorageClass)(unsafe.Pointer(&in.Items)) return nil } diff --git a/pkg/auth/OWNERS b/pkg/auth/OWNERS index 8e8b9ee3776c9..2d126eacd8c39 100644 --- a/pkg/auth/OWNERS +++ b/pkg/auth/OWNERS @@ -3,3 +3,4 @@ approvers: - liggitt - deads2k reviewers: +- enj diff --git a/pkg/bootstrap/api/BUILD b/pkg/bootstrap/api/BUILD index dfb7d6976381b..db7be057bf6c7 100644 --- a/pkg/bootstrap/api/BUILD +++ b/pkg/bootstrap/api/BUILD @@ -3,12 +3,14 @@ package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( name = "go_default_library", srcs = [ "doc.go", + "helpers.go", "types.go", ], deps = ["//vendor/k8s.io/api/core/v1:go_default_library"], @@ -26,3 +28,9 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["helpers_test.go"], + library = ":go_default_library", +) diff --git a/pkg/bootstrap/api/helpers.go b/pkg/bootstrap/api/helpers.go new file mode 100644 index 0000000000000..639d61a33c3f7 --- /dev/null +++ b/pkg/bootstrap/api/helpers.go @@ -0,0 +1,34 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "regexp" +) + +var bootstrapGroupRegexp = regexp.MustCompile(`\A` + BootstrapGroupPattern + `\z`) + +// ValidateBootstrapGroupName checks if the provided group name is a valid +// bootstrap group name. Returns nil if valid or a validation error if invalid. +// TODO(mattmoyer): this validation should migrate out to client-go (see https://github.com/kubernetes/client-go/issues/114) +func ValidateBootstrapGroupName(name string) error { + if bootstrapGroupRegexp.Match([]byte(name)) { + return nil + } + return fmt.Errorf("bootstrap group %q is invalid (must match %s)", name, BootstrapGroupPattern) +} diff --git a/pkg/bootstrap/api/helpers_test.go b/pkg/bootstrap/api/helpers_test.go new file mode 100644 index 0000000000000..177687150c5c6 --- /dev/null +++ b/pkg/bootstrap/api/helpers_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "strings" + "testing" +) + +func TestValidateBootstrapGroupName(t *testing.T) { + tests := []struct { + name string + input string + valid bool + }{ + {"valid", "system:bootstrappers:foo", true}, + {"valid nested", "system:bootstrappers:foo:bar:baz", true}, + {"valid with dashes and number", "system:bootstrappers:foo-bar-42", true}, + {"invalid uppercase", "system:bootstrappers:Foo", false}, + {"missing prefix", "foo", false}, + {"prefix with no body", "system:bootstrappers:", false}, + {"invalid spaces", "system:bootstrappers: ", false}, + {"invalid asterisk", "system:bootstrappers:*", false}, + {"trailing colon", "system:bootstrappers:foo:", false}, + {"trailing dash", "system:bootstrappers:foo-", false}, + {"script tags", "system:bootstrappers:", false}, + {"too long", "system:bootstrappers:" + strings.Repeat("x", 300), false}, + } + for _, test := range tests { + err := ValidateBootstrapGroupName(test.input) + if err != nil && test.valid { + t.Errorf("test %q: ValidateBootstrapGroupName(%q) returned unexpected error: %v", test.name, test.input, err) + } + if err == nil && !test.valid { + t.Errorf("test %q: ValidateBootstrapGroupName(%q) was supposed to return an error but didn't", test.name, test.input) + } + } +} diff --git a/pkg/bootstrap/api/types.go b/pkg/bootstrap/api/types.go index a7cca0c9bd86e..a4e67a1c2491e 100644 --- a/pkg/bootstrap/api/types.go +++ b/pkg/bootstrap/api/types.go @@ -51,6 +51,11 @@ const ( // describes what the bootstrap token is used for. Optional. BootstrapTokenDescriptionKey = "description" + // BootstrapTokenExtraGroupsKey is a comma-separated list of group names. + // The bootstrap token will authenticate as these groups in addition to the + // "system:bootstrappers" group. + BootstrapTokenExtraGroupsKey = "auth-extra-groups" + // BootstrapTokenUsagePrefix is the prefix for the other usage constants that specifies different // functions of a bootstrap token BootstrapTokenUsagePrefix = "usage-bootstrap-" @@ -63,7 +68,8 @@ const ( // BootstrapTokenUsageAuthentication signals that this token should be used // as a bearer token to authenticate against the Kubernetes API. The bearer // token takes the form "." and authenticates as the - // user "system:bootstrap:" in the group "system:bootstrappers". + // user "system:bootstrap:" in the "system:bootstrappers" group + // as well as any groups specified using BootstrapTokenExtraGroupsKey. // Value must be "true". Any other value is assumed to be false. Optional. BootstrapTokenUsageAuthentication = "usage-bootstrap-authentication" @@ -80,6 +86,12 @@ const ( // authenticate as. The full username given is "system:bootstrap:". BootstrapUserPrefix = "system:bootstrap:" - // BootstrapGroup is the group bootstrapping bearer tokens authenticate in. - BootstrapGroup = "system:bootstrappers" + // BootstrapGroupPattern is the valid regex pattern that all groups + // assigned to a bootstrap token by BootstrapTokenExtraGroupsKey must match. + // See also ValidateBootstrapGroupName(). + BootstrapGroupPattern = "system:bootstrappers:[a-z0-9:-]{0,255}[a-z0-9]" + + // BootstrapDefaultGroup is the default group for bootstrapping bearer + // tokens (in addition to any groups from BootstrapTokenExtraGroupsKey). + BootstrapDefaultGroup = "system:bootstrappers" ) diff --git a/pkg/cloudprovider/OWNERS b/pkg/cloudprovider/OWNERS index 5abbe51cb2c03..249a4a546b174 100644 --- a/pkg/cloudprovider/OWNERS +++ b/pkg/cloudprovider/OWNERS @@ -38,3 +38,4 @@ reviewers: - jdef - freehan - jingxu97 +- wlan0 diff --git a/pkg/cloudprovider/cloud.go b/pkg/cloudprovider/cloud.go index 5c4651f7d3ad1..87bb8081db2d1 100644 --- a/pkg/cloudprovider/cloud.go +++ b/pkg/cloudprovider/cloud.go @@ -124,7 +124,7 @@ type Instances interface { // ProviderID is a unique identifier of the node. This will not be called // from the node whose nodeaddresses are being queried. i.e. local metadata // services cannot be used in this method to obtain nodeaddresses - NodeAddressesByProviderID(providerId string) ([]v1.NodeAddress, error) + NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) // ExternalID returns the cloud provider ID of the node with the specified NodeName. // Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound) ExternalID(nodeName types.NodeName) (string, error) @@ -140,6 +140,9 @@ type Instances interface { // CurrentNodeName returns the name of the node we are currently running on // On most clouds (e.g. GCE) this is the hostname, so we provide the hostname CurrentNodeName(hostname string) (types.NodeName, error) + // InstanceExistsByProviderID returns true if the instance for the given provider id still is running. + // If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. + InstanceExistsByProviderID(providerID string) (bool, error) } // Route is a representation of an advanced routing rule. @@ -184,5 +187,18 @@ type Zone struct { // Zones is an abstract, pluggable interface for zone enumeration. type Zones interface { // GetZone returns the Zone containing the current failure zone and locality region that the program is running in + // In most cases, this method is called from the kubelet querying a local metadata service to aquire its zone. + // For the case of external cloud providers, use GetZoneByProviderID or GetZoneByNodeName since GetZone + // can no longer be called from the kubelets. GetZone() (Zone, error) + + // GetZoneByProviderID returns the Zone containing the current zone and locality region of the node specified by providerId + // This method is particularly used in the context of external cloud providers where node initialization must be down + // outside the kubelets. + GetZoneByProviderID(providerID string) (Zone, error) + + // GetZoneByNodeName returns the Zone containing the current zone and locality region of the node specified by node name + // This method is particularly used in the context of external cloud providers where node initialization must be down + // outside the kubelets. + GetZoneByNodeName(nodeName types.NodeName) (Zone, error) } diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index 704c5e9966e4f..ad52baea29689 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -1101,6 +1101,12 @@ func (c *Cloud) ExternalID(nodeName types.NodeName) (string, error) { return orEmpty(instance.InstanceId), nil } +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (c *Cloud) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + // InstanceID returns the cloud provider ID of the node with the specified nodeName. func (c *Cloud) InstanceID(nodeName types.NodeName) (string, error) { // In the future it is possible to also return an endpoint as: @@ -1201,6 +1207,20 @@ func (c *Cloud) GetZone() (cloudprovider.Zone, error) { }, nil } +// GetZoneByProviderID implements Zones.GetZoneByProviderID +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (c *Cloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented") +} + +// GetZoneByNodeName implements Zones.GetZoneByNodeName +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (c *Cloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not imeplemented") +} + // Abstraction around AWS Instance Types // There isn't an API to get information for a particular instance type (that I know of) type awsInstanceType struct { @@ -2487,8 +2507,15 @@ func (c *Cloud) findELBSubnets(internalELB bool) ([]string, error) { continue } - // TODO: Should this be an error? - glog.Warningf("Found multiple subnets in AZ %q; making arbitrary choice between subnets %q and %q", az, *existing.SubnetId, *subnet.SubnetId) + // If we have two subnets for the same AZ we arbitrarily choose the one that is first lexicographically. + // TODO: Should this be an error. + if strings.Compare(*existing.SubnetId, *subnet.SubnetId) > 0 { + glog.Warningf("Found multiple subnets in AZ %q; choosing %q between subnets %q and %q", az, *subnet.SubnetId, *existing.SubnetId, *subnet.SubnetId) + subnetsByAZ[az] = subnet + continue + } + + glog.Warningf("Found multiple subnets in AZ %q; choosing %q between subnets %q and %q", az, *existing.SubnetId, *existing.SubnetId, *subnet.SubnetId) continue } diff --git a/pkg/cloudprovider/providers/aws/aws_test.go b/pkg/cloudprovider/providers/aws/aws_test.go index b1302510e63c6..ee2712b5205bb 100644 --- a/pkg/cloudprovider/providers/aws/aws_test.go +++ b/pkg/cloudprovider/providers/aws/aws_test.go @@ -792,12 +792,18 @@ func TestSubnetIDsinVPC(t *testing.T) { } } - // test with 4 subnets from 3 different AZs - // add duplicate az subnet + // Test with 5 subnets from 3 different AZs. + // Add 2 duplicate AZ subnets lexicographically chosen one is the middle element in array to + // check that we both choose the correct entry when it comes after and before another element + // in the same AZ. subnets[3] = make(map[string]string) - subnets[3]["id"] = "subnet-c0000002" + subnets[3]["id"] = "subnet-c0000000" subnets[3]["az"] = "af-south-1c" + subnets[4] = make(map[string]string) + subnets[4]["id"] = "subnet-c0000002" + subnets[4]["az"] = "af-south-1c" awsServices.ec2.Subnets = constructSubnets(subnets) + routeTables["subnet-c0000000"] = true routeTables["subnet-c0000002"] = true awsServices.ec2.RouteTables = constructRouteTables(routeTables) @@ -812,6 +818,16 @@ func TestSubnetIDsinVPC(t *testing.T) { return } + expected := []*string{aws.String("subnet-a0000001"), aws.String("subnet-b0000001"), aws.String("subnet-c0000000")} + for _, s := range result { + if !contains(expected, s) { + t.Errorf("Unexpected subnet '%s' found", s) + return + } + } + + delete(routeTables, "subnet-c0000002") + // test with 6 subnets from 3 different AZs // with 3 private subnets subnets[4] = make(map[string]string) @@ -825,7 +841,7 @@ func TestSubnetIDsinVPC(t *testing.T) { routeTables["subnet-a0000001"] = false routeTables["subnet-b0000001"] = false routeTables["subnet-c0000001"] = false - routeTables["subnet-c0000002"] = true + routeTables["subnet-c0000000"] = true routeTables["subnet-d0000001"] = true routeTables["subnet-d0000002"] = true awsServices.ec2.RouteTables = constructRouteTables(routeTables) @@ -840,7 +856,7 @@ func TestSubnetIDsinVPC(t *testing.T) { return } - expected := []*string{aws.String("subnet-c0000002"), aws.String("subnet-d0000001"), aws.String("subnet-d0000002")} + expected = []*string{aws.String("subnet-c0000000"), aws.String("subnet-d0000001"), aws.String("subnet-d0000002")} for _, s := range result { if !contains(expected, s) { t.Errorf("Unexpected subnet '%s' found", s) diff --git a/pkg/cloudprovider/providers/azure/azure_instances.go b/pkg/cloudprovider/providers/azure/azure_instances.go index b94b8225d6423..85e7f6acd1c57 100644 --- a/pkg/cloudprovider/providers/azure/azure_instances.go +++ b/pkg/cloudprovider/providers/azure/azure_instances.go @@ -17,6 +17,7 @@ limitations under the License. package azure import ( + "errors" "fmt" "k8s.io/api/core/v1" @@ -86,6 +87,12 @@ func (az *Cloud) ExternalID(name types.NodeName) (string, error) { return az.InstanceID(name) } +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (az *Cloud) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + func (az *Cloud) isCurrentInstance(name types.NodeName) (bool, error) { nodeName := mapNodeNameToVMName(name) metadataName, err := az.metadata.Text("instance/compute/name") diff --git a/pkg/cloudprovider/providers/azure/azure_zones.go b/pkg/cloudprovider/providers/azure/azure_zones.go index ab2dd9e3b27f3..3d993d414c60c 100644 --- a/pkg/cloudprovider/providers/azure/azure_zones.go +++ b/pkg/cloudprovider/providers/azure/azure_zones.go @@ -18,11 +18,13 @@ package azure import ( "encoding/json" + "errors" "io" "io/ioutil" "net/http" "sync" + "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" ) @@ -55,6 +57,20 @@ func (az *Cloud) GetZone() (cloudprovider.Zone, error) { return zone, nil } +// GetZoneByProviderID implements Zones.GetZoneByProviderID +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (az *Cloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented") +} + +// GetZoneByNodeName implements Zones.GetZoneByNodeName +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (az *Cloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not imeplemented") +} + func fetchFaultDomain() (*string, error) { resp, err := http.Get(instanceInfoURL) if err != nil { diff --git a/pkg/cloudprovider/providers/cloudstack/BUILD b/pkg/cloudprovider/providers/cloudstack/BUILD index ecfe91d54172f..0302fae33e488 100644 --- a/pkg/cloudprovider/providers/cloudstack/BUILD +++ b/pkg/cloudprovider/providers/cloudstack/BUILD @@ -19,6 +19,7 @@ go_library( "//vendor/github.com/xanzy/go-cloudstack/cloudstack:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/cloudstack/cloudstack.go b/pkg/cloudprovider/providers/cloudstack/cloudstack.go index 1b86d8243ab39..4e0bea291e24b 100644 --- a/pkg/cloudprovider/providers/cloudstack/cloudstack.go +++ b/pkg/cloudprovider/providers/cloudstack/cloudstack.go @@ -17,12 +17,15 @@ limitations under the License. package cloudstack import ( + "errors" "fmt" "io" "github.com/golang/glog" "github.com/xanzy/go-cloudstack/cloudstack" "gopkg.in/gcfg.v1" + + "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/controller" ) @@ -130,3 +133,17 @@ func (cs *CSCloud) GetZone() (cloudprovider.Zone, error) { glog.V(2).Infof("Current zone is %v", cs.zone) return cloudprovider.Zone{Region: cs.zone}, nil } + +// GetZoneByProviderID implements Zones.GetZoneByProviderID +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (cs *CSCloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented") +} + +// GetZoneByNodeName implements Zones.GetZoneByNodeName +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (cs *CSCloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not imeplemented") +} diff --git a/pkg/cloudprovider/providers/fake/fake.go b/pkg/cloudprovider/providers/fake/fake.go index 05c10c146cc2b..98fdc8d6641fd 100644 --- a/pkg/cloudprovider/providers/fake/fake.go +++ b/pkg/cloudprovider/providers/fake/fake.go @@ -47,8 +47,12 @@ type FakeUpdateBalancerCall struct { // FakeCloud is a test-double implementation of Interface, LoadBalancer, Instances, and Routes. It is useful for testing. type FakeCloud struct { - Exists bool - Err error + Exists bool + Err error + + ExistsByProviderID bool + ErrByProviderID error + Calls []string Addresses []v1.NodeAddress ExtID map[types.NodeName]string @@ -234,6 +238,13 @@ func (f *FakeCloud) InstanceTypeByProviderID(providerID string) (string, error) return f.InstanceTypes[types.NodeName(providerID)], nil } +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (f *FakeCloud) InstanceExistsByProviderID(providerID string) (bool, error) { + f.addCall("instance-exists-by-provider-id") + return f.ExistsByProviderID, f.ErrByProviderID +} + // List is a test-spy implementation of Instances.List. // It adds an entry "list" into the internal method call record. func (f *FakeCloud) List(filter string) ([]types.NodeName, error) { @@ -252,6 +263,22 @@ func (f *FakeCloud) GetZone() (cloudprovider.Zone, error) { return f.Zone, f.Err } +// GetZoneByProviderID implements Zones.GetZoneByProviderID +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (f *FakeCloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) { + f.addCall("get-zone-by-provider-id") + return f.Zone, f.Err +} + +// GetZoneByNodeName implements Zones.GetZoneByNodeName +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (f *FakeCloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) { + f.addCall("get-zone-by-node-name") + return f.Zone, f.Err +} + func (f *FakeCloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) { f.Lock.Lock() defer f.Lock.Unlock() diff --git a/pkg/cloudprovider/providers/gce/BUILD b/pkg/cloudprovider/providers/gce/BUILD index 086c4cafa4c86..5f2e696a5bd0d 100644 --- a/pkg/cloudprovider/providers/gce/BUILD +++ b/pkg/cloudprovider/providers/gce/BUILD @@ -13,6 +13,7 @@ go_library( "gce.go", "gce_addresses.go", "gce_addresses_fakes.go", + "gce_alpha.go", "gce_annotations.go", "gce_backendservice.go", "gce_cert.go", @@ -21,6 +22,7 @@ go_library( "gce_disks.go", "gce_firewall.go", "gce_forwardingrule.go", + "gce_forwardingrule_fakes.go", "gce_healthchecks.go", "gce_instancegroup.go", "gce_instances.go", @@ -36,6 +38,7 @@ go_library( "gce_urlmap.go", "gce_util.go", "gce_zones.go", + "kms.go", "metrics.go", "token_source.go", ], @@ -69,6 +72,8 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", + "//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/tools/cache:go_default_library", "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", @@ -82,12 +87,16 @@ go_test( "gce_healthchecks_test.go", "gce_loadbalancer_external_test.go", "gce_test.go", + "metrics_test.go", ], library = ":go_default_library", deps = [ "//pkg/cloudprovider:go_default_library", "//pkg/kubelet/apis:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/golang.org/x/oauth2/google:go_default_library", + "//vendor/google.golang.org/api/compute/v0.alpha:go_default_library", + "//vendor/google.golang.org/api/compute/v0.beta:go_default_library", "//vendor/google.golang.org/api/compute/v1:go_default_library", "//vendor/google.golang.org/api/googleapi:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index a4dd5eab159d6..0decfcaa25405 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -21,6 +21,7 @@ import ( "io" "net/http" "regexp" + "strconv" "strings" "sync" "time" @@ -31,6 +32,8 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/server/options/encryptionconfig" + "k8s.io/apiserver/pkg/storage/value/encrypt/envelope" "k8s.io/client-go/util/flowcontrol" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/controller" @@ -80,6 +83,11 @@ const ( gceComputeAPIEndpoint = "https://www.googleapis.com/compute/v1/" ) +// gceObject is an abstraction of all GCE API object in go client +type gceObject interface { + MarshalJSON() ([]byte, error) +} + // GCECloud is an implementation of Interface, LoadBalancer and Instances for Google Compute Engine. type GCECloud struct { // ClusterID contains functionality for getting (and initializing) the ingress-uid. Call GCECloud.Initialize() @@ -98,6 +106,7 @@ type GCECloud struct { managedZones []string // List of zones we are spanning (for multi-AZ clusters, primarily when running on master) networkURL string subnetworkURL string + secondaryRangeName string networkProjectID string onXPN bool nodeTags []string // List of tags to use on firewall rules for load balancers @@ -113,6 +122,11 @@ type GCECloud struct { // lock to prevent shared resources from being prematurely deleted while the operation is // in progress. sharedResourceLock sync.Mutex + // AlphaFeatureGate gates gce alpha features in GCECloud instance. + // Related wrapper functions that interacts with gce alpha api should examine whether + // the corresponding api is enabled. + // If not enabled, it should return error. + AlphaFeatureGate *AlphaFeatureGate } type ServiceManager interface { @@ -133,37 +147,64 @@ type GCEServiceManager struct { gce *GCECloud } +type ConfigGlobal struct { + TokenURL string `gcfg:"token-url"` + TokenBody string `gcfg:"token-body"` + // ProjectID and NetworkProjectID can either be the numeric or string-based + // unique identifier that starts with [a-z]. + ProjectID string `gcfg:"project-id"` + // NetworkProjectID refers to the project which owns the network being used. + NetworkProjectID string `gcfg:"network-project-id"` + NetworkName string `gcfg:"network-name"` + SubnetworkName string `gcfg:"subnetwork-name"` + // SecondaryRangeName is the name of the secondary range to allocate IP + // aliases. The secondary range must be present on the subnetwork the + // cluster is attached to. + SecondaryRangeName string `gcfg:"secondary-range-name"` + NodeTags []string `gcfg:"node-tags"` + NodeInstancePrefix string `gcfg:"node-instance-prefix"` + Multizone bool `gcfg:"multizone"` + // ApiEndpoint is the GCE compute API endpoint to use. If this is blank, + // then the default endpoint is used. + ApiEndpoint string `gcfg:"api-endpoint"` + // LocalZone specifies the GCE zone that gce cloud client instance is + // located in (i.e. where the controller will be running). If this is + // blank, then the local zone will be discovered via the metadata server. + LocalZone string `gcfg:"local-zone"` + // Possible values: List of api names separated by comma. Default to none. + // For example: MyFeatureFlag + AlphaFeatures []string `gcfg:"alpha-features"` +} + +// ConfigFile is the struct used to parse the /etc/gce.conf configuration file. type ConfigFile struct { - Global struct { - TokenURL string `gcfg:"token-url"` - TokenBody string `gcfg:"token-body"` - ProjectID string `gcfg:"project-id"` - NetworkName string `gcfg:"network-name"` - SubnetworkName string `gcfg:"subnetwork-name"` - NodeTags []string `gcfg:"node-tags"` - NodeInstancePrefix string `gcfg:"node-instance-prefix"` - Multizone bool `gcfg:"multizone"` - // Specifying ApiEndpoint will override the default GCE compute API endpoint. - ApiEndpoint string `gcfg:"api-endpoint"` - LocalZone string `gcfg:"local-zone"` - } + Global ConfigGlobal `gcfg:"global"` } // CloudConfig includes all the necessary configuration for creating GCECloud type CloudConfig struct { ApiEndpoint string ProjectID string + NetworkProjectID string Region string Zone string ManagedZones []string + NetworkName string NetworkURL string + SubnetworkName string SubnetworkURL string + SecondaryRangeName string NodeTags []string NodeInstancePrefix string TokenSource oauth2.TokenSource UseMetadataServer bool + AlphaFeatureGate *AlphaFeatureGate } +// kmsPluginRegisterOnce prevents the cloudprovider from registering its KMS plugin +// more than once in the KMS plugin registry. +var kmsPluginRegisterOnce sync.Once + func init() { cloudprovider.RegisterCloudProvider( ProviderName, @@ -182,11 +223,6 @@ func (g *GCECloud) GetKMSService() *cloudkms.Service { return g.cloudkmsService } -// Returns the ProjectID corresponding to the project this cloud is in. -func (g *GCECloud) GetProjectID() string { - return g.projectID -} - // newGCECloud creates a new instance of GCECloud. func newGCECloud(config io.Reader) (gceCloud *GCECloud, err error) { var cloudConfig *CloudConfig @@ -240,6 +276,12 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err cloudConfig.NodeTags = configFile.Global.NodeTags cloudConfig.NodeInstancePrefix = configFile.Global.NodeInstancePrefix + + alphaFeatureGate, err := NewAlphaFeatureGate(configFile.Global.AlphaFeatures) + if err != nil { + glog.Errorf("Encountered error for creating alpha feature gate: %v", err) + } + cloudConfig.AlphaFeatureGate = alphaFeatureGate } // retrieve projectID and zone @@ -249,6 +291,7 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err return nil, err } } + if configFile != nil { if configFile.Global.ProjectID != "" { cloudConfig.ProjectID = configFile.Global.ProjectID @@ -256,6 +299,9 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err if configFile.Global.LocalZone != "" { cloudConfig.Zone = configFile.Global.LocalZone } + if configFile.Global.NetworkProjectID != "" { + cloudConfig.NetworkProjectID = configFile.Global.NetworkProjectID + } } // retrieve region @@ -270,37 +316,46 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err cloudConfig.ManagedZones = nil // Use all zones in region } - // generate networkURL + // Determine if network parameter is URL or Name if configFile != nil && configFile.Global.NetworkName != "" { if strings.Contains(configFile.Global.NetworkName, "/") { cloudConfig.NetworkURL = configFile.Global.NetworkName } else { - cloudConfig.NetworkURL = gceNetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, configFile.Global.NetworkName) + cloudConfig.NetworkName = configFile.Global.NetworkName } } else { - networkName, err := getNetworkNameViaMetadata() + cloudConfig.NetworkName, err = getNetworkNameViaMetadata() if err != nil { return nil, err } - cloudConfig.NetworkURL = gceNetworkURL("", cloudConfig.ProjectID, networkName) } - // generate subnetworkURL + // Determine if subnetwork parameter is URL or Name + // If cluster is on a GCP network of mode=custom, then `SubnetName` must be specified in config file. if configFile != nil && configFile.Global.SubnetworkName != "" { if strings.Contains(configFile.Global.SubnetworkName, "/") { cloudConfig.SubnetworkURL = configFile.Global.SubnetworkName } else { - cloudConfig.SubnetworkURL = gceSubnetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, cloudConfig.Region, configFile.Global.SubnetworkName) + cloudConfig.SubnetworkName = configFile.Global.SubnetworkName } } + + if configFile != nil { + cloudConfig.SecondaryRangeName = configFile.Global.SecondaryRangeName + } + return cloudConfig, err } -// Creates a GCECloud object using the specified parameters. +// CreateGCECloud creates a GCECloud object using the specified parameters. // If no networkUrl is specified, loads networkName via rest call. // If no tokenSource is specified, uses oauth2.DefaultTokenSource. // If managedZones is nil / empty all zones in the region will be managed. func CreateGCECloud(config *CloudConfig) (*GCECloud, error) { + // Use ProjectID for NetworkProjectID, if it wasn't explicitly set. + if config.NetworkProjectID == "" { + config.NetworkProjectID = config.ProjectID + } client, err := newOauthClient(config.TokenSource) if err != nil { @@ -349,19 +404,34 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) { return nil, err } - if config.NetworkURL == "" { - networkName, err := getNetworkNameViaAPICall(service, config.ProjectID) + // ProjectID and.NetworkProjectID may be project number or name. + projID, netProjID := tryConvertToProjectNames(config.ProjectID, config.NetworkProjectID, service) + + onXPN := projID != netProjID + + var networkURL string + var subnetURL string + + if config.NetworkName == "" && config.NetworkURL == "" { + // TODO: Stop using this call and return an error. + // This function returns the first network in a list of networks for a project. The project + // should be set via configuration instead of randomly taking the first. + networkName, err := getNetworkNameViaAPICall(service, config.NetworkProjectID) if err != nil { return nil, err } - config.NetworkURL = gceNetworkURL(config.ApiEndpoint, config.ProjectID, networkName) + networkURL = gceNetworkURL(config.ApiEndpoint, netProjID, networkName) + } else if config.NetworkURL != "" { + networkURL = config.NetworkURL + } else { + networkURL = gceNetworkURL(config.ApiEndpoint, netProjID, config.NetworkName) } - networkProjectID, err := getProjectIDInURL(config.NetworkURL) - if err != nil { - return nil, err + if config.SubnetworkURL != "" { + subnetURL = config.SubnetworkURL + } else if config.SubnetworkName != "" { + subnetURL = gceSubnetworkURL(config.ApiEndpoint, netProjID, config.Region, config.SubnetworkName) } - onXPN := networkProjectID != config.ProjectID if len(config.ManagedZones) == 0 { config.ManagedZones, err = getZonesForRegion(service, config.ProjectID, config.Region) @@ -381,24 +451,62 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) { serviceBeta: serviceBeta, containerService: containerService, cloudkmsService: cloudkmsService, - projectID: config.ProjectID, - networkProjectID: networkProjectID, + projectID: projID, + networkProjectID: netProjID, onXPN: onXPN, region: config.Region, localZone: config.Zone, managedZones: config.ManagedZones, - networkURL: config.NetworkURL, - subnetworkURL: config.SubnetworkURL, + networkURL: networkURL, + subnetworkURL: subnetURL, + secondaryRangeName: config.SecondaryRangeName, nodeTags: config.NodeTags, nodeInstancePrefix: config.NodeInstancePrefix, useMetadataServer: config.UseMetadataServer, operationPollRateLimiter: operationPollRateLimiter, + AlphaFeatureGate: config.AlphaFeatureGate, } gce.manager = &GCEServiceManager{gce} + + // Registering the KMS plugin only the first time. + kmsPluginRegisterOnce.Do(func() { + // Register the Google Cloud KMS based service in the KMS plugin registry. + encryptionconfig.KMSPluginRegistry.Register(KMSServiceName, func(config io.Reader) (envelope.Service, error) { + return gce.getGCPCloudKMSService(config) + }) + }) + return gce, nil } +func tryConvertToProjectNames(configProject, configNetworkProject string, service *compute.Service) (projID, netProjID string) { + projID = configProject + if isProjectNumber(projID) { + projName, err := getProjectID(service, projID) + if err != nil { + glog.Warningf("Failed to retrieve project %v while trying to retrieve its name. err %v", projID, err) + } else { + projID = projName + } + } + + netProjID = projID + if configNetworkProject != configProject { + netProjID = configNetworkProject + } + if isProjectNumber(netProjID) { + netProjName, err := getProjectID(service, netProjID) + if err != nil { + glog.Warningf("Failed to retrieve network project %v while trying to retrieve its name. err %v", netProjID, err) + } else { + netProjID = netProjName + } + } + + return projID, netProjID +} + // Initialize takes in a clientBuilder and spawns a goroutine for watching the clusterid configmap. // This must be called before utilizing the funcs of gce.ClusterID func (gce *GCECloud) Initialize(clientBuilder controller.ControllerClientBuilder) { @@ -435,6 +543,16 @@ func (gce *GCECloud) ProviderName() string { return ProviderName } +// ProjectID returns the ProjectID corresponding to the project this cloud is in. +func (g *GCECloud) ProjectID() string { + return g.projectID +} + +// NetworkProjectID returns the ProjectID corresponding to the project this cluster's network is in. +func (g *GCECloud) NetworkProjectID() string { + return g.networkProjectID +} + // Region returns the region func (gce *GCECloud) Region() string { return gce.region @@ -474,6 +592,13 @@ func (gce *GCECloud) HasClusterID() bool { return true } +// Project IDs cannot have a digit for the first characeter. If the id contains a digit, +// then it must be a project number. +func isProjectNumber(idOrNumber string) bool { + _, err := strconv.ParseUint(idOrNumber, 10, 64) + return err == nil +} + // GCECloud implements cloudprovider.Interface. var _ cloudprovider.Interface = (*GCECloud)(nil) @@ -531,6 +656,16 @@ func getNetworkNameViaAPICall(svc *compute.Service, projectID string) (string, e return networkList.Items[0].Name, nil } +// getProjectID returns the project's string ID given a project number or string +func getProjectID(svc *compute.Service, projectNumberOrID string) (string, error) { + proj, err := svc.Projects.Get(projectNumberOrID).Do() + if err != nil { + return "", err + } + + return proj.Name, nil +} + func getZonesForRegion(svc *compute.Service, projectID, region string) ([]string, error) { // TODO: use PageToken to list all not just the first 500 listCall := svc.Zones.List(projectID) diff --git a/pkg/cloudprovider/providers/gce/gce_addresses.go b/pkg/cloudprovider/providers/gce/gce_addresses.go index 0a7b0d1997412..7e963e901bd38 100644 --- a/pkg/cloudprovider/providers/gce/gce_addresses.go +++ b/pkg/cloudprovider/providers/gce/gce_addresses.go @@ -18,17 +18,18 @@ package gce import ( "fmt" - "time" "github.com/golang/glog" + computealpha "google.golang.org/api/compute/v0.alpha" compute "google.golang.org/api/compute/v1" ) func newAddressMetricContext(request, region string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"address_" + request, region, unusedMetricLabel}, - } + return newAddressMetricContextWithVersion(request, region, computeV1Version) +} + +func newAddressMetricContextWithVersion(request, region, version string) *metricContext { + return newGenericMetricContext("address", request, region, unusedMetricLabel, version) } // ReserveGlobalAddress creates a global address. @@ -71,6 +72,16 @@ func (gce *GCECloud) ReserveRegionAddress(addr *compute.Address, region string) return gce.waitForRegionOp(op, region, mc) } +// ReserveAlphaRegionAddress creates an Alpha, regional address. +func (gce *GCECloud) ReserveAlphaRegionAddress(addr *computealpha.Address, region string) error { + mc := newAddressMetricContextWithVersion("reserve", region, computeAlphaVersion) + op, err := gce.serviceAlpha.Addresses.Insert(gce.projectID, region, addr).Do() + if err != nil { + return mc.Observe(err) + } + return gce.waitForRegionOp(op, region, mc) +} + // DeleteRegionAddress deletes a region address by name. func (gce *GCECloud) DeleteRegionAddress(name, region string) error { mc := newAddressMetricContext("delete", region) @@ -88,6 +99,13 @@ func (gce *GCECloud) GetRegionAddress(name, region string) (*compute.Address, er return v, mc.Observe(err) } +// GetAlphaRegionAddress returns the Alpha, regional address by name. +func (gce *GCECloud) GetAlphaRegionAddress(name, region string) (*computealpha.Address, error) { + mc := newAddressMetricContextWithVersion("get", region, computeAlphaVersion) + v, err := gce.serviceAlpha.Addresses.Get(gce.projectID, region, name).Do() + return v, mc.Observe(err) +} + // GetRegionAddressByIP returns the regional address matching the given IP // address. func (gce *GCECloud) GetRegionAddressByIP(region, ipAddress string) (*compute.Address, error) { diff --git a/pkg/cloudprovider/providers/gce/gce_addresses_fakes.go b/pkg/cloudprovider/providers/gce/gce_addresses_fakes.go index 156aaeb6d2352..94c86e95d9e3b 100644 --- a/pkg/cloudprovider/providers/gce/gce_addresses_fakes.go +++ b/pkg/cloudprovider/providers/gce/gce_addresses_fakes.go @@ -17,9 +17,11 @@ limitations under the License. package gce import ( + "encoding/json" "fmt" "net/http" + computealpha "google.golang.org/api/compute/v0.alpha" compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" ) @@ -31,17 +33,32 @@ type FakeCloudAddressService struct { reservedAddrs map[string]bool // addrsByRegionAndName // Outer key is for region string; inner key is for address name. - addrsByRegionAndName map[string]map[string]*compute.Address + addrsByRegionAndName map[string]map[string]*computealpha.Address } +// FakeCloudAddressService Implements CloudAddressService +var _ CloudAddressService = &FakeCloudAddressService{} + func NewFakeCloudAddressService() *FakeCloudAddressService { return &FakeCloudAddressService{ reservedAddrs: make(map[string]bool), - addrsByRegionAndName: make(map[string]map[string]*compute.Address), + addrsByRegionAndName: make(map[string]map[string]*computealpha.Address), } } -func (cas *FakeCloudAddressService) ReserveRegionAddress(addr *compute.Address, region string) error { +// SetRegionalAddresses sets the addresses of ther region. This is used for +// setting the test environment. +func (cas *FakeCloudAddressService) SetRegionalAddresses(region string, addrs []*computealpha.Address) { + // Reset addresses in the region. + cas.addrsByRegionAndName[region] = make(map[string]*computealpha.Address) + + for _, addr := range addrs { + cas.reservedAddrs[addr.Address] = true + cas.addrsByRegionAndName[region][addr.Name] = addr + } +} + +func (cas *FakeCloudAddressService) ReserveAlphaRegionAddress(addr *computealpha.Address, region string) error { if addr.Address == "" { addr.Address = fmt.Sprintf("1.2.3.%d", cas.count) cas.count++ @@ -52,7 +69,7 @@ func (cas *FakeCloudAddressService) ReserveRegionAddress(addr *compute.Address, } if _, exists := cas.addrsByRegionAndName[region]; !exists { - cas.addrsByRegionAndName[region] = make(map[string]*compute.Address) + cas.addrsByRegionAndName[region] = make(map[string]*computealpha.Address) } if _, exists := cas.addrsByRegionAndName[region][addr.Name]; exists { @@ -64,7 +81,12 @@ func (cas *FakeCloudAddressService) ReserveRegionAddress(addr *compute.Address, return nil } -func (cas *FakeCloudAddressService) GetRegionAddress(name, region string) (*compute.Address, error) { +func (cas *FakeCloudAddressService) ReserveRegionAddress(addr *compute.Address, region string) error { + alphaAddr := convertToAlphaAddress(addr) + return cas.ReserveAlphaRegionAddress(alphaAddr, region) +} + +func (cas *FakeCloudAddressService) GetAlphaRegionAddress(name, region string) (*computealpha.Address, error) { if _, exists := cas.addrsByRegionAndName[region]; !exists { return nil, makeGoogleAPINotFoundError("") } @@ -76,6 +98,26 @@ func (cas *FakeCloudAddressService) GetRegionAddress(name, region string) (*comp } } +func (cas *FakeCloudAddressService) GetRegionAddress(name, region string) (*compute.Address, error) { + addr, err := cas.GetAlphaRegionAddress(name, region) + if addr != nil { + return convertToV1Address(addr), err + } + return nil, err +} + +func (cas *FakeCloudAddressService) DeleteRegionAddress(name, region string) error { + if _, exists := cas.addrsByRegionAndName[region]; !exists { + return makeGoogleAPINotFoundError("") + } + + if _, exists := cas.addrsByRegionAndName[region][name]; !exists { + return makeGoogleAPINotFoundError("") + } + delete(cas.addrsByRegionAndName[region], name) + return nil +} + func (cas *FakeCloudAddressService) GetRegionAddressByIP(region, ipAddress string) (*compute.Address, error) { if _, exists := cas.addrsByRegionAndName[region]; !exists { return nil, makeGoogleAPINotFoundError("") @@ -83,8 +125,32 @@ func (cas *FakeCloudAddressService) GetRegionAddressByIP(region, ipAddress strin for _, addr := range cas.addrsByRegionAndName[region] { if addr.Address == ipAddress { - return addr, nil + return convertToV1Address(addr), nil } } return nil, makeGoogleAPINotFoundError("") } + +func convertToV1Address(object gceObject) *compute.Address { + enc, err := object.MarshalJSON() + if err != nil { + panic(fmt.Sprintf("Failed to encode to json: %v", err)) + } + var addr compute.Address + if err := json.Unmarshal(enc, &addr); err != nil { + panic(fmt.Sprintf("Failed to convert GCE apiObject %v to v1 address: %v", object, err)) + } + return &addr +} + +func convertToAlphaAddress(object gceObject) *computealpha.Address { + enc, err := object.MarshalJSON() + if err != nil { + panic(fmt.Sprintf("Failed to encode to json: %v", err)) + } + var addr computealpha.Address + if err := json.Unmarshal(enc, &addr); err != nil { + panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha address: %v", object, err)) + } + return &addr +} diff --git a/pkg/cloudprovider/providers/gce/gce_alpha.go b/pkg/cloudprovider/providers/gce/gce_alpha.go new file mode 100644 index 0000000000000..fb659c9391eea --- /dev/null +++ b/pkg/cloudprovider/providers/gce/gce_alpha.go @@ -0,0 +1,47 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "fmt" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +// All known alpha features +var knownAlphaFeatures = map[string]bool{} + +type AlphaFeatureGate struct { + features map[string]bool +} + +func (af *AlphaFeatureGate) Enabled(key string) bool { + return af.features[key] +} + +func NewAlphaFeatureGate(features []string) (*AlphaFeatureGate, error) { + errList := []error{} + featureMap := make(map[string]bool) + for _, name := range features { + if _, ok := knownAlphaFeatures[name]; !ok { + errList = append(errList, fmt.Errorf("alpha feature %q is not supported.", name)) + } else { + featureMap[name] = true + } + } + return &AlphaFeatureGate{featureMap}, utilerrors.NewAggregate(errList) +} diff --git a/pkg/cloudprovider/providers/gce/gce_backendservice.go b/pkg/cloudprovider/providers/gce/gce_backendservice.go index d460c6157d86a..7bf3d73e0d888 100644 --- a/pkg/cloudprovider/providers/gce/gce_backendservice.go +++ b/pkg/cloudprovider/providers/gce/gce_backendservice.go @@ -18,16 +18,12 @@ package gce import ( "net/http" - "time" compute "google.golang.org/api/compute/v1" ) func newBackendServiceMetricContext(request, region string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"backendservice_" + request, region, unusedMetricLabel}, - } + return newGenericMetricContext("backendservice", request, region, unusedMetricLabel, computeV1Version) } // GetGlobalBackendService retrieves a backend by name. diff --git a/pkg/cloudprovider/providers/gce/gce_cert.go b/pkg/cloudprovider/providers/gce/gce_cert.go index 77af58bbe3c43..e981fe71765e4 100644 --- a/pkg/cloudprovider/providers/gce/gce_cert.go +++ b/pkg/cloudprovider/providers/gce/gce_cert.go @@ -18,16 +18,12 @@ package gce import ( "net/http" - "time" compute "google.golang.org/api/compute/v1" ) func newCertMetricContext(request string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"cert_" + request, unusedMetricLabel, unusedMetricLabel}, - } + return newGenericMetricContext("cert", request, unusedMetricLabel, unusedMetricLabel, computeV1Version) } // GetSslCertificate returns the SslCertificate by name. diff --git a/pkg/cloudprovider/providers/gce/gce_clusters.go b/pkg/cloudprovider/providers/gce/gce_clusters.go index 4524d483f693c..dad330ca5d559 100644 --- a/pkg/cloudprovider/providers/gce/gce_clusters.go +++ b/pkg/cloudprovider/providers/gce/gce_clusters.go @@ -16,13 +16,8 @@ limitations under the License. package gce -import "time" - func newClustersMetricContext(request, zone string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"clusters_" + request, unusedMetricLabel, zone}, - } + return newGenericMetricContext("clusters", request, unusedMetricLabel, zone, computeV1Version) } func (gce *GCECloud) ListClusters() ([]string, error) { diff --git a/pkg/cloudprovider/providers/gce/gce_disks.go b/pkg/cloudprovider/providers/gce/gce_disks.go index c916989b5ee2e..57ca223da7336 100644 --- a/pkg/cloudprovider/providers/gce/gce_disks.go +++ b/pkg/cloudprovider/providers/gce/gce_disks.go @@ -21,7 +21,6 @@ import ( "fmt" "net/http" "strings" - "time" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" @@ -85,10 +84,7 @@ type GCEDisk struct { } func newDiskMetricContext(request, zone string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"disk_" + request, unusedMetricLabel, zone}, - } + return newGenericMetricContext("disk", request, unusedMetricLabel, zone, computeV1Version) } func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bool) error { diff --git a/pkg/cloudprovider/providers/gce/gce_firewall.go b/pkg/cloudprovider/providers/gce/gce_firewall.go index 668f2e0552224..2e5297275f978 100644 --- a/pkg/cloudprovider/providers/gce/gce_firewall.go +++ b/pkg/cloudprovider/providers/gce/gce_firewall.go @@ -17,53 +17,48 @@ limitations under the License. package gce import ( - "time" - compute "google.golang.org/api/compute/v1" ) func newFirewallMetricContext(request string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"firewall_" + request, unusedMetricLabel, unusedMetricLabel}, - } + return newGenericMetricContext("firewall", request, unusedMetricLabel, unusedMetricLabel, computeV1Version) } // GetFirewall returns the Firewall by name. func (gce *GCECloud) GetFirewall(name string) (*compute.Firewall, error) { mc := newFirewallMetricContext("get") - v, err := gce.service.Firewalls.Get(gce.projectID, name).Do() + v, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), name).Do() return v, mc.Observe(err) } // CreateFirewall creates the passed firewall func (gce *GCECloud) CreateFirewall(f *compute.Firewall) error { mc := newFirewallMetricContext("create") - op, err := gce.service.Firewalls.Insert(gce.projectID, f).Do() + op, err := gce.service.Firewalls.Insert(gce.NetworkProjectID(), f).Do() if err != nil { return mc.Observe(err) } - return gce.waitForGlobalOp(op, mc) + return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc) } // DeleteFirewall deletes the given firewall rule. func (gce *GCECloud) DeleteFirewall(name string) error { mc := newFirewallMetricContext("delete") - op, err := gce.service.Firewalls.Delete(gce.projectID, name).Do() + op, err := gce.service.Firewalls.Delete(gce.NetworkProjectID(), name).Do() if err != nil { return mc.Observe(err) } - return gce.waitForGlobalOp(op, mc) + return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc) } // UpdateFirewall applies the given firewall as an update to an existing service. func (gce *GCECloud) UpdateFirewall(f *compute.Firewall) error { mc := newFirewallMetricContext("update") - op, err := gce.service.Firewalls.Update(gce.projectID, f.Name, f).Do() + op, err := gce.service.Firewalls.Update(gce.NetworkProjectID(), f.Name, f).Do() if err != nil { return mc.Observe(err) } - return gce.waitForGlobalOp(op, mc) + return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc) } diff --git a/pkg/cloudprovider/providers/gce/gce_forwardingrule.go b/pkg/cloudprovider/providers/gce/gce_forwardingrule.go index 47c5f20c3b643..0473c44266d63 100644 --- a/pkg/cloudprovider/providers/gce/gce_forwardingrule.go +++ b/pkg/cloudprovider/providers/gce/gce_forwardingrule.go @@ -17,16 +17,15 @@ limitations under the License. package gce import ( - "time" - + computealpha "google.golang.org/api/compute/v0.alpha" compute "google.golang.org/api/compute/v1" ) func newForwardingRuleMetricContext(request, region string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"forwardingrule_" + request, region, unusedMetricLabel}, - } + return newForwardingRuleMetricContextWithVersion(request, region, computeV1Version) +} +func newForwardingRuleMetricContextWithVersion(request, region, version string) *metricContext { + return newGenericMetricContext("forwardingrule", request, region, unusedMetricLabel, version) } // CreateGlobalForwardingRule creates the passed GlobalForwardingRule @@ -85,6 +84,13 @@ func (gce *GCECloud) GetRegionForwardingRule(name, region string) (*compute.Forw return v, mc.Observe(err) } +// GetAlphaRegionForwardingRule returns the Alpha forwarding rule by name & region. +func (gce *GCECloud) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) { + mc := newForwardingRuleMetricContextWithVersion("get", region, computeAlphaVersion) + v, err := gce.serviceAlpha.ForwardingRules.Get(gce.projectID, region, name).Do() + return v, mc.Observe(err) +} + // ListRegionForwardingRules lists all RegionalForwardingRules in the project & region. func (gce *GCECloud) ListRegionForwardingRules(region string) (*compute.ForwardingRuleList, error) { mc := newForwardingRuleMetricContext("list", region) @@ -105,6 +111,18 @@ func (gce *GCECloud) CreateRegionForwardingRule(rule *compute.ForwardingRule, re return gce.waitForRegionOp(op, region, mc) } +// CreateAlphaRegionForwardingRule creates and returns an Alpha +// forwarding fule in the given region. +func (gce *GCECloud) CreateAlphaRegionForwardingRule(rule *computealpha.ForwardingRule, region string) error { + mc := newForwardingRuleMetricContextWithVersion("create", region, computeAlphaVersion) + op, err := gce.serviceAlpha.ForwardingRules.Insert(gce.projectID, region, rule).Do() + if err != nil { + return mc.Observe(err) + } + + return gce.waitForRegionOp(op, region, mc) +} + // DeleteRegionForwardingRule deletes the RegionalForwardingRule by name & region. func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error { mc := newForwardingRuleMetricContext("delete", region) diff --git a/pkg/cloudprovider/providers/gce/gce_forwardingrule_fakes.go b/pkg/cloudprovider/providers/gce/gce_forwardingrule_fakes.go new file mode 100644 index 0000000000000..0cc0188223aec --- /dev/null +++ b/pkg/cloudprovider/providers/gce/gce_forwardingrule_fakes.go @@ -0,0 +1,127 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "encoding/json" + "fmt" + "net/http" + + computealpha "google.golang.org/api/compute/v0.alpha" + compute "google.golang.org/api/compute/v1" + "google.golang.org/api/googleapi" +) + +type FakeCloudForwardingRuleService struct { + // fwdRulesByRegionAndName + // Outer key is for region string; inner key is for fwdRuleess name. + fwdRulesByRegionAndName map[string]map[string]*computealpha.ForwardingRule +} + +// FakeCloudForwardingRuleService Implements CloudForwardingRuleService +var _ CloudForwardingRuleService = &FakeCloudForwardingRuleService{} + +func NewFakeCloudForwardingRuleService() *FakeCloudForwardingRuleService { + return &FakeCloudForwardingRuleService{ + fwdRulesByRegionAndName: make(map[string]map[string]*computealpha.ForwardingRule), + } +} + +// SetRegionalForwardingRulees sets the fwdRuleesses of ther region. This is used for +// setting the test environment. +func (f *FakeCloudForwardingRuleService) SetRegionalForwardingRulees(region string, fwdRules []*computealpha.ForwardingRule) { + // Reset fwdRuleesses in the region. + f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule) + + for _, fwdRule := range fwdRules { + f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule + } +} + +func (f *FakeCloudForwardingRuleService) CreateAlphaRegionForwardingRule(fwdRule *computealpha.ForwardingRule, region string) error { + if _, exists := f.fwdRulesByRegionAndName[region]; !exists { + f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule) + } + + if _, exists := f.fwdRulesByRegionAndName[region][fwdRule.Name]; exists { + return &googleapi.Error{Code: http.StatusConflict} + } + + f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule + return nil +} + +func (f *FakeCloudForwardingRuleService) CreateRegionForwardingRule(fwdRule *compute.ForwardingRule, region string) error { + alphafwdRule := convertToAlphaForwardingRule(fwdRule) + return f.CreateAlphaRegionForwardingRule(alphafwdRule, region) +} + +func (f *FakeCloudForwardingRuleService) DeleteRegionForwardingRule(name, region string) error { + if _, exists := f.fwdRulesByRegionAndName[region]; !exists { + return makeGoogleAPINotFoundError("") + } + + if _, exists := f.fwdRulesByRegionAndName[region][name]; !exists { + return makeGoogleAPINotFoundError("") + } + delete(f.fwdRulesByRegionAndName[region], name) + return nil +} + +func (f *FakeCloudForwardingRuleService) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) { + if _, exists := f.fwdRulesByRegionAndName[region]; !exists { + return nil, makeGoogleAPINotFoundError("") + } + + if fwdRule, exists := f.fwdRulesByRegionAndName[region][name]; !exists { + return nil, makeGoogleAPINotFoundError("") + } else { + return fwdRule, nil + } +} + +func (f *FakeCloudForwardingRuleService) GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error) { + fwdRule, err := f.GetAlphaRegionForwardingRule(name, region) + if fwdRule != nil { + return convertToV1ForwardingRule(fwdRule), err + } + return nil, err +} + +func convertToV1ForwardingRule(object gceObject) *compute.ForwardingRule { + enc, err := object.MarshalJSON() + if err != nil { + panic(fmt.Sprintf("Failed to encode to json: %v", err)) + } + var fwdRule compute.ForwardingRule + if err := json.Unmarshal(enc, &fwdRule); err != nil { + panic(fmt.Sprintf("Failed to convert GCE apiObject %v to v1 fwdRuleess: %v", object, err)) + } + return &fwdRule +} + +func convertToAlphaForwardingRule(object gceObject) *computealpha.ForwardingRule { + enc, err := object.MarshalJSON() + if err != nil { + panic(fmt.Sprintf("Failed to encode to json: %v", err)) + } + var fwdRule computealpha.ForwardingRule + if err := json.Unmarshal(enc, &fwdRule); err != nil { + panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha fwdRuleess: %v", object, err)) + } + return &fwdRule +} diff --git a/pkg/cloudprovider/providers/gce/gce_healthchecks.go b/pkg/cloudprovider/providers/gce/gce_healthchecks.go index 9fd2989b2a67c..85f077f0007c3 100644 --- a/pkg/cloudprovider/providers/gce/gce_healthchecks.go +++ b/pkg/cloudprovider/providers/gce/gce_healthchecks.go @@ -17,8 +17,6 @@ limitations under the License. package gce import ( - "time" - "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/master/ports" utilversion "k8s.io/kubernetes/pkg/util/version" @@ -45,10 +43,7 @@ func init() { } func newHealthcheckMetricContext(request string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"healthcheck_" + request, unusedMetricLabel, unusedMetricLabel}, - } + return newGenericMetricContext("healthcheck", request, unusedMetricLabel, unusedMetricLabel, computeV1Version) } // GetHttpHealthCheck returns the given HttpHealthCheck by name. diff --git a/pkg/cloudprovider/providers/gce/gce_instancegroup.go b/pkg/cloudprovider/providers/gce/gce_instancegroup.go index 044847ab3b42f..67083224bb2cc 100644 --- a/pkg/cloudprovider/providers/gce/gce_instancegroup.go +++ b/pkg/cloudprovider/providers/gce/gce_instancegroup.go @@ -16,17 +16,10 @@ limitations under the License. package gce -import ( - "time" - - compute "google.golang.org/api/compute/v1" -) +import compute "google.golang.org/api/compute/v1" func newInstanceGroupMetricContext(request string, zone string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"instancegroup_" + request, unusedMetricLabel, zone}, - } + return newGenericMetricContext("instancegroup", request, unusedMetricLabel, zone, computeV1Version) } // CreateInstanceGroup creates an instance group with the given diff --git a/pkg/cloudprovider/providers/gce/gce_instances.go b/pkg/cloudprovider/providers/gce/gce_instances.go index 8d997ae9ed4f5..c6e31dee61e8d 100644 --- a/pkg/cloudprovider/providers/gce/gce_instances.go +++ b/pkg/cloudprovider/providers/gce/gce_instances.go @@ -17,7 +17,9 @@ limitations under the License. package gce import ( + "errors" "fmt" + "net" "net/http" "strconv" "strings" @@ -25,6 +27,7 @@ import ( "cloud.google.com/go/compute/metadata" "github.com/golang/glog" + computealpha "google.golang.org/api/compute/v0.alpha" computebeta "google.golang.org/api/compute/v0.beta" compute "google.golang.org/api/compute/v1" @@ -41,10 +44,7 @@ const ( ) func newInstancesMetricContext(request, zone string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"instances_" + request, unusedMetricLabel, zone}, - } + return newGenericMetricContext("instances", request, unusedMetricLabel, zone, computeV1Version) } func splitNodesByZone(nodes []*v1.Node) map[string][]*v1.Node { @@ -154,6 +154,12 @@ func (gce *GCECloud) ExternalID(nodeName types.NodeName) (string, error) { return strconv.FormatUint(inst.ID, 10), nil } +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (gce *GCECloud) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + // InstanceID returns the cloud provider ID of the node with the specified NodeName. func (gce *GCECloud) InstanceID(nodeName types.NodeName) (string, error) { instanceName := mapNodeNameToInstanceName(nodeName) @@ -318,6 +324,43 @@ func (gce *GCECloud) AliasRanges(nodeName types.NodeName) (cidrs []string, err e return } +// AddAliasToInstance adds an alias to the given instance from the named +// secondary range. +func (gce *GCECloud) AddAliasToInstance(nodeName types.NodeName, alias *net.IPNet) error { + + v1instance, err := gce.getInstanceByName(mapNodeNameToInstanceName(nodeName)) + if err != nil { + return err + } + instance, err := gce.serviceAlpha.Instances.Get(gce.projectID, v1instance.Zone, v1instance.Name).Do() + if err != nil { + return err + } + + switch len(instance.NetworkInterfaces) { + case 0: + return fmt.Errorf("Instance %q has no network interfaces", nodeName) + case 1: + default: + glog.Warningf("Instance %q has more than one network interface, using only the first (%v)", + nodeName, instance.NetworkInterfaces) + } + + iface := instance.NetworkInterfaces[0] + iface.AliasIpRanges = append(iface.AliasIpRanges, &computealpha.AliasIpRange{ + IpCidrRange: alias.String(), + SubnetworkRangeName: gce.secondaryRangeName, + }) + + mc := newInstancesMetricContext("addalias", v1instance.Zone) + op, err := gce.serviceAlpha.Instances.UpdateNetworkInterface( + gce.projectID, instance.Zone, instance.Name, iface.Name, iface).Do() + if err != nil { + return mc.Observe(err) + } + return gce.waitForZoneOp(op, v1instance.Zone, mc) +} + // Gets the named instances, returning cloudprovider.InstanceNotFound if any instance is not found func (gce *GCECloud) getInstancesByNames(names []string) ([]*gceInstance, error) { instances := make(map[string]*gceInstance) diff --git a/pkg/cloudprovider/providers/gce/gce_interfaces.go b/pkg/cloudprovider/providers/gce/gce_interfaces.go index b8c98d082e1c2..96bfa343b85cb 100644 --- a/pkg/cloudprovider/providers/gce/gce_interfaces.go +++ b/pkg/cloudprovider/providers/gce/gce_interfaces.go @@ -16,12 +16,32 @@ limitations under the License. package gce -import compute "google.golang.org/api/compute/v1" +import ( + computealpha "google.golang.org/api/compute/v0.alpha" + compute "google.golang.org/api/compute/v1" +) // CloudAddressService is an interface for managing addresses type CloudAddressService interface { ReserveRegionAddress(*compute.Address, string) error GetRegionAddress(string, string) (*compute.Address, error) - // TODO: Mock `DeleteRegionAddress(name, region string) endpoint + GetRegionAddressByIP(region, ipAddress string) (*compute.Address, error) + DeleteRegionAddress(name, region string) error // TODO: Mock Global endpoints + + // Alpha API. + GetAlphaRegionAddress(name, region string) (*computealpha.Address, error) + ReserveAlphaRegionAddress(addr *computealpha.Address, region string) error +} + +// CloudForwardingRuleService is an interface for managing forwarding rules. +// TODO: Expand the interface to include more methods. +type CloudForwardingRuleService interface { + GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error) + CreateRegionForwardingRule(rule *compute.ForwardingRule, region string) error + DeleteRegionForwardingRule(name, region string) error + + // Alpha API. + GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) + CreateAlphaRegionForwardingRule(rule *computealpha.ForwardingRule, region string) error } diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer.go index 7026df8fea12a..ba2fa03cdb8ce 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer.go @@ -21,7 +21,6 @@ import ( "fmt" "net" "strings" - "time" "github.com/golang/glog" @@ -40,10 +39,7 @@ var ( ) func newLoadBalancerMetricContext(request, region string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"loadbalancer_" + request, region, unusedMetricLabel}, - } + return newGenericMetricContext("loadbalancer", request, region, unusedMetricLabel, computeV1Version) } type lbScheme string diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go index f8d3ff4603bec..129086e77c5e6 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go @@ -55,7 +55,7 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a } loadBalancerName := cloudprovider.GetLoadBalancerName(apiService) - loadBalancerIP := apiService.Spec.LoadBalancerIP + requestedIP := apiService.Spec.LoadBalancerIP ports := apiService.Spec.Ports portStr := []string{} for _, p := range apiService.Spec.Ports { @@ -66,10 +66,10 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name} glog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", - loadBalancerName, gce.region, loadBalancerIP, portStr, hostNames, serviceName, apiService.Annotations) + loadBalancerName, gce.region, requestedIP, portStr, hostNames, serviceName, apiService.Annotations) // Check if the forwarding rule exists, and if so, what its IP is. - fwdRuleExists, fwdRuleNeedsUpdate, fwdRuleIP, err := gce.forwardingRuleNeedsUpdate(loadBalancerName, gce.region, loadBalancerIP, ports) + fwdRuleExists, fwdRuleNeedsUpdate, fwdRuleIP, err := gce.forwardingRuleNeedsUpdate(loadBalancerName, gce.region, requestedIP, ports) if err != nil { return nil, err } @@ -93,7 +93,7 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a // forwarding rule creation as the last thing that needs to be done in this // function in order to maintain the invariant that "if the forwarding rule // exists, the LB has been fully created". - ipAddress := "" + ipAddressToUse := "" // Through this process we try to keep track of whether it is safe to // release the IP that was allocated. If the user specifically asked for @@ -110,75 +110,42 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a } if isSafeToReleaseIP { if err := gce.DeleteRegionAddress(loadBalancerName, gce.region); err != nil && !isNotFound(err) { - glog.Errorf("failed to release static IP %s for load balancer (%v(%v), %v): %v", ipAddress, loadBalancerName, serviceName, gce.region, err) + glog.Errorf("Failed to release static IP %s for load balancer (%v(%v), %v): %v", ipAddressToUse, loadBalancerName, serviceName, gce.region, err) } else if isNotFound(err) { - glog.V(2).Infof("EnsureLoadBalancer(%v(%v)): address %s is not reserved.", loadBalancerName, serviceName, ipAddress) + glog.V(2).Infof("EnsureLoadBalancer(%v(%v)): address %s is not reserved.", loadBalancerName, serviceName, ipAddressToUse) } else { - glog.V(2).Infof("EnsureLoadBalancer(%v(%v)): released static IP %s", loadBalancerName, serviceName, ipAddress) + glog.V(2).Infof("EnsureLoadBalancer(%v(%v)): released static IP %s", loadBalancerName, serviceName, ipAddressToUse) } } else { - glog.Warningf("orphaning static IP %s during update of load balancer (%v(%v), %v): %v", ipAddress, loadBalancerName, serviceName, gce.region, err) + glog.Warningf("orphaning static IP %s during update of load balancer (%v(%v), %v): %v", ipAddressToUse, loadBalancerName, serviceName, gce.region, err) } }() - if loadBalancerIP != "" { - // If a specific IP address has been requested, we have to respect the - // user's request and use that IP. If the forwarding rule was already using - // a different IP, it will be harmlessly abandoned because it was only an - // ephemeral IP (or it was a different static IP owned by the user, in which - // case we shouldn't delete it anyway). - if existingAddress, err := gce.GetRegionAddressByIP(gce.region, loadBalancerIP); err != nil && !isNotFound(err) { - return nil, fmt.Errorf("failed to test if this GCE project owns the static IP %s: %v", loadBalancerIP, err) - } else if err == nil { - // The requested IP is a static IP, owned and managed by the user. - isUserOwnedIP = true - isSafeToReleaseIP = false - ipAddress = loadBalancerIP - glog.V(4).Infof("EnsureLoadBalancer(%v(%v)): using user-provided static IP %s (name: %s)", loadBalancerName, serviceName, ipAddress, existingAddress.Name) - } else if loadBalancerIP == fwdRuleIP { - // The requested IP is not a static IP, but is currently assigned - // to this forwarding rule, so we can keep it. - isUserOwnedIP = false - isSafeToReleaseIP = true - ipAddress, _, err = ensureStaticIP(gce, loadBalancerName, serviceName.String(), gce.region, fwdRuleIP) - if err != nil { - return nil, fmt.Errorf("failed to ensure static IP %s: %v", fwdRuleIP, err) - } - glog.V(4).Infof("EnsureLoadBalancer(%v(%v)): using user-provided non-static IP %s", loadBalancerName, serviceName, ipAddress) - } else { - // The requested IP is not static and it is not assigned to the - // current forwarding rule. It might be attached to a different - // rule or it might not be part of this project at all. Either - // way, we can't use it. - return nil, fmt.Errorf("requested ip %s is neither static nor assigned to LB %s(%v): %v", loadBalancerIP, loadBalancerName, serviceName, err) + lbRefStr := fmt.Sprintf("%v(%v)", loadBalancerName, serviceName) + if requestedIP != "" { + // If user requests a specific IP address, verify first. No mutation to + // the GCE resources will be performed in the verification process. + isUserOwnedIP, err = verifyUserRequestedIP(gce, gce.region, requestedIP, fwdRuleIP, lbRefStr) + if err != nil { + return nil, err } - } else { - // The user did not request a specific IP. - isUserOwnedIP = false - - // This will either allocate a new static IP if the forwarding rule didn't - // already have an IP, or it will promote the forwarding rule's current - // IP from ephemeral to static, or it will just get the IP if it is - // already static. - existed := false - ipAddress, existed, err = ensureStaticIP(gce, loadBalancerName, serviceName.String(), gce.region, fwdRuleIP) + ipAddressToUse = requestedIP + } + + if !isUserOwnedIP { + // If we are not using the user-owned IP, either promote the + // emphemeral IP used by the fwd rule, or create a new static IP. + ipAddr, existed, err := ensureStaticIP(gce, loadBalancerName, serviceName.String(), gce.region, fwdRuleIP) if err != nil { - return nil, fmt.Errorf("failed to ensure static IP %s: %v", fwdRuleIP, err) - } - if existed { - // If the IP was not specifically requested by the user, but it - // already existed, it seems to be a failed update cycle. We can - // use this IP and try to run through the process again, but we - // should not release the IP unless it is explicitly flagged as OK. - isSafeToReleaseIP = false - glog.V(4).Infof("EnsureLoadBalancer(%v(%v)): adopting static IP %s", loadBalancerName, serviceName, ipAddress) - } else { - // For total clarity. The IP did not pre-exist and the user did - // not ask for a particular one, so we can release the IP in case - // of failure or success. - isSafeToReleaseIP = true - glog.V(4).Infof("EnsureLoadBalancer(%v(%v)): allocated static IP %s", loadBalancerName, serviceName, ipAddress) + return nil, fmt.Errorf("failed to ensure a static IP for the LB: %v", err) } + glog.V(4).Infof("EnsureLoadBalancer(%s): ensured IP address %s", lbRefStr, ipAddr) + // If the IP was not owned by the user, but it already existed, it + // could indicate that the previous update cycle failed. We can use + // this IP and try to run through the process again, but we should + // not release the IP unless it is explicitly flagged as OK. + isSafeToReleaseIP = !existed + ipAddressToUse = ipAddr } // Deal with the firewall next. The reason we do this here rather than last @@ -190,13 +157,13 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a return nil, err } - firewallExists, firewallNeedsUpdate, err := gce.firewallNeedsUpdate(loadBalancerName, serviceName.String(), gce.region, ipAddress, ports, sourceRanges) + firewallExists, firewallNeedsUpdate, err := gce.firewallNeedsUpdate(loadBalancerName, serviceName.String(), gce.region, ipAddressToUse, ports, sourceRanges) if err != nil { return nil, err } if firewallNeedsUpdate { - desc := makeFirewallDescription(serviceName.String(), ipAddress) + desc := makeFirewallDescription(serviceName.String(), ipAddressToUse) // Unlike forwarding rules and target pools, firewalls can be updated // without needing to be deleted and recreated. if firewallExists { @@ -293,7 +260,7 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a createInstances = createInstances[:maxTargetPoolCreateInstances] } // Pass healthchecks to createTargetPool which needs them as health check links in the target pool - if err := gce.createTargetPool(loadBalancerName, serviceName.String(), ipAddress, gce.region, clusterID, createInstances, affinityType, hcToCreate); err != nil { + if err := gce.createTargetPool(loadBalancerName, serviceName.String(), ipAddressToUse, gce.region, clusterID, createInstances, affinityType, hcToCreate); err != nil { return nil, fmt.Errorf("failed to create target pool %s: %v", loadBalancerName, err) } if hcToCreate != nil { @@ -315,8 +282,8 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a } } if tpNeedsUpdate || fwdRuleNeedsUpdate { - glog.Infof("EnsureLoadBalancer(%v(%v)): creating forwarding rule, IP %s", loadBalancerName, serviceName, ipAddress) - if err := gce.createForwardingRule(loadBalancerName, serviceName.String(), gce.region, ipAddress, ports); err != nil { + glog.Infof("EnsureLoadBalancer(%v(%v)): creating forwarding rule, IP %s", loadBalancerName, serviceName, ipAddressToUse) + if err := gce.createForwardingRule(loadBalancerName, serviceName.String(), gce.region, ipAddressToUse, ports); err != nil { return nil, fmt.Errorf("failed to create forwarding rule %s: %v", loadBalancerName, err) } // End critical section. It is safe to release the static IP (which @@ -324,11 +291,11 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a // of a user-requested IP, the "is user-owned" flag will be set, // preventing it from actually being released. isSafeToReleaseIP = true - glog.Infof("EnsureLoadBalancer(%v(%v)): created forwarding rule, IP %s", loadBalancerName, serviceName, ipAddress) + glog.Infof("EnsureLoadBalancer(%v(%v)): created forwarding rule, IP %s", loadBalancerName, serviceName, ipAddressToUse) } status := &v1.LoadBalancerStatus{} - status.Ingress = []v1.LoadBalancerIngress{{IP: ipAddress}} + status.Ingress = []v1.LoadBalancerIngress{{IP: ipAddressToUse}} return status, nil } @@ -456,6 +423,42 @@ func (gce *GCECloud) DeleteExternalTargetPoolAndChecks(name, region, clusterID s return nil } +// verifyUserRequestedIP checks the user-provided IP to see whether it can be +// used for the LB. It also returns whether the IP is considered owned by the +// user. +func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string) (isUserOwnedIP bool, err error) { + if requestedIP == "" { + return false, nil + } + // If a specific IP address has been requested, we have to respect the + // user's request and use that IP. If the forwarding rule was already using + // a different IP, it will be harmlessly abandoned because it was only an + // ephemeral IP (or it was a different static IP owned by the user, in which + // case we shouldn't delete it anyway). + existingAddress, err := s.GetRegionAddressByIP(region, requestedIP) + if err != nil && !isNotFound(err) { + glog.Errorf("verifyUserRequestedIP: failed to check whether the requested IP %q for LB %s exists: %v", requestedIP, lbRef, err) + return false, err + } + if err == nil { + // The requested IP is a static IP, owned and managed by the user. + glog.V(4).Infof("verifyUserRequestedIP: the requested static IP %q (name: %s) for LB %s exists.", requestedIP, existingAddress.Name, lbRef) + return true, nil + } + if requestedIP == fwdRuleIP { + // The requested IP is not a static IP, but is currently assigned + // to this forwarding rule, so we can just use it. + glog.V(4).Infof("verifyUserRequestedIP: the requested IP %q is not static, but is currently in use by for LB %s", requestedIP, lbRef) + return false, nil + } + // The requested IP is not static and it is not assigned to the + // current forwarding rule. It might be attached to a different + // rule or it might not be part of this project at all. Either + // way, we can't use it. + glog.Errorf("verifyUserRequestedIP: requested IP %q for LB %s is neither static nor assigned to the LB", requestedIP, lbRef) + return false, fmt.Errorf("requested ip %q is neither static nor assigned to the LB", requestedIP) +} + func (gce *GCECloud) createTargetPool(name, serviceName, ipAddress, region, clusterID string, hosts []*gceInstance, affinityType v1.ServiceAffinity, hc *compute.HttpHealthCheck) error { // health check management is coupled with targetPools to prevent leaks. A // target pool is the only thing that requires a health check, so we delete @@ -726,7 +729,7 @@ func (gce *GCECloud) firewallNeedsUpdate(name, serviceName, region, ipAddress st return false, false, nil } - fw, err := gce.service.Firewalls.Get(gce.projectID, makeFirewallName(name)).Do() + fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), makeFirewallName(name)).Do() if err != nil { if isHTTPErrorCode(err, http.StatusNotFound) { return false, true, nil @@ -773,7 +776,7 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(serviceName, ipAddress, regio ports := []v1.ServicePort{{Protocol: "tcp", Port: hcPort}} fwName := MakeHealthCheckFirewallName(clusterID, hcName, isNodesHealthCheck) - fw, err := gce.service.Firewalls.Get(gce.projectID, fwName).Do() + fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), fwName).Do() if err != nil { if !isHTTPErrorCode(err, http.StatusNotFound) { return fmt.Errorf("error getting firewall for health checks: %v", err) diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go index 0dff1a2945a09..d0f26f8910d35 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go @@ -16,7 +16,14 @@ limitations under the License. package gce -import "testing" +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + computealpha "google.golang.org/api/compute/v0.alpha" +) func TestEnsureStaticIP(t *testing.T) { fcas := NewFakeCloudAddressService() @@ -37,3 +44,44 @@ func TestEnsureStaticIP(t *testing.T) { t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, fcas, ipName, serviceName, region, ip, ipPrime, existed, err, ip) } } + +func TestVerifyRequestedIP(t *testing.T) { + region := "test-region" + lbRef := "test-lb" + s := NewFakeCloudAddressService() + + for desc, tc := range map[string]struct { + requestedIP string + fwdRuleIP string + addrList []*computealpha.Address + expectErr bool + expectUserOwned bool + }{ + "requested IP exists": { + requestedIP: "1.1.1.1", + addrList: []*computealpha.Address{{Name: "foo", Address: "1.1.1.1"}}, + expectErr: false, + expectUserOwned: true, + }, + "requested IP is not static, but is in use by the fwd rule": { + requestedIP: "1.1.1.1", + fwdRuleIP: "1.1.1.1", + expectErr: false, + }, + "requested IP is not static and is not used by the fwd rule": { + requestedIP: "1.1.1.1", + fwdRuleIP: "2.2.2.2", + expectErr: true, + }, + "no requested IP": { + expectErr: false, + }, + } { + t.Run(desc, func(t *testing.T) { + s.SetRegionalAddresses(region, tc.addrList) + isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef) + assert.Equal(t, tc.expectErr, err != nil, fmt.Sprintf("err: %v", err)) + assert.Equal(t, tc.expectUserOwned, isUserOwnedIP, desc) + }) + } +} diff --git a/pkg/cloudprovider/providers/gce/gce_op.go b/pkg/cloudprovider/providers/gce/gce_op.go index 631a4f415145a..b354d79cea706 100644 --- a/pkg/cloudprovider/providers/gce/gce_op.go +++ b/pkg/cloudprovider/providers/gce/gce_op.go @@ -17,17 +17,20 @@ limitations under the License. package gce import ( + "encoding/json" "fmt" "time" "k8s.io/apimachinery/pkg/util/wait" "github.com/golang/glog" - compute "google.golang.org/api/compute/v1" + computealpha "google.golang.org/api/compute/v0.alpha" + computebeta "google.golang.org/api/compute/v0.beta" + computev1 "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" ) -func (gce *GCECloud) waitForOp(op *compute.Operation, getOperation func(operationName string) (*compute.Operation, error), mc *metricContext) error { +func (gce *GCECloud) waitForOp(op *computev1.Operation, getOperation func(operationName string) (*computev1.Operation, error), mc *metricContext) error { if op == nil { return mc.Observe(fmt.Errorf("operation must not be nil")) } @@ -72,11 +75,11 @@ func (gce *GCECloud) waitForOp(op *compute.Operation, getOperation func(operatio }) } -func opIsDone(op *compute.Operation) bool { +func opIsDone(op *computev1.Operation) bool { return op != nil && op.Status == "DONE" } -func getErrorFromOp(op *compute.Operation) error { +func getErrorFromOp(op *computev1.Operation) error { if op != nil && op.Error != nil && len(op.Error.Errors) > 0 { err := &googleapi.Error{ Code: int(op.HttpErrorStatusCode), @@ -89,20 +92,89 @@ func getErrorFromOp(op *compute.Operation) error { return nil } -func (gce *GCECloud) waitForGlobalOp(op *compute.Operation, mc *metricContext) error { - return gce.waitForOp(op, func(operationName string) (*compute.Operation, error) { - return gce.service.GlobalOperations.Get(gce.projectID, operationName).Do() - }, mc) +func (gce *GCECloud) waitForGlobalOp(op gceObject, mc *metricContext) error { + return gce.waitForGlobalOpInProject(op, gce.ProjectID(), mc) } -func (gce *GCECloud) waitForRegionOp(op *compute.Operation, region string, mc *metricContext) error { - return gce.waitForOp(op, func(operationName string) (*compute.Operation, error) { - return gce.service.RegionOperations.Get(gce.projectID, region, operationName).Do() - }, mc) +func (gce *GCECloud) waitForRegionOp(op gceObject, region string, mc *metricContext) error { + return gce.waitForRegionOpInProject(op, gce.ProjectID(), region, mc) } -func (gce *GCECloud) waitForZoneOp(op *compute.Operation, zone string, mc *metricContext) error { - return gce.waitForOp(op, func(operationName string) (*compute.Operation, error) { - return gce.service.ZoneOperations.Get(gce.projectID, zone, operationName).Do() - }, mc) +func (gce *GCECloud) waitForZoneOp(op gceObject, zone string, mc *metricContext) error { + return gce.waitForZoneOpInProject(op, gce.ProjectID(), zone, mc) +} + +func (gce *GCECloud) waitForGlobalOpInProject(op gceObject, projectID string, mc *metricContext) error { + switch v := op.(type) { + case *computealpha.Operation: + return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) { + op, err := gce.serviceAlpha.GlobalOperations.Get(projectID, operationName).Do() + return convertToV1Operation(op), err + }, mc) + case *computebeta.Operation: + return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) { + op, err := gce.serviceBeta.GlobalOperations.Get(projectID, operationName).Do() + return convertToV1Operation(op), err + }, mc) + case *computev1.Operation: + return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) { + return gce.service.GlobalOperations.Get(projectID, operationName).Do() + }, mc) + default: + return fmt.Errorf("unexpected type: %T", v) + } +} + +func (gce *GCECloud) waitForRegionOpInProject(op gceObject, projectID, region string, mc *metricContext) error { + switch v := op.(type) { + case *computealpha.Operation: + return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) { + op, err := gce.serviceAlpha.RegionOperations.Get(projectID, region, operationName).Do() + return convertToV1Operation(op), err + }, mc) + case *computebeta.Operation: + return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) { + op, err := gce.serviceBeta.RegionOperations.Get(projectID, region, operationName).Do() + return convertToV1Operation(op), err + }, mc) + case *computev1.Operation: + return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) { + return gce.service.RegionOperations.Get(projectID, region, operationName).Do() + }, mc) + default: + return fmt.Errorf("unexpected type: %T", v) + } +} + +func (gce *GCECloud) waitForZoneOpInProject(op gceObject, projectID, zone string, mc *metricContext) error { + switch v := op.(type) { + case *computealpha.Operation: + return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) { + op, err := gce.serviceAlpha.ZoneOperations.Get(projectID, zone, operationName).Do() + return convertToV1Operation(op), err + }, mc) + case *computebeta.Operation: + return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) { + op, err := gce.serviceBeta.ZoneOperations.Get(projectID, zone, operationName).Do() + return convertToV1Operation(op), err + }, mc) + case *computev1.Operation: + return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) { + return gce.service.ZoneOperations.Get(projectID, zone, operationName).Do() + }, mc) + default: + return fmt.Errorf("unexpected type: %T", v) + } +} + +func convertToV1Operation(object gceObject) *computev1.Operation { + enc, err := object.MarshalJSON() + if err != nil { + panic(fmt.Sprintf("Failed to encode to json: %v", err)) + } + var op computev1.Operation + if err := json.Unmarshal(enc, &op); err != nil { + panic(fmt.Sprintf("Failed to convert GCE apiObject %v to v1 operation: %v", object, err)) + } + return &op } diff --git a/pkg/cloudprovider/providers/gce/gce_routes.go b/pkg/cloudprovider/providers/gce/gce_routes.go index c5d3a37c24592..87184aef1142c 100644 --- a/pkg/cloudprovider/providers/gce/gce_routes.go +++ b/pkg/cloudprovider/providers/gce/gce_routes.go @@ -20,8 +20,6 @@ import ( "fmt" "net/http" "path" - "strings" - "time" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" @@ -31,10 +29,7 @@ import ( ) func newRoutesMetricContext(request string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"routes_" + request, unusedMetricLabel, unusedMetricLabel}, - } + return newGenericMetricContext("routes", request, unusedMetricLabel, unusedMetricLabel, computeV1Version) } func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) { @@ -43,10 +38,15 @@ func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, err page := 0 for ; page == 0 || (pageToken != "" && page < maxPages); page++ { mc := newRoutesMetricContext("list_page") - listCall := gce.service.Routes.List(gce.projectID) + listCall := gce.service.Routes.List(gce.NetworkProjectID()) prefix := truncateClusterName(clusterName) - listCall = listCall.Filter("name eq " + prefix + "-.*") + // Filter for routes starting with clustername AND belonging to the + // relevant gcp network AND having description = "k8s-node-route". + filter := "(name eq " + prefix + "-.*) " + filter = filter + "(network eq " + gce.NetworkURL() + ") " + filter = filter + "(description eq " + k8sNodeRouteTag + ")" + listCall = listCall.Filter(filter) if pageToken != "" { listCall = listCall.PageToken(pageToken) } @@ -58,18 +58,6 @@ func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, err } pageToken = res.NextPageToken for _, r := range res.Items { - if r.Network != gce.networkURL { - continue - } - // Not managed if route description != "k8s-node-route" - if r.Description != k8sNodeRouteTag { - continue - } - // Not managed if route name doesn't start with - if !strings.HasPrefix(r.Name, prefix) { - continue - } - target := path.Base(r.NextHopInstance) // TODO: Should we lastComponent(target) this? targetNodeName := types.NodeName(target) // NodeName == Instance Name on GCE @@ -92,11 +80,11 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo } mc := newRoutesMetricContext("create") - insertOp, err := gce.service.Routes.Insert(gce.projectID, &compute.Route{ + insertOp, err := gce.service.Routes.Insert(gce.NetworkProjectID(), &compute.Route{ Name: routeName, DestRange: route.DestinationCIDR, NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", targetInstance.Zone, targetInstance.Name), - Network: gce.networkURL, + Network: gce.NetworkURL(), Priority: 1000, Description: k8sNodeRouteTag, }).Do() @@ -108,16 +96,16 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo return mc.Observe(err) } } - return gce.waitForGlobalOp(insertOp, mc) + return gce.waitForGlobalOpInProject(insertOp, gce.NetworkProjectID(), mc) } func (gce *GCECloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error { mc := newRoutesMetricContext("delete") - deleteOp, err := gce.service.Routes.Delete(gce.projectID, route.Name).Do() + deleteOp, err := gce.service.Routes.Delete(gce.NetworkProjectID(), route.Name).Do() if err != nil { return mc.Observe(err) } - return gce.waitForGlobalOp(deleteOp, mc) + return gce.waitForGlobalOpInProject(deleteOp, gce.NetworkProjectID(), mc) } func truncateClusterName(clusterName string) string { diff --git a/pkg/cloudprovider/providers/gce/gce_targetpool.go b/pkg/cloudprovider/providers/gce/gce_targetpool.go index 9de1f6befb934..0efe15adfc0f3 100644 --- a/pkg/cloudprovider/providers/gce/gce_targetpool.go +++ b/pkg/cloudprovider/providers/gce/gce_targetpool.go @@ -16,17 +16,10 @@ limitations under the License. package gce -import ( - "time" - - compute "google.golang.org/api/compute/v1" -) +import compute "google.golang.org/api/compute/v1" func newTargetPoolMetricContext(request, region string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"targetpool_" + request, region, unusedMetricLabel}, - } + return newGenericMetricContext("targetpool", request, region, unusedMetricLabel, computeV1Version) } // GetTargetPool returns the TargetPool by name. diff --git a/pkg/cloudprovider/providers/gce/gce_targetproxy.go b/pkg/cloudprovider/providers/gce/gce_targetproxy.go index af5a52855d4bd..b92206cfc4e0e 100644 --- a/pkg/cloudprovider/providers/gce/gce_targetproxy.go +++ b/pkg/cloudprovider/providers/gce/gce_targetproxy.go @@ -18,16 +18,12 @@ package gce import ( "net/http" - "time" compute "google.golang.org/api/compute/v1" ) func newTargetProxyMetricContext(request string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"targetproxy_" + request, unusedMetricLabel, unusedMetricLabel}, - } + return newGenericMetricContext("targetproxy", request, unusedMetricLabel, unusedMetricLabel, computeV1Version) } // GetTargetHttpProxy returns the UrlMap by name. diff --git a/pkg/cloudprovider/providers/gce/gce_test.go b/pkg/cloudprovider/providers/gce/gce_test.go index a4ac14734b411..cc8a70300b7a3 100644 --- a/pkg/cloudprovider/providers/gce/gce_test.go +++ b/pkg/cloudprovider/providers/gce/gce_test.go @@ -17,12 +17,55 @@ limitations under the License. package gce import ( - "golang.org/x/oauth2/google" + "encoding/json" "reflect" "strings" "testing" + + "golang.org/x/oauth2/google" + + computealpha "google.golang.org/api/compute/v0.alpha" + computebeta "google.golang.org/api/compute/v0.beta" + computev1 "google.golang.org/api/compute/v1" ) +func TestReadConfigFile(t *testing.T) { + const s = `[Global] +token-url = my-token-url +token-body = my-token-body +project-id = my-project +network-project-id = my-network-project +network-name = my-network +subnetwork-name = my-subnetwork +secondary-range-name = my-secondary-range +node-tags = my-node-tag1 +node-instance-prefix = my-prefix +multizone = true + ` + reader := strings.NewReader(s) + config, err := readConfig(reader) + if err != nil { + t.Fatalf("Unexpected config parsing error %v", err) + } + + expected := &ConfigFile{Global: ConfigGlobal{ + TokenURL: "my-token-url", + TokenBody: "my-token-body", + ProjectID: "my-project", + NetworkProjectID: "my-network-project", + NetworkName: "my-network", + SubnetworkName: "my-subnetwork", + SecondaryRangeName: "my-secondary-range", + NodeTags: []string{"my-node-tag1"}, + NodeInstancePrefix: "my-prefix", + Multizone: true, + }} + + if !reflect.DeepEqual(expected, config) { + t.Fatalf("Expected config file values to be read into ConfigFile struct. \nExpected:\n%+v\nActual:\n%+v", expected, config) + } +} + func TestExtraKeyInConfig(t *testing.T) { const s = `[Global] project-id = my-project @@ -258,182 +301,272 @@ func TestSplitProviderID(t *testing.T) { } func TestGenerateCloudConfigs(t *testing.T) { + configBoilerplate := ConfigGlobal{ + TokenURL: "", + TokenBody: "", + ProjectID: "project-id", + NetworkName: "network-name", + SubnetworkName: "", + SecondaryRangeName: "", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + Multizone: false, + ApiEndpoint: "", + LocalZone: "us-central1-a", + AlphaFeatures: []string{}, + } + + cloudBoilerplate := CloudConfig{ + ApiEndpoint: "", + ProjectID: "project-id", + NetworkProjectID: "", + Region: "us-central1", + Zone: "us-central1-a", + ManagedZones: []string{"us-central1-a"}, + NetworkName: "network-name", + SubnetworkName: "", + NetworkURL: "", + SubnetworkURL: "", + SecondaryRangeName: "", + NodeTags: []string{"node-tag"}, + TokenSource: google.ComputeTokenSource(""), + NodeInstancePrefix: "node-prefix", + UseMetadataServer: true, + AlphaFeatureGate: &AlphaFeatureGate{map[string]bool{}}, + } + testCases := []struct { - TokenURL string - TokenBody string - ProjectID string - NetworkName string - SubnetworkName string - NodeTags []string - NodeInstancePrefix string - Multizone bool - ApiEndpoint string - LocalZone string - cloudConfig *CloudConfig + name string + config func() ConfigGlobal + cloud func() CloudConfig }{ { - TokenURL: "", - TokenBody: "", - ProjectID: "project-id", - NetworkName: "network-name", - SubnetworkName: "subnetwork-name", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - Multizone: false, - ApiEndpoint: "", - LocalZone: "us-central1-a", - cloudConfig: &CloudConfig{ - ApiEndpoint: "", - ProjectID: "project-id", - Region: "us-central1", - Zone: "us-central1-a", - ManagedZones: []string{"us-central1-a"}, - NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name", - SubnetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - TokenSource: google.ComputeTokenSource(""), - UseMetadataServer: true, + name: "Empty Config", + config: func() ConfigGlobal { return configBoilerplate }, + cloud: func() CloudConfig { return cloudBoilerplate }, + }, + { + name: "Nil token URL", + config: func() ConfigGlobal { + v := configBoilerplate + v.TokenURL = "nil" + return v + }, + cloud: func() CloudConfig { + v := cloudBoilerplate + v.TokenSource = nil + return v }, }, - // nil token source { - TokenURL: "nil", - TokenBody: "", - ProjectID: "project-id", - NetworkName: "network-name", - SubnetworkName: "subnetwork-name", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - Multizone: false, - ApiEndpoint: "", - LocalZone: "us-central1-a", - cloudConfig: &CloudConfig{ - ApiEndpoint: "", - ProjectID: "project-id", - Region: "us-central1", - Zone: "us-central1-a", - ManagedZones: []string{"us-central1-a"}, - NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name", - SubnetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - TokenSource: nil, - UseMetadataServer: true, + name: "Network Project ID", + config: func() ConfigGlobal { + v := configBoilerplate + v.NetworkProjectID = "my-awesome-project" + return v + }, + cloud: func() CloudConfig { + v := cloudBoilerplate + v.NetworkProjectID = "my-awesome-project" + return v }, }, - // specified api endpoint { - TokenURL: "", - TokenBody: "", - ProjectID: "project-id", - NetworkName: "network-name", - SubnetworkName: "subnetwork-name", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - Multizone: false, - ApiEndpoint: "https://www.googleapis.com/compute/staging_v1/", - LocalZone: "us-central1-a", - cloudConfig: &CloudConfig{ - ApiEndpoint: "https://www.googleapis.com/compute/staging_v1/", - ProjectID: "project-id", - Region: "us-central1", - Zone: "us-central1-a", - ManagedZones: []string{"us-central1-a"}, - NetworkURL: "https://www.googleapis.com/compute/staging_v1/projects/project-id/global/networks/network-name", - SubnetworkURL: "https://www.googleapis.com/compute/staging_v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - TokenSource: google.ComputeTokenSource(""), - UseMetadataServer: true, + name: "Specified API Endpint", + config: func() ConfigGlobal { + v := configBoilerplate + v.ApiEndpoint = "https://www.googleapis.com/compute/staging_v1/" + return v + }, + cloud: func() CloudConfig { + v := cloudBoilerplate + v.ApiEndpoint = "https://www.googleapis.com/compute/staging_v1/" + return v }, }, - // empty subnet-name { - TokenURL: "", - TokenBody: "", - ProjectID: "project-id", - NetworkName: "network-name", - SubnetworkName: "", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - Multizone: false, - ApiEndpoint: "", - LocalZone: "us-central1-a", - cloudConfig: &CloudConfig{ - ApiEndpoint: "", - ProjectID: "project-id", - Region: "us-central1", - Zone: "us-central1-a", - ManagedZones: []string{"us-central1-a"}, - NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name", - SubnetworkURL: "", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - TokenSource: google.ComputeTokenSource(""), - UseMetadataServer: true, + name: "Network & Subnetwork names", + config: func() ConfigGlobal { + v := configBoilerplate + v.NetworkName = "my-network" + v.SubnetworkName = "my-subnetwork" + return v + }, + cloud: func() CloudConfig { + v := cloudBoilerplate + v.NetworkName = "my-network" + v.SubnetworkName = "my-subnetwork" + return v }, }, - // multi zone { - TokenURL: "", - TokenBody: "", - ProjectID: "project-id", - NetworkName: "network-name", - SubnetworkName: "subnetwork-name", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - Multizone: true, - ApiEndpoint: "", - LocalZone: "us-central1-a", - cloudConfig: &CloudConfig{ - ApiEndpoint: "", - ProjectID: "project-id", - Region: "us-central1", - Zone: "us-central1-a", - ManagedZones: nil, - NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name", - SubnetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name", - NodeTags: []string{"node-tag"}, - NodeInstancePrefix: "node-prefix", - TokenSource: google.ComputeTokenSource(""), - UseMetadataServer: true, + name: "Network & Subnetwork URLs", + config: func() ConfigGlobal { + v := configBoilerplate + v.NetworkName = "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/my-network" + v.SubnetworkName = "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/my-subnetwork" + return v + }, + cloud: func() CloudConfig { + v := cloudBoilerplate + v.NetworkName = "" + v.SubnetworkName = "" + v.NetworkURL = "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/my-network" + v.SubnetworkURL = "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/my-subnetwork" + return v + }, + }, + { + name: "Multizone", + config: func() ConfigGlobal { + v := configBoilerplate + v.Multizone = true + return v + }, + cloud: func() CloudConfig { + v := cloudBoilerplate + v.ManagedZones = nil + return v + }, + }, + { + name: "Secondary Range Name", + config: func() ConfigGlobal { + v := configBoilerplate + v.SecondaryRangeName = "my-secondary" + return v + }, + cloud: func() CloudConfig { + v := cloudBoilerplate + v.SecondaryRangeName = "my-secondary" + return v }, }, } for _, tc := range testCases { - cloudConfig, err := generateCloudConfig(&ConfigFile{ - Global: struct { - TokenURL string `gcfg:"token-url"` - TokenBody string `gcfg:"token-body"` - ProjectID string `gcfg:"project-id"` - NetworkName string `gcfg:"network-name"` - SubnetworkName string `gcfg:"subnetwork-name"` - NodeTags []string `gcfg:"node-tags"` - NodeInstancePrefix string `gcfg:"node-instance-prefix"` - Multizone bool `gcfg:"multizone"` - ApiEndpoint string `gcfg:"api-endpoint"` - LocalZone string `gcfg:"local-zone"` - }{ - TokenURL: tc.TokenURL, - TokenBody: tc.TokenBody, - ProjectID: tc.ProjectID, - NetworkName: tc.NetworkName, - SubnetworkName: tc.SubnetworkName, - NodeTags: tc.NodeTags, - NodeInstancePrefix: tc.NodeInstancePrefix, - Multizone: tc.Multizone, - ApiEndpoint: tc.ApiEndpoint, - LocalZone: tc.LocalZone, - }, + t.Run(tc.name, func(t *testing.T) { + resultCloud, err := generateCloudConfig(&ConfigFile{Global: tc.config()}) + if err != nil { + t.Fatalf("Unexpect error: %v", err) + } + + v := tc.cloud() + if !reflect.DeepEqual(*resultCloud, v) { + t.Errorf("Got: \n%v\nWant\n%v\n", v, *resultCloud) + } }) - if err != nil { - t.Fatalf("Unexpect error: %v", err) + } +} + +func TestConvertToV1Operation(t *testing.T) { + v1Op := getTestOperation() + enc, _ := v1Op.MarshalJSON() + var op interface{} + var alphaOp computealpha.Operation + var betaOp computebeta.Operation + + if err := json.Unmarshal(enc, &alphaOp); err != nil { + t.Errorf("Failed to unmarshal operation: %v", err) + } + + if err := json.Unmarshal(enc, &betaOp); err != nil { + t.Errorf("Failed to unmarshal operation: %v", err) + } + + op = convertToV1Operation(&alphaOp) + if _, ok := op.(*computev1.Operation); ok { + if !reflect.DeepEqual(op, v1Op) { + t.Errorf("Failed to maintain consistency across conversion") } + } else { + t.Errorf("Expect output to be type v1 operation, but got %v", op) + } + + op = convertToV1Operation(&betaOp) + if _, ok := op.(*computev1.Operation); ok { + if !reflect.DeepEqual(op, v1Op) { + t.Errorf("Failed to maintain consistency across conversion") + } + } else { + t.Errorf("Expect output to be type v1 operation, but got %v", op) + } +} + +func getTestOperation() *computev1.Operation { + return &computev1.Operation{ + Name: "test", + Description: "test", + Id: uint64(12345), + Error: &computev1.OperationError{ + Errors: []*computev1.OperationErrorErrors{ + { + Code: "555", + Message: "error", + }, + }, + }, + } +} - if !reflect.DeepEqual(cloudConfig, tc.cloudConfig) { - t.Errorf("Expecting cloud config: %v, but got %v", tc.cloudConfig, cloudConfig) +func TestNewAlphaFeatureGate(t *testing.T) { + knownAlphaFeatures["foo"] = true + knownAlphaFeatures["bar"] = true + + testCases := []struct { + alphaFeatures []string + expectEnabled []string + expectDisabled []string + expectError bool + }{ + // enable foo bar + { + alphaFeatures: []string{"foo", "bar"}, + expectEnabled: []string{"foo", "bar"}, + expectDisabled: []string{"aaa"}, + expectError: false, + }, + // no alpha feature + { + alphaFeatures: []string{}, + expectEnabled: []string{}, + expectDisabled: []string{"foo", "bar"}, + expectError: false, + }, + // unsupported alpha feature + { + alphaFeatures: []string{"aaa", "foo"}, + expectError: true, + expectEnabled: []string{"foo"}, + expectDisabled: []string{"aaa"}, + }, + // enable foo + { + alphaFeatures: []string{"foo"}, + expectEnabled: []string{"foo"}, + expectDisabled: []string{"bar"}, + expectError: false, + }, + } + + for _, tc := range testCases { + featureGate, err := NewAlphaFeatureGate(tc.alphaFeatures) + + if (tc.expectError && err == nil) || (!tc.expectError && err != nil) { + t.Errorf("Expect error to be %v, but got error %v", tc.expectError, err) + } + + for _, key := range tc.expectEnabled { + if !featureGate.Enabled(key) { + t.Errorf("Expect %q to be enabled.", key) + } + } + for _, key := range tc.expectDisabled { + if featureGate.Enabled(key) { + t.Errorf("Expect %q to be disabled.", key) + } } } + delete(knownAlphaFeatures, "foo") + delete(knownAlphaFeatures, "bar") } diff --git a/pkg/cloudprovider/providers/gce/gce_urlmap.go b/pkg/cloudprovider/providers/gce/gce_urlmap.go index f19db2f23a708..60184ac435492 100644 --- a/pkg/cloudprovider/providers/gce/gce_urlmap.go +++ b/pkg/cloudprovider/providers/gce/gce_urlmap.go @@ -18,16 +18,12 @@ package gce import ( "net/http" - "time" compute "google.golang.org/api/compute/v1" ) func newUrlMapMetricContext(request string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"urlmap_" + request, unusedMetricLabel, unusedMetricLabel}, - } + return newGenericMetricContext("urlmap", request, unusedMetricLabel, unusedMetricLabel, computeV1Version) } // GetUrlMap returns the UrlMap by name. diff --git a/pkg/cloudprovider/providers/gce/gce_zones.go b/pkg/cloudprovider/providers/gce/gce_zones.go index 3179e75789285..107bb878f5491 100644 --- a/pkg/cloudprovider/providers/gce/gce_zones.go +++ b/pkg/cloudprovider/providers/gce/gce_zones.go @@ -17,20 +17,18 @@ limitations under the License. package gce import ( + "errors" "fmt" - "time" + "strings" compute "google.golang.org/api/compute/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" - "strings" ) func newZonesMetricContext(request, region string) *metricContext { - return &metricContext{ - start: time.Now(), - attributes: []string{"zones_" + request, region, unusedMetricLabel}, - } + return newGenericMetricContext("zones", request, region, unusedMetricLabel, computeV1Version) } // GetZone creates a cloudprovider.Zone of the current zone and region @@ -41,6 +39,20 @@ func (gce *GCECloud) GetZone() (cloudprovider.Zone, error) { }, nil } +// GetZoneByProviderID implements Zones.GetZoneByProviderID +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (gce *GCECloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented") +} + +// GetZoneByNodeName implements Zones.GetZoneByNodeName +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (gce *GCECloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not imeplemented") +} + // ListZonesInRegion returns all zones in a GCP region func (gce *GCECloud) ListZonesInRegion(region string) ([]*compute.Zone, error) { mc := newZonesMetricContext("list", region) diff --git a/pkg/cloudprovider/providers/gce/kms.go b/pkg/cloudprovider/providers/gce/kms.go new file mode 100644 index 0000000000000..fbe62f523b55b --- /dev/null +++ b/pkg/cloudprovider/providers/gce/kms.go @@ -0,0 +1,167 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "encoding/base64" + "fmt" + "io" + + "github.com/golang/glog" + cloudkms "google.golang.org/api/cloudkms/v1" + "google.golang.org/api/googleapi" + gcfg "gopkg.in/gcfg.v1" + "k8s.io/apiserver/pkg/storage/value/encrypt/envelope" +) + +const ( + // KMSServiceName is the name of the cloudkms provider registered by this cloud. + KMSServiceName = "gcp-cloudkms" + + defaultGKMSKeyRing = "google-container-engine" + defaultGKMSKeyRingLocation = "global" +) + +// gkmsConfig contains the GCE specific KMS configuration for setting up a KMS connection. +type gkmsConfig struct { + Global struct { + // location is the KMS location of the KeyRing to be used for encryption. + // It can be found by checking the available KeyRings in the IAM UI. + // This is not the same as the GCP location of the project. + // +optional + Location string `gcfg:"kms-location"` + // keyRing is the keyRing of the hosted key to be used. The default value is "google-kubernetes". + // +optional + KeyRing string `gcfg:"kms-keyring"` + // cryptoKey is the name of the key to be used for encryption of Data-Encryption-Keys. + CryptoKey string `gcfg:"kms-cryptokey"` + } +} + +// readGCPCloudKMSConfig parses and returns the configuration parameters for Google Cloud KMS. +func readGCPCloudKMSConfig(reader io.Reader) (*gkmsConfig, error) { + cfg := &gkmsConfig{} + if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil { + glog.Errorf("Couldn't read Google Cloud KMS config: %v", err) + return nil, err + } + return cfg, nil +} + +// gkmsService provides Encrypt and Decrypt methods which allow cryptographic operations +// using Google Cloud KMS service. +type gkmsService struct { + parentName string + cloudkmsService *cloudkms.Service +} + +// getGCPCloudKMSService provides a Google Cloud KMS based implementation of envelope.Service. +func (gce *GCECloud) getGCPCloudKMSService(config io.Reader) (envelope.Service, error) { + kmsConfig, err := readGCPCloudKMSConfig(config) + if err != nil { + return nil, err + } + + // Hosting on GCE/GKE with Google KMS encryption provider + cloudkmsService := gce.GetKMSService() + + // Set defaults for location and keyRing. + location := kmsConfig.Global.Location + if len(location) == 0 { + location = defaultGKMSKeyRingLocation + } + keyRing := kmsConfig.Global.KeyRing + if len(keyRing) == 0 { + keyRing = defaultGKMSKeyRing + } + + cryptoKey := kmsConfig.Global.CryptoKey + if len(cryptoKey) == 0 { + return nil, fmt.Errorf("missing cryptoKey for cloudprovided KMS: " + KMSServiceName) + } + + parentName := fmt.Sprintf("projects/%s/locations/%s", gce.projectID, location) + + // Create the keyRing if it does not exist yet + _, err = cloudkmsService.Projects.Locations.KeyRings.Create(parentName, + &cloudkms.KeyRing{}).KeyRingId(keyRing).Do() + if err != nil && unrecoverableCreationError(err) { + return nil, err + } + parentName = parentName + "/keyRings/" + keyRing + + // Create the cryptoKey if it does not exist yet + _, err = cloudkmsService.Projects.Locations.KeyRings.CryptoKeys.Create(parentName, + &cloudkms.CryptoKey{ + Purpose: "ENCRYPT_DECRYPT", + }).CryptoKeyId(cryptoKey).Do() + if err != nil && unrecoverableCreationError(err) { + return nil, err + } + parentName = parentName + "/cryptoKeys/" + cryptoKey + + service := &gkmsService{ + parentName: parentName, + cloudkmsService: cloudkmsService, + } + + // Sanity check before startup. For non-GCP clusters, the user's account may not have permissions to create + // the key. We need to verify the existence of the key before apiserver startup. + _, err = service.Encrypt([]byte("test")) + if err != nil { + return nil, fmt.Errorf("failed to encrypt data using Google cloudkms, using key %s. Ensure that the keyRing and cryptoKey exist. Got error: %v", parentName, err) + } + + return service, nil +} + +// Decrypt decrypts a base64 representation of encrypted bytes. +func (t *gkmsService) Decrypt(data string) ([]byte, error) { + resp, err := t.cloudkmsService.Projects.Locations.KeyRings.CryptoKeys. + Decrypt(t.parentName, &cloudkms.DecryptRequest{ + Ciphertext: data, + }).Do() + if err != nil { + return nil, err + } + return base64.StdEncoding.DecodeString(resp.Plaintext) +} + +// Encrypt encrypts bytes, and returns base64 representation of the ciphertext. +func (t *gkmsService) Encrypt(data []byte) (string, error) { + resp, err := t.cloudkmsService.Projects.Locations.KeyRings.CryptoKeys. + Encrypt(t.parentName, &cloudkms.EncryptRequest{ + Plaintext: base64.StdEncoding.EncodeToString(data), + }).Do() + if err != nil { + return "", err + } + return resp.Ciphertext, nil +} + +// unrecoverableCreationError decides if Kubernetes should ignore the encountered Google KMS +// error. Only to be used for errors seen while creating a KeyRing or CryptoKey. +func unrecoverableCreationError(err error) bool { + apiError, isAPIError := err.(*googleapi.Error) + // 409 means the object exists. + // 403 means we do not have permission to create the object, the user must do it. + // Else, it is an unrecoverable error. + if !isAPIError || (apiError.Code != 409 && apiError.Code != 403) { + return true + } + return false +} diff --git a/pkg/cloudprovider/providers/gce/metrics.go b/pkg/cloudprovider/providers/gce/metrics.go index 16536898f9b5f..649bd11b132b3 100644 --- a/pkg/cloudprovider/providers/gce/metrics.go +++ b/pkg/cloudprovider/providers/gce/metrics.go @@ -22,21 +22,33 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +const ( + // Version strings for recording metrics. + computeV1Version = "v1" + computeAlphaVersion = "alpha" + computeBetaVersion = "beta" +) + type apiCallMetrics struct { latency *prometheus.HistogramVec errors *prometheus.CounterVec } var ( - apiMetrics = registerAPIMetrics( + metricLabels = []string{ "request", // API function that is begin invoked. "region", // region (optional). "zone", // zone (optional). - ) + "version", // API version. + } + + apiMetrics = registerAPIMetrics(metricLabels...) ) type metricContext struct { - start time.Time + start time.Time + // The cardinalities of attributes and metricLabels (defined above) must + // match, or prometheus will panic. attributes []string } @@ -54,6 +66,13 @@ func (mc *metricContext) Observe(err error) error { return err } +func newGenericMetricContext(prefix, request, region, zone, version string) *metricContext { + return &metricContext{ + start: time.Now(), + attributes: []string{prefix + "_" + request, region, zone, version}, + } +} + // registerApiMetrics adds metrics definitions for a category of API calls. func registerAPIMetrics(attributes ...string) *apiCallMetrics { metrics := &apiCallMetrics{ diff --git a/federation/apis/federation/v1beta1/conversion.go b/pkg/cloudprovider/providers/gce/metrics_test.go similarity index 54% rename from federation/apis/federation/v1beta1/conversion.go rename to pkg/cloudprovider/providers/gce/metrics_test.go index a6a8a867e8eb2..89fde2ac4b663 100644 --- a/federation/apis/federation/v1beta1/conversion.go +++ b/pkg/cloudprovider/providers/gce/metrics_test.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,23 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package gce import ( - "fmt" + "testing" - "k8s.io/apimachinery/pkg/runtime" + "github.com/stretchr/testify/assert" ) -func addConversionFuncs(scheme *runtime.Scheme) error { - return scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.String(), "Cluster", - func(label, value string) (string, string, error) { - switch label { - case "metadata.name": - return label, value, nil - default: - return "", "", fmt.Errorf("field label not supported: %s", label) - } - }, - ) +func TestVerifyMetricLabelCardinality(t *testing.T) { + mc := newGenericMetricContext("foo", "get", "us-central1", "", "alpha") + assert.Len(t, mc.attributes, len(metricLabels), "cardinalities of labels and values must match") } diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 14f70c297b8bc..0233892321a13 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -49,7 +49,10 @@ import ( "k8s.io/kubernetes/pkg/controller" ) -const ProviderName = "openstack" +const ( + ProviderName = "openstack" + AvailabilityZone = "availability_zone" +) var ErrNotFound = errors.New("Failed to find object") var ErrMultipleResults = errors.New("Multiple results where only one expected") @@ -534,6 +537,7 @@ func (os *OpenStack) Zones() (cloudprovider.Zones, bool) { return os, true } + func (os *OpenStack) GetZone() (cloudprovider.Zone, error) { md, err := getMetadata() if err != nil { @@ -549,6 +553,60 @@ func (os *OpenStack) GetZone() (cloudprovider.Zone, error) { return zone, nil } +// GetZoneByProviderID implements Zones.GetZoneByProviderID +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (os *OpenStack) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) { + instanceID, err := instanceIDFromProviderID(providerID) + if err != nil { + return cloudprovider.Zone{}, err + } + + compute, err := os.NewComputeV2() + if err != nil { + return cloudprovider.Zone{}, err + } + + srv, err := servers.Get(compute, instanceID).Extract() + if err != nil { + return cloudprovider.Zone{}, err + } + + zone := cloudprovider.Zone{ + FailureDomain: srv.Metadata[AvailabilityZone], + Region: os.region, + } + glog.V(4).Infof("The instance %s in zone %v", srv.Name, zone) + + return zone, nil +} + +// GetZoneByNodeName implements Zones.GetZoneByNodeName +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (os *OpenStack) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) { + compute, err := os.NewComputeV2() + if err != nil { + return cloudprovider.Zone{}, err + } + + srv, err := getServerByName(compute, nodeName) + if err != nil { + if err == ErrNotFound { + return cloudprovider.Zone{}, cloudprovider.InstanceNotFound + } + return cloudprovider.Zone{}, err + } + + zone := cloudprovider.Zone{ + FailureDomain: srv.Metadata[AvailabilityZone], + Region: os.region, + } + glog.V(4).Infof("The instance %s in zone %v", srv.Name, zone) + + return zone, nil +} + func (os *OpenStack) Routes() (cloudprovider.Routes, bool) { glog.V(4).Info("openstack.Routes() called") diff --git a/pkg/cloudprovider/providers/openstack/openstack_instances.go b/pkg/cloudprovider/providers/openstack/openstack_instances.go index 5f010918116e5..a9037117b2527 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_instances.go +++ b/pkg/cloudprovider/providers/openstack/openstack_instances.go @@ -19,7 +19,7 @@ package openstack import ( "errors" "fmt" - "net/url" + "regexp" "github.com/golang/glog" "github.com/gophercloud/gophercloud" @@ -110,6 +110,12 @@ func (i *Instances) ExternalID(name types.NodeName) (string, error) { return srv.ID, nil } +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (i *Instances) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + // InstanceID returns the kubelet's cloud provider ID. func (os *OpenStack) InstanceID() (string, error) { if len(os.localInstanceID) == 0 { @@ -126,6 +132,9 @@ func (os *OpenStack) InstanceID() (string, error) { func (i *Instances) InstanceID(name types.NodeName) (string, error) { srv, err := getServerByName(i.compute, name) if err != nil { + if err == ErrNotFound { + return "", cloudprovider.InstanceNotFound + } return "", err } // In the future it is possible to also return an endpoint as: @@ -177,14 +186,16 @@ func srvInstanceType(srv *servers.Server) (string, error) { return "", fmt.Errorf("flavor name/id not found") } +// instanceIDFromProviderID splits a provider's id and return instanceID. +// A providerID is build out of '${ProviderName}:///${instance-id}'which contains ':///'. +// See cloudprovider.GetInstanceProviderID and Instances.InstanceID. func instanceIDFromProviderID(providerID string) (instanceID string, err error) { - parsedID, err := url.Parse(providerID) - if err != nil { - return "", err - } - if parsedID.Scheme != ProviderName { - return "", fmt.Errorf("unrecognized provider %q", parsedID.Scheme) - } + // If Instances.InstanceID or cloudprovider.GetInstanceProviderID is changed, the regexp should be changed too. + var providerIdRegexp = regexp.MustCompile(`^` + ProviderName + `:///([^/]+)$`) - return parsedID.Host, nil + matches := providerIdRegexp.FindStringSubmatch(providerID) + if len(matches) != 2 { + return "", fmt.Errorf("ProviderID \"%s\" didn't match expected format \"openstack:///InstanceID\"", providerID) + } + return matches[1], nil } diff --git a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go index 610ef37c988ce..dc8df75a29a9f 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go +++ b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go @@ -68,6 +68,11 @@ const ( errorStatus = "ERROR" ServiceAnnotationLoadBalancerFloatingNetworkId = "loadbalancer.openstack.org/floating-network-id" + + // ServiceAnnotationLoadBalancerInternal is the annotation used on the service + // to indicate that we want an internal loadbalancer service. + // If the value of ServiceAnnotationLoadBalancerInternal is false, it indicates that we want an external loadbalancer service. Default to true. + ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/openstack-internal-load-balancer" ) // LoadBalancer implementation for LBaaS v1 @@ -501,7 +506,7 @@ func createNodeSecurityGroup(client *gophercloud.ServiceClient, nodeSecurityGrou return nil } -func (lbaas *LbaasV2) createLoadBalancer(service *v1.Service, name string) (*loadbalancers.LoadBalancer, error) { +func (lbaas *LbaasV2) createLoadBalancer(service *v1.Service, name string, internalAnnotation bool) (*loadbalancers.LoadBalancer, error) { createOpts := loadbalancers.CreateOpts{ Name: name, Description: fmt.Sprintf("Kubernetes external service %s", name), @@ -509,7 +514,7 @@ func (lbaas *LbaasV2) createLoadBalancer(service *v1.Service, name string) (*loa } loadBalancerIP := service.Spec.LoadBalancerIP - if loadBalancerIP != "" { + if loadBalancerIP != "" && internalAnnotation { createOpts.VipAddress = loadBalancerIP } @@ -626,6 +631,24 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *v1.Serv floatingPool := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerFloatingNetworkId, lbaas.opts.FloatingNetworkId) glog.V(4).Infof("EnsureLoadBalancer using floatingPool: %v", floatingPool) + var internalAnnotation bool + internal := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerInternal, "true") + switch internal { + case "true": + glog.V(4).Infof("Ensure an internal loadbalancer service.") + internalAnnotation = true + case "false": + if len(floatingPool) != 0 { + glog.V(4).Infof("Ensure an external loadbalancer service.") + internalAnnotation = false + } else { + return nil, fmt.Errorf("floating-network-id or loadbalancer.openstack.org/floating-network-id should be specified when service.beta.kubernetes.io/openstack-internal-load-balancer is false") + } + default: + return nil, fmt.Errorf("unknow service.beta.kubernetes.io/openstack-internal-load-balancer annotation: %v, specify \"true\" or \"false\".", + internal) + } + // Check for TCP protocol on each port // TODO: Convert all error messages to use an event recorder for _, port := range ports { @@ -661,7 +684,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *v1.Serv return nil, fmt.Errorf("Error getting loadbalancer %s: %v", name, err) } glog.V(2).Infof("Creating loadbalancer %s", name) - loadbalancer, err = lbaas.createLoadBalancer(apiService, name) + loadbalancer, err = lbaas.createLoadBalancer(apiService, name, internalAnnotation) if err != nil { // Unknown error, retry later return nil, fmt.Errorf("Error creating loadbalancer %s: %v", name, err) @@ -855,12 +878,18 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *v1.Serv if err != nil && err != ErrNotFound { return nil, fmt.Errorf("Error getting floating ip for port %s: %v", portID, err) } - if floatIP == nil && floatingPool != "" { + if floatIP == nil && floatingPool != "" && !internalAnnotation { glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, portID) floatIPOpts := floatingips.CreateOpts{ FloatingNetworkID: floatingPool, PortID: portID, } + + loadBalancerIP := apiService.Spec.LoadBalancerIP + if loadBalancerIP != "" { + floatIPOpts.FloatingIP = loadBalancerIP + } + floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract() if err != nil { return nil, fmt.Errorf("Error creating LB floatingip %+v: %v", floatIPOpts, err) @@ -1116,7 +1145,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(clusterName string, service *v1. return nil } - if lbaas.opts.FloatingNetworkId != "" && loadbalancer != nil { + if loadbalancer != nil && loadbalancer.VipPortID != "" { portID := loadbalancer.VipPortID floatingIP, err := getFloatingIPByPortID(lbaas.network, portID) if err != nil && err != ErrNotFound { @@ -1144,10 +1173,12 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(clusterName string, service *v1. if err != nil && err != ErrNotFound { return fmt.Errorf("Error getting pool for listener %s: %v", listener.ID, err) } - poolIDs = append(poolIDs, pool.ID) - // If create-monitor of cloud-config is false, pool has not monitor. - if pool.MonitorID != "" { - monitorIDs = append(monitorIDs, pool.MonitorID) + if pool != nil { + poolIDs = append(poolIDs, pool.ID) + // If create-monitor of cloud-config is false, pool has not monitor. + if pool.MonitorID != "" { + monitorIDs = append(monitorIDs, pool.MonitorID) + } } } @@ -1281,6 +1312,27 @@ func (lb *LbaasV1) EnsureLoadBalancer(clusterName string, apiService *v1.Service lb.opts.SubnetId = subnetID } + floatingPool := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerFloatingNetworkId, lb.opts.FloatingNetworkId) + glog.V(4).Infof("EnsureLoadBalancer using floatingPool: %v", floatingPool) + + var internalAnnotation bool + internal := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerInternal, "true") + switch internal { + case "true": + glog.V(4).Infof("Ensure an internal loadbalancer service.") + internalAnnotation = true + case "false": + if len(floatingPool) != 0 { + glog.V(4).Infof("Ensure an external loadbalancer service.") + internalAnnotation = false + } else { + return nil, fmt.Errorf("floating-network-id or loadbalancer.openstack.org/floating-network-id should be specified when service.beta.kubernetes.io/openstack-internal-load-balancer is false") + } + default: + return nil, fmt.Errorf("unknow service.beta.kubernetes.io/openstack-internal-load-balancer annotation: %v, specify \"true\" or \"false\".", + internal) + } + ports := apiService.Spec.Ports if len(ports) > 1 { return nil, fmt.Errorf("multiple ports are not supported in openstack v1 load balancers") @@ -1391,7 +1443,7 @@ func (lb *LbaasV1) EnsureLoadBalancer(clusterName string, apiService *v1.Service } loadBalancerIP := apiService.Spec.LoadBalancerIP - if loadBalancerIP != "" { + if loadBalancerIP != "" && internalAnnotation { createOpts.Address = loadBalancerIP } @@ -1404,11 +1456,17 @@ func (lb *LbaasV1) EnsureLoadBalancer(clusterName string, apiService *v1.Service status.Ingress = []v1.LoadBalancerIngress{{IP: vip.Address}} - if lb.opts.FloatingNetworkId != "" { + if floatingPool != "" && !internalAnnotation { floatIPOpts := floatingips.CreateOpts{ - FloatingNetworkID: lb.opts.FloatingNetworkId, + FloatingNetworkID: floatingPool, PortID: vip.PortID, } + + loadBalancerIP := apiService.Spec.LoadBalancerIP + if loadBalancerIP != "" { + floatIPOpts.FloatingIP = loadBalancerIP + } + floatIP, err := floatingips.Create(lb.network, floatIPOpts).Extract() if err != nil { return nil, fmt.Errorf("Error creating floatingip for openstack load balancer %s: %v", name, err) @@ -1492,7 +1550,7 @@ func (lb *LbaasV1) EnsureLoadBalancerDeleted(clusterName string, service *v1.Ser return err } - if lb.opts.FloatingNetworkId != "" && vip != nil { + if vip != nil && vip.PortID != "" { floatingIP, err := getFloatingIPByPortID(lb.network, vip.PortID) if err != nil && !isNotFound(err) { return err diff --git a/pkg/cloudprovider/providers/openstack/openstack_test.go b/pkg/cloudprovider/providers/openstack/openstack_test.go index 7e8ed5901b3c9..3306d69584a94 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_test.go +++ b/pkg/cloudprovider/providers/openstack/openstack_test.go @@ -557,17 +557,22 @@ func TestInstanceIDFromProviderID(t *testing.T) { fail bool }{ { - providerID: "openstack://7b9cf879-7146-417c-abfd-cb4272f0c935", + providerID: ProviderName + "://" + "/" + "7b9cf879-7146-417c-abfd-cb4272f0c935", instanceID: "7b9cf879-7146-417c-abfd-cb4272f0c935", fail: false, }, + { + providerID: "openstack://7b9cf879-7146-417c-abfd-cb4272f0c935", + instanceID: "", + fail: true, + }, { providerID: "7b9cf879-7146-417c-abfd-cb4272f0c935", instanceID: "", fail: true, }, { - providerID: "other-provider://7b9cf879-7146-417c-abfd-cb4272f0c935", + providerID: "other-provider:///7b9cf879-7146-417c-abfd-cb4272f0c935", instanceID: "", fail: true, }, diff --git a/pkg/cloudprovider/providers/ovirt/ovirt.go b/pkg/cloudprovider/providers/ovirt/ovirt.go index fe71113ed0af7..d255979412627 100644 --- a/pkg/cloudprovider/providers/ovirt/ovirt.go +++ b/pkg/cloudprovider/providers/ovirt/ovirt.go @@ -211,6 +211,12 @@ func (v *OVirtCloud) ExternalID(nodeName types.NodeName) (string, error) { return instance.UUID, nil } +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (v *OVirtCloud) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + // InstanceID returns the cloud provider ID of the node with the specified NodeName. func (v *OVirtCloud) InstanceID(nodeName types.NodeName) (string, error) { name := mapNodeNameToInstanceName(nodeName) diff --git a/pkg/cloudprovider/providers/photon/photon.go b/pkg/cloudprovider/providers/photon/photon.go index e9c9fbc87a719..3173409978689 100644 --- a/pkg/cloudprovider/providers/photon/photon.go +++ b/pkg/cloudprovider/providers/photon/photon.go @@ -470,6 +470,12 @@ func (pc *PCCloud) ExternalID(nodeName k8stypes.NodeName) (string, error) { } } +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (pc *PCCloud) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + // InstanceID returns the cloud provider ID of the specified instance. func (pc *PCCloud) InstanceID(nodeName k8stypes.NodeName) (string, error) { name := string(nodeName) @@ -521,6 +527,20 @@ func (pc *PCCloud) GetZone() (cloudprovider.Zone, error) { return pc.Zone, nil } +// GetZoneByProviderID implements Zones.GetZoneByProviderID +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (pc *PCCloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented") +} + +// GetZoneByNodeName implements Zones.GetZoneByNodeName +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (pc *PCCloud) GetZoneByNodeName(nodeName k8stypes.NodeName) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not imeplemented") +} + // Routes returns a false since the interface is not supported for photon controller. func (pc *PCCloud) Routes() (cloudprovider.Routes, bool) { return nil, false diff --git a/pkg/cloudprovider/providers/rackspace/rackspace.go b/pkg/cloudprovider/providers/rackspace/rackspace.go index b087bfa4cea49..18fce399dc195 100644 --- a/pkg/cloudprovider/providers/rackspace/rackspace.go +++ b/pkg/cloudprovider/providers/rackspace/rackspace.go @@ -436,6 +436,12 @@ func (i *Instances) ExternalID(nodeName types.NodeName) (string, error) { return probeInstanceID(i.compute, serverName) } +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (i *Instances) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + // InstanceID returns the cloud provider ID of the kubelet's instance. func (rs *Rackspace) InstanceID() (string, error) { return readInstanceID() @@ -554,6 +560,20 @@ func (os *Rackspace) GetZone() (cloudprovider.Zone, error) { return cloudprovider.Zone{Region: os.region}, nil } +// GetZoneByProviderID implements Zones.GetZoneByProviderID +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (os *Rackspace) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented") +} + +// GetZoneByNodeName implements Zones.GetZoneByNodeName +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (os *Rackspace) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not imeplemented") +} + // Create a volume of given size (in GiB) func (rs *Rackspace) CreateVolume(name string, size int, vtype, availability string, tags *map[string]string) (string, string, error) { return "", "", errors.New("unimplemented") diff --git a/pkg/cloudprovider/providers/vsphere/vclib/utils.go b/pkg/cloudprovider/providers/vsphere/vclib/utils.go index 8b80b4a4482f4..3ae00e8c660ad 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/utils.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/utils.go @@ -27,6 +27,21 @@ import ( "github.com/vmware/govmomi/vim25/types" ) +// IsNotFound return true if err is NotFoundError or DefaultNotFoundError +func IsNotFound(err error) bool { + _, ok := err.(*find.NotFoundError) + if ok { + return true + } + + _, ok = err.(*find.DefaultNotFoundError) + if ok { + return true + } + + return false +} + func getFinder(dc *Datacenter) *find.Finder { finder := find.NewFinder(dc.Client(), true) finder.SetDatacenter(dc.Datacenter) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go index aed47f3c56607..91d70535b8f6f 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go @@ -188,9 +188,9 @@ func (vm *VirtualMachine) GetResourcePool(ctx context.Context) (*object.Resource return object.NewResourcePool(vm.Client(), vmMoList[0].ResourcePool.Reference()), nil } -// Exists checks if the VM exists. -// Returns false if VM doesn't exist or VM is in powerOff state. -func (vm *VirtualMachine) Exists(ctx context.Context) (bool, error) { +// IsActive checks if the VM is active. +// Returns true if VM is in poweredOn state. +func (vm *VirtualMachine) IsActive(ctx context.Context) (bool, error) { vmMoList, err := vm.Datacenter.GetVMMoList(ctx, []*VirtualMachine{vm}, []string{"summary"}) if err != nil { glog.Errorf("Failed to get VM Managed object with property summary. err: +%v", err) @@ -199,11 +199,7 @@ func (vm *VirtualMachine) Exists(ctx context.Context) (bool, error) { if vmMoList[0].Summary.Runtime.PowerState == ActivePowerState { return true, nil } - if vmMoList[0].Summary.Config.Template == false { - glog.Warningf("VM is not in %s state", ActivePowerState) - } else { - glog.Warningf("VM is a template") - } + return false, nil } diff --git a/pkg/cloudprovider/providers/vsphere/vsphere.go b/pkg/cloudprovider/providers/vsphere/vsphere.go index 60086d52297d5..3f0e66582101d 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere.go @@ -376,6 +376,12 @@ func (vs *VSphere) ExternalID(nodeName k8stypes.NodeName) (string, error) { return vs.InstanceID(nodeName) } +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (vs *VSphere) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + // InstanceID returns the cloud provider ID of the node with the specified Name. func (vs *VSphere) InstanceID(nodeName k8stypes.NodeName) (string, error) { if vs.localInstanceID == nodeNameToVMName(nodeName) { @@ -391,18 +397,22 @@ func (vs *VSphere) InstanceID(nodeName k8stypes.NodeName) (string, error) { } vm, err := vs.getVMByName(ctx, nodeName) if err != nil { + if vclib.IsNotFound(err) { + return "", cloudprovider.InstanceNotFound + } glog.Errorf("Failed to get VM object for node: %q. err: +%v", nodeNameToVMName(nodeName), err) return "", err } - nodeExist, err := vm.Exists(ctx) + isActive, err := vm.IsActive(ctx) if err != nil { - glog.Errorf("Failed to check whether node %q exist. err: %+v.", nodeNameToVMName(nodeName), err) + glog.Errorf("Failed to check whether node %q is active. err: %+v.", nodeNameToVMName(nodeName), err) return "", err } - if nodeExist { + if isActive { return "/" + vm.InventoryPath, nil } - return "", cloudprovider.InstanceNotFound + + return "", fmt.Errorf("The node %q is not active", nodeNameToVMName(nodeName)) } // InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID @@ -494,6 +504,12 @@ func (vs *VSphere) DetachDisk(volPath string, nodeName k8stypes.NodeName) error } vm, err := vs.getVMByName(ctx, nodeName) if err != nil { + // If node doesn't exist, disk is already detached from node. + if vclib.IsNotFound(err) { + glog.Infof("Node %q does not exist, disk %s is already detached from node.", nodeNameToVMName(nodeName), volPath) + return nil + } + glog.Errorf("Failed to get VM object for node: %q. err: +%v", nodeNameToVMName(nodeName), err) return err } @@ -530,22 +546,15 @@ func (vs *VSphere) DiskIsAttached(volPath string, nodeName k8stypes.NodeName) (b } vm, err := vs.getVMByName(ctx, nodeName) if err != nil { + if vclib.IsNotFound(err) { + glog.Warningf("Node %q does not exist, vsphere CP will assume disk %v is not attached to it.", nodeName, volPath) + // make the disk as detached and return false without error. + return false, nil + } glog.Errorf("Failed to get VM object for node: %q. err: +%v", vSphereInstance, err) return false, err } - nodeExist, err := vm.Exists(ctx) - if err != nil { - glog.Errorf("Failed to check whether node %q exist. err: %+v", vSphereInstance, err) - return false, err - } - if !nodeExist { - glog.Errorf("DiskIsAttached failed to determine whether disk %q is still attached: node %q is powered off", - volPath, - vSphereInstance) - return false, fmt.Errorf("DiskIsAttached failed to determine whether disk %q is still attached: node %q is powered off", - volPath, - vSphereInstance) - } + attached, err := vm.IsDiskAttached(ctx, volPath) if err != nil { glog.Errorf("DiskIsAttached failed to determine whether disk %q is still attached on node %q", @@ -584,22 +593,19 @@ func (vs *VSphere) DisksAreAttached(volPaths []string, nodeName k8stypes.NodeNam } vm, err := vs.getVMByName(ctx, nodeName) if err != nil { + if vclib.IsNotFound(err) { + glog.Warningf("Node %q does not exist, vsphere CP will assume all disks %v are not attached to it.", nodeName, volPaths) + // make all the disks as detached and return false without error. + attached := make(map[string]bool) + for _, volPath := range volPaths { + attached[volPath] = false + } + return attached, nil + } glog.Errorf("Failed to get VM object for node: %q. err: +%v", vSphereInstance, err) return nil, err } - nodeExist, err := vm.Exists(ctx) - if err != nil { - glog.Errorf("Failed to check whether node %q exist. err: %+v", vSphereInstance, err) - return nil, err - } - if !nodeExist { - glog.Errorf("DisksAreAttached failed to determine whether disks %v are still attached: node %q does not exist", - volPaths, - vSphereInstance) - return nil, fmt.Errorf("DisksAreAttached failed to determine whether disks %v are still attached: node %q does not exist", - volPaths, - vSphereInstance) - } + for _, volPath := range volPaths { result, err := vm.IsDiskAttached(ctx, volPath) if err == nil { diff --git a/pkg/controller/cloud/OWNERS b/pkg/controller/cloud/OWNERS new file mode 100644 index 0000000000000..5a58eee8cea76 --- /dev/null +++ b/pkg/controller/cloud/OWNERS @@ -0,0 +1,8 @@ +approvers: +- thockin +- luxas +- wlan0 +reviewers: +- thockin +- luxas +- wlan0 diff --git a/pkg/controller/cloud/node_controller.go b/pkg/controller/cloud/node_controller.go index 26e1036781f2d..0accda13d4e9e 100644 --- a/pkg/controller/cloud/node_controller.go +++ b/pkg/controller/cloud/node_controller.go @@ -228,7 +228,7 @@ func (cnc *CloudNodeController) MonitorNode() { time.Sleep(retrySleepTime) } if currentReadyCondition == nil { - glog.Errorf("Update status of Node %v from CloudNodeController exceeds retry count.", node.Name) + glog.Errorf("Update status of Node %v from CloudNodeController exceeds retry count or the Node was deleted.", node.Name) continue } // If the known node status says that Node is NotReady, then check if the node has been removed @@ -237,32 +237,42 @@ func (cnc *CloudNodeController) MonitorNode() { if currentReadyCondition.Status != v1.ConditionTrue { // Check with the cloud provider to see if the node still exists. If it // doesn't, delete the node immediately. - if _, err := instances.ExternalID(types.NodeName(node.Name)); err != nil { - if err == cloudprovider.InstanceNotFound { - glog.V(2).Infof("Deleting node no longer present in cloud provider: %s", node.Name) - ref := &v1.ObjectReference{ - Kind: "Node", - Name: node.Name, - UID: types.UID(node.UID), - Namespace: "", - } - glog.V(2).Infof("Recording %s event message for node %s", "DeletingNode", node.Name) - cnc.recorder.Eventf(ref, v1.EventTypeNormal, fmt.Sprintf("Deleting Node %v because it's not present according to cloud provider", node.Name), "Node %s event: %s", node.Name, "DeletingNode") - go func(nodeName string) { - defer utilruntime.HandleCrash() - if err := cnc.kubeClient.CoreV1().Nodes().Delete(node.Name, nil); err != nil { - glog.Errorf("unable to delete node %q: %v", node.Name, err) - } - }(node.Name) - } - glog.Errorf("Error getting node data from cloud: %v", err) + exists, err := ensureNodeExistsByProviderIDOrExternalID(instances, node) + if err != nil { + glog.Errorf("Error getting data for node %s from cloud: %v", node.Name, err) + continue + } + + if exists { + // Continue checking the remaining nodes since the current one is fine. + continue + } + + glog.V(2).Infof("Deleting node since it is no longer present in cloud provider: %s", node.Name) + + ref := &v1.ObjectReference{ + Kind: "Node", + Name: node.Name, + UID: types.UID(node.UID), + Namespace: "", } + glog.V(2).Infof("Recording %s event message for node %s", "DeletingNode", node.Name) + + cnc.recorder.Eventf(ref, v1.EventTypeNormal, fmt.Sprintf("Deleting Node %v because it's not present according to cloud provider", node.Name), "Node %s event: %s", node.Name, "DeletingNode") + + go func(nodeName string) { + defer utilruntime.HandleCrash() + if err := cnc.kubeClient.CoreV1().Nodes().Delete(nodeName, nil); err != nil { + glog.Errorf("unable to delete node %q: %v", nodeName, err) + } + }(node.Name) + } } } } -// This processes nodes that were added into the cluster, and cloud initializea them if appropriate +// This processes nodes that were added into the cluster, and cloud initialize them if appropriate func (cnc *CloudNodeController) AddCloudNode(obj interface{}) { node := obj.(*v1.Node) @@ -284,6 +294,18 @@ func (cnc *CloudNodeController) AddCloudNode(obj interface{}) { return err } + if curNode.Spec.ProviderID == "" { + providerID, err := cloudprovider.GetInstanceProviderID(cnc.cloud, types.NodeName(curNode.Name)) + if err == nil { + curNode.Spec.ProviderID = providerID + } else { + // we should attempt to set providerID on curNode, but + // we can continue if we fail since we will attempt to set + // node addresses given the node name in getNodeAddressesByProviderIDOrName + glog.Errorf("failed to set node provider id: %v", err) + } + } + nodeAddresses, err := getNodeAddressesByProviderIDOrName(instances, curNode) if err != nil { glog.Errorf("%v", err) @@ -321,7 +343,7 @@ func (cnc *CloudNodeController) AddCloudNode(obj interface{}) { } if zones, ok := cnc.cloud.Zones(); ok { - zone, err := zones.GetZone() + zone, err := getZoneByProviderIDOrName(zones, curNode) if err != nil { return fmt.Errorf("failed to get zone from cloud provider: %v", err) } @@ -372,6 +394,25 @@ func excludeTaintFromList(taints []v1.Taint, toExclude v1.Taint) []v1.Taint { return newTaints } +// ensureNodeExistsByProviderIDOrExternalID first checks if the instance exists by the provider id and then by calling external id with node name +func ensureNodeExistsByProviderIDOrExternalID(instances cloudprovider.Instances, node *v1.Node) (bool, error) { + exists, err := instances.InstanceExistsByProviderID(node.Spec.ProviderID) + if err != nil { + providerIDErr := err + _, err = instances.ExternalID(types.NodeName(node.Name)) + if err == nil { + return true, nil + } + if err == cloudprovider.InstanceNotFound { + return false, nil + } + + return false, fmt.Errorf("InstanceExistsByProviderID: Error fetching by providerID: %v Error fetching by NodeName: %v", providerIDErr, err) + } + + return exists, nil +} + func getNodeAddressesByProviderIDOrName(instances cloudprovider.Instances, node *v1.Node) ([]v1.NodeAddress, error) { nodeAddresses, err := instances.NodeAddressesByProviderID(node.Spec.ProviderID) if err != nil { @@ -430,3 +471,18 @@ func getInstanceTypeByProviderIDOrName(instances cloudprovider.Instances, node * } return instanceType, err } + +// getZoneByProviderIDorName will attempt to get the zone of node using its providerID +// then it's name. If both attempts fail, an error is returned +func getZoneByProviderIDOrName(zones cloudprovider.Zones, node *v1.Node) (cloudprovider.Zone, error) { + zone, err := zones.GetZoneByProviderID(node.Spec.ProviderID) + if err != nil { + providerIDErr := err + zone, err = zones.GetZoneByNodeName(types.NodeName(node.Name)) + if err != nil { + return cloudprovider.Zone{}, fmt.Errorf("Zone: Error fetching by providerID: %v Error fetching by NodeName: %v", providerIDErr, err) + } + } + + return zone, nil +} diff --git a/pkg/controller/cloud/node_controller_test.go b/pkg/controller/cloud/node_controller_test.go index 033dbe9088900..7c3a0074ea65a 100644 --- a/pkg/controller/cloud/node_controller_test.go +++ b/pkg/controller/cloud/node_controller_test.go @@ -17,6 +17,8 @@ limitations under the License. package cloud import ( + "errors" + "reflect" "testing" "time" @@ -39,6 +41,119 @@ import ( "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" ) +func TestEnsureNodeExistsByProviderIDOrNodeName(t *testing.T) { + + testCases := []struct { + testName string + node *v1.Node + expectedCalls []string + existsByNodeName bool + existsByProviderID bool + nodeNameErr error + providerIDErr error + }{ + { + testName: "node exists by provider id", + existsByProviderID: true, + providerIDErr: nil, + existsByNodeName: false, + nodeNameErr: errors.New("unimplemented"), + expectedCalls: []string{"instance-exists-by-provider-id"}, + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + }, + Spec: v1.NodeSpec{ + ProviderID: "node0", + }, + }, + }, + { + testName: "does not exist by provider id", + existsByProviderID: false, + providerIDErr: nil, + existsByNodeName: false, + nodeNameErr: errors.New("unimplemented"), + expectedCalls: []string{"instance-exists-by-provider-id"}, + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + }, + Spec: v1.NodeSpec{ + ProviderID: "node0", + }, + }, + }, + { + testName: "node exists by node name", + existsByProviderID: false, + providerIDErr: errors.New("unimplemented"), + existsByNodeName: true, + nodeNameErr: nil, + expectedCalls: []string{"instance-exists-by-provider-id", "external-id"}, + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + }, + Spec: v1.NodeSpec{ + ProviderID: "node0", + }, + }, + }, + { + testName: "does not exist by node name", + existsByProviderID: false, + providerIDErr: errors.New("unimplemented"), + existsByNodeName: false, + nodeNameErr: cloudprovider.InstanceNotFound, + expectedCalls: []string{"instance-exists-by-provider-id", "external-id"}, + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + }, + Spec: v1.NodeSpec{ + ProviderID: "node0", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + fc := &fakecloud.FakeCloud{ + Exists: tc.existsByNodeName, + ExistsByProviderID: tc.existsByProviderID, + Err: tc.nodeNameErr, + ErrByProviderID: tc.providerIDErr, + } + + instances, _ := fc.Instances() + exists, err := ensureNodeExistsByProviderIDOrExternalID(instances, tc.node) + if err != nil { + t.Error(err) + } + + if !reflect.DeepEqual(fc.Calls, tc.expectedCalls) { + t.Errorf("expected cloud provider methods `%v` to be called but `%v` was called ", tc.expectedCalls, fc.Calls) + } + + if tc.existsByProviderID && tc.existsByProviderID != exists { + t.Errorf("expected exist by provider id to be `%t` but got `%t`", tc.existsByProviderID, exists) + } + + if tc.existsByNodeName && tc.existsByNodeName != exists { + t.Errorf("expected exist by node name to be `%t` but got `%t`", tc.existsByNodeName, exists) + } + + if !tc.existsByNodeName && !tc.existsByProviderID && exists { + t.Error("node is not supposed to exist") + } + + }) + } + +} + // This test checks that the node is deleted when kubelet stops reporting // and cloud provider says node is gone func TestNodeDeleted(t *testing.T) { @@ -105,9 +220,12 @@ func TestNodeDeleted(t *testing.T) { eventBroadcaster := record.NewBroadcaster() cloudNodeController := &CloudNodeController{ - kubeClient: fnh, - nodeInformer: factory.Core().V1().Nodes(), - cloud: &fakecloud.FakeCloud{Err: cloudprovider.InstanceNotFound}, + kubeClient: fnh, + nodeInformer: factory.Core().V1().Nodes(), + cloud: &fakecloud.FakeCloud{ + ExistsByProviderID: false, + Err: nil, + }, nodeMonitorPeriod: 1 * time.Second, recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-controller-manager"}), nodeStatusUpdateFrequency: 1 * time.Second, @@ -520,7 +638,8 @@ func TestNodeAddresses(t *testing.T) { FailureDomain: "us-west-1a", Region: "us-west", }, - Err: nil, + ExistsByProviderID: true, + Err: nil, } eventBroadcaster := record.NewBroadcaster() @@ -639,7 +758,8 @@ func TestNodeProvidedIPAddresses(t *testing.T) { FailureDomain: "us-west-1a", Region: "us-west", }, - Err: nil, + ExistsByProviderID: true, + Err: nil, } eventBroadcaster := record.NewBroadcaster() @@ -804,3 +924,179 @@ func TestNodeAddressesChangeDetected(t *testing.T) { t.Errorf("Node address changes are not detected correctly") } } + +// This test checks that a node is set with the correct providerID +func TestNodeProviderID(t *testing.T) { + fnh := &testutil.FakeNodeHandler{ + Existing: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: map[string]string{}, + }, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionUnknown, + LastHeartbeatTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), + LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), + }, + }, + }, + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: "ImproveCoverageTaint", + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }, + { + Key: algorithm.TaintExternalCloudProvider, + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + Clientset: fake.NewSimpleClientset(&v1.PodList{}), + DeleteWaitChan: make(chan struct{}), + } + + factory := informers.NewSharedInformerFactory(fnh, controller.NoResyncPeriodFunc()) + + fakeCloud := &fakecloud.FakeCloud{ + InstanceTypes: map[types.NodeName]string{}, + Addresses: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: "node0.cloud.internal", + }, + { + Type: v1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: v1.NodeExternalIP, + Address: "132.143.154.163", + }, + }, + Provider: "test", + ExtID: map[types.NodeName]string{ + types.NodeName("node0"): "12345", + }, + Err: nil, + } + + eventBroadcaster := record.NewBroadcaster() + cloudNodeController := &CloudNodeController{ + kubeClient: fnh, + nodeInformer: factory.Core().V1().Nodes(), + cloud: fakeCloud, + nodeMonitorPeriod: 5 * time.Second, + nodeStatusUpdateFrequency: 1 * time.Second, + recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-controller-manager"}), + } + eventBroadcaster.StartLogging(glog.Infof) + + cloudNodeController.AddCloudNode(fnh.Existing[0]) + + if len(fnh.UpdatedNodes) != 1 || fnh.UpdatedNodes[0].Name != "node0" { + t.Errorf("Node was not updated") + } + + if fnh.UpdatedNodes[0].Spec.ProviderID != "test://12345" { + t.Errorf("Node ProviderID not set correctly") + } +} + +// This test checks that a node's provider ID will not be overwritten +func TestNodeProviderIDAlreadySet(t *testing.T) { + fnh := &testutil.FakeNodeHandler{ + Existing: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: map[string]string{}, + }, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionUnknown, + LastHeartbeatTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), + LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), + }, + }, + }, + Spec: v1.NodeSpec{ + ProviderID: "test-provider-id", + Taints: []v1.Taint{ + { + Key: "ImproveCoverageTaint", + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }, + { + Key: algorithm.TaintExternalCloudProvider, + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + Clientset: fake.NewSimpleClientset(&v1.PodList{}), + DeleteWaitChan: make(chan struct{}), + } + + factory := informers.NewSharedInformerFactory(fnh, controller.NoResyncPeriodFunc()) + + fakeCloud := &fakecloud.FakeCloud{ + InstanceTypes: map[types.NodeName]string{}, + Addresses: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: "node0.cloud.internal", + }, + { + Type: v1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: v1.NodeExternalIP, + Address: "132.143.154.163", + }, + }, + Provider: "test", + ExtID: map[types.NodeName]string{ + types.NodeName("node0"): "12345", + }, + Err: nil, + } + + eventBroadcaster := record.NewBroadcaster() + cloudNodeController := &CloudNodeController{ + kubeClient: fnh, + nodeInformer: factory.Core().V1().Nodes(), + cloud: fakeCloud, + nodeMonitorPeriod: 5 * time.Second, + nodeStatusUpdateFrequency: 1 * time.Second, + recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cloud-controller-manager"}), + } + eventBroadcaster.StartLogging(glog.Infof) + + cloudNodeController.AddCloudNode(fnh.Existing[0]) + + if len(fnh.UpdatedNodes) != 1 || fnh.UpdatedNodes[0].Name != "node0" { + t.Errorf("Node was not updated") + } + + // CCM node controller should not overwrite provider if it's already set + if fnh.UpdatedNodes[0].Spec.ProviderID != "test-provider-id" { + t.Errorf("Node ProviderID not set correctly") + } +} diff --git a/pkg/controller/controller_utils.go b/pkg/controller/controller_utils.go index d331ccf3f6ed6..4fcd8a1885ce0 100644 --- a/pkg/controller/controller_utils.go +++ b/pkg/controller/controller_utils.go @@ -711,8 +711,8 @@ func (s ByLogging) Less(i, j int) bool { return maxContainerRestarts(s[i]) > maxContainerRestarts(s[j]) } // 6. older pods < newer pods < empty timestamp pods - if !s[i].CreationTimestamp.Equal(s[j].CreationTimestamp) { - return afterOrZero(s[j].CreationTimestamp, s[i].CreationTimestamp) + if !s[i].CreationTimestamp.Equal(&s[j].CreationTimestamp) { + return afterOrZero(&s[j].CreationTimestamp, &s[i].CreationTimestamp) } return false } @@ -751,31 +751,31 @@ func (s ActivePods) Less(i, j int) bool { return maxContainerRestarts(s[i]) > maxContainerRestarts(s[j]) } // 6. Empty creation time pods < newer pods < older pods - if !s[i].CreationTimestamp.Equal(s[j].CreationTimestamp) { - return afterOrZero(s[i].CreationTimestamp, s[j].CreationTimestamp) + if !s[i].CreationTimestamp.Equal(&s[j].CreationTimestamp) { + return afterOrZero(&s[i].CreationTimestamp, &s[j].CreationTimestamp) } return false } // afterOrZero checks if time t1 is after time t2; if one of them // is zero, the zero time is seen as after non-zero time. -func afterOrZero(t1, t2 metav1.Time) bool { +func afterOrZero(t1, t2 *metav1.Time) bool { if t1.Time.IsZero() || t2.Time.IsZero() { return t1.Time.IsZero() } return t1.After(t2.Time) } -func podReadyTime(pod *v1.Pod) metav1.Time { +func podReadyTime(pod *v1.Pod) *metav1.Time { if podutil.IsPodReady(pod) { for _, c := range pod.Status.Conditions { // we only care about pod ready conditions if c.Type == v1.PodReady && c.Status == v1.ConditionTrue { - return c.LastTransitionTime + return &c.LastTransitionTime } } } - return metav1.Time{} + return &metav1.Time{} } func maxContainerRestarts(pod *v1.Pod) int { @@ -841,10 +841,10 @@ type ControllersByCreationTimestamp []*v1.ReplicationController func (o ControllersByCreationTimestamp) Len() int { return len(o) } func (o ControllersByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o ControllersByCreationTimestamp) Less(i, j int) bool { - if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { return o[i].Name < o[j].Name } - return o[i].CreationTimestamp.Before(o[j].CreationTimestamp) + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) } // ReplicaSetsByCreationTimestamp sorts a list of ReplicaSet by creation timestamp, using their names as a tie breaker. @@ -853,10 +853,10 @@ type ReplicaSetsByCreationTimestamp []*extensions.ReplicaSet func (o ReplicaSetsByCreationTimestamp) Len() int { return len(o) } func (o ReplicaSetsByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o ReplicaSetsByCreationTimestamp) Less(i, j int) bool { - if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { return o[i].Name < o[j].Name } - return o[i].CreationTimestamp.Before(o[j].CreationTimestamp) + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) } // ReplicaSetsBySizeOlder sorts a list of ReplicaSet by size in descending order, using their creation timestamp or name as a tie breaker. @@ -1030,14 +1030,14 @@ func WaitForCacheSync(controllerName string, stopCh <-chan struct{}, cacheSyncs } // ComputeHash returns a hash value calculated from pod template and a collisionCount to avoid hash collision -func ComputeHash(template *v1.PodTemplateSpec, collisionCount *int64) uint32 { +func ComputeHash(template *v1.PodTemplateSpec, collisionCount *int32) uint32 { podTemplateSpecHasher := fnv.New32a() hashutil.DeepHashObject(podTemplateSpecHasher, *template) // Add collisionCount in the hash if it exists. if collisionCount != nil { collisionCountBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(collisionCountBytes, uint64(*collisionCount)) + binary.LittleEndian.PutUint32(collisionCountBytes, uint32(*collisionCount)) podTemplateSpecHasher.Write(collisionCountBytes) } diff --git a/pkg/controller/controller_utils_test.go b/pkg/controller/controller_utils_test.go index 0b3842c53834d..a72359585fafe 100644 --- a/pkg/controller/controller_utils_test.go +++ b/pkg/controller/controller_utils_test.go @@ -453,23 +453,26 @@ func int64P(num int64) *int64 { } func TestComputeHash(t *testing.T) { + collisionCount := int32(1) + otherCollisionCount := int32(2) + maxCollisionCount := int32(math.MaxInt32) tests := []struct { name string template *v1.PodTemplateSpec - collisionCount *int64 - otherCollisionCount *int64 + collisionCount *int32 + otherCollisionCount *int32 }{ { name: "simple", template: &v1.PodTemplateSpec{}, - collisionCount: int64P(1), - otherCollisionCount: int64P(2), + collisionCount: &collisionCount, + otherCollisionCount: &otherCollisionCount, }, { name: "using math.MaxInt64", template: &v1.PodTemplateSpec{}, collisionCount: nil, - otherCollisionCount: int64P(int64(math.MaxInt64)), + otherCollisionCount: &maxCollisionCount, }, } diff --git a/pkg/controller/cronjob/utils.go b/pkg/controller/cronjob/utils.go index dbdd8dd96c8db..abee870998393 100644 --- a/pkg/controller/cronjob/utils.go +++ b/pkg/controller/cronjob/utils.go @@ -169,8 +169,6 @@ func getRecentUnmetScheduleTimes(sj batchv2alpha1.CronJob, now time.Time) ([]tim return starts, nil } -// XXX unit test this - // getJobFromTemplate makes a Job from a CronJob func getJobFromTemplate(sj *batchv2alpha1.CronJob, scheduledTime time.Time) (*batchv1.Job, error) { // TODO: consider adding the following labels: @@ -200,7 +198,7 @@ func getJobFromTemplate(sj *batchv2alpha1.CronJob, scheduledTime time.Time) (*ba return job, nil } -// Return Unix Epoch Time +// getTimeHash returns Unix Epoch Time func getTimeHash(scheduledTime time.Time) int64 { return scheduledTime.Unix() } @@ -251,9 +249,9 @@ func (o byJobStartTime) Less(i, j int) bool { return o[i].Status.StartTime != nil } - if (*o[i].Status.StartTime).Equal(*o[j].Status.StartTime) { + if o[i].Status.StartTime.Equal(o[j].Status.StartTime) { return o[i].Name < o[j].Name } - return (*o[i].Status.StartTime).Before(*o[j].Status.StartTime) + return o[i].Status.StartTime.Before(o[j].Status.StartTime) } diff --git a/pkg/controller/daemon/BUILD b/pkg/controller/daemon/BUILD index acf6eb0d66ef6..29d9999169b00 100644 --- a/pkg/controller/daemon/BUILD +++ b/pkg/controller/daemon/BUILD @@ -72,6 +72,8 @@ go_test( "//pkg/securitycontext:go_default_library", "//pkg/util/labels:go_default_library", "//plugin/pkg/scheduler/algorithm:go_default_library", + "//plugin/pkg/scheduler/algorithm/predicates:go_default_library", + "//plugin/pkg/scheduler/schedulercache:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", diff --git a/pkg/controller/daemon/daemon_controller.go b/pkg/controller/daemon/daemon_controller.go index 93673f07766cd..8c4eac1564d07 100644 --- a/pkg/controller/daemon/daemon_controller.go +++ b/pkg/controller/daemon/daemon_controller.go @@ -1351,7 +1351,7 @@ func NodeConditionPredicates(nodeInfo *schedulercache.NodeInfo) (bool, []algorit // TODO: There are other node status that the DaemonSet should ideally respect too, // e.g. MemoryPressure, and DiskPressure if c.Type == v1.NodeOutOfDisk && c.Status == v1.ConditionTrue { - reasons = append(reasons, predicates.ErrNodeSelectorNotMatch) + reasons = append(reasons, predicates.ErrNodeOutOfDisk) break } } @@ -1366,10 +1366,10 @@ func (o byCreationTimestamp) Len() int { return len(o) } func (o byCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o byCreationTimestamp) Less(i, j int) bool { - if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { return o[i].Name < o[j].Name } - return o[i].CreationTimestamp.Before(o[j].CreationTimestamp) + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) } type podByCreationTimestamp []*v1.Pod @@ -1378,8 +1378,8 @@ func (o podByCreationTimestamp) Len() int { return len(o) } func (o podByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o podByCreationTimestamp) Less(i, j int) bool { - if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { return o[i].Name < o[j].Name } - return o[i].CreationTimestamp.Before(o[j].CreationTimestamp) + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) } diff --git a/pkg/controller/daemon/daemon_controller_test.go b/pkg/controller/daemon/daemon_controller_test.go index f8e49d48854ca..a708a4b997927 100644 --- a/pkg/controller/daemon/daemon_controller_test.go +++ b/pkg/controller/daemon/daemon_controller_test.go @@ -47,6 +47,8 @@ import ( "k8s.io/kubernetes/pkg/securitycontext" labelsutil "k8s.io/kubernetes/pkg/util/labels" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" + "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates" + "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" ) var ( @@ -1394,6 +1396,7 @@ func setDaemonSetCritical(ds *extensions.DaemonSet) { func TestNodeShouldRunDaemonPod(t *testing.T) { cases := []struct { podsOnNode []*v1.Pod + nodeCondition []v1.NodeCondition ds *extensions.DaemonSet wantToRun, shouldSchedule, shouldContinueRunning bool err error @@ -1414,6 +1417,23 @@ func TestNodeShouldRunDaemonPod(t *testing.T) { shouldSchedule: true, shouldContinueRunning: true, }, + { + ds: &extensions.DaemonSet{ + Spec: extensions.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: simpleDaemonSetLabel}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: simpleDaemonSetLabel, + }, + Spec: resourcePodSpec("", "50M", "0.5"), + }, + }, + }, + nodeCondition: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}}, + wantToRun: true, + shouldSchedule: false, + shouldContinueRunning: true, + }, { ds: &extensions.DaemonSet{ Spec: extensions.DaemonSetSpec{ @@ -1484,6 +1504,7 @@ func TestNodeShouldRunDaemonPod(t *testing.T) { for i, c := range cases { for _, strategy := range updateStrategies() { node := newNode("test-node", nil) + node.Status.Conditions = append(node.Status.Conditions, c.nodeCondition...) node.Status.Allocatable = allocatableResources("100M", "1") manager, _, _ := newTestController() manager.nodeStore.Add(node) @@ -2153,3 +2174,57 @@ func getQueuedKeys(queue workqueue.RateLimitingInterface) []string { sort.Strings(keys) return keys } + +func TestPredicates(t *testing.T) { + type args struct { + pod *v1.Pod + node *v1.Node + } + tests := []struct { + name string + args args + want bool + wantRes []algorithm.PredicateFailureReason + wantErr bool + }{ + { + name: "retrun OutOfDiskErr if outOfDisk", + args: args{ + pod: newPod("pod1-", "node-0", nil, nil), + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-0", + }, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + {Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}, + }, + Allocatable: v1.ResourceList{ + v1.ResourcePods: resource.MustParse("100"), + }, + }, + }, + }, + want: false, + wantRes: []algorithm.PredicateFailureReason{predicates.ErrNodeOutOfDisk}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nodeInfo := schedulercache.NewNodeInfo(tt.args.pod) + nodeInfo.SetNode(tt.args.node) + + got, res, err := Predicates(tt.args.pod, nodeInfo) + if (err != nil) != tt.wantErr { + t.Errorf("%s (error): error = %v, wantErr %v", tt.name, err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("%s (fit): got = %v, want %v", tt.name, got, tt.want) + } + if !reflect.DeepEqual(res, tt.wantRes) { + t.Errorf("%s (reasons): got = %v, want %v", tt.name, res, tt.wantRes) + } + }) + } +} diff --git a/pkg/controller/daemon/update.go b/pkg/controller/daemon/update.go index 72486594f0e84..49ff90ce787be 100644 --- a/pkg/controller/daemon/update.go +++ b/pkg/controller/daemon/update.go @@ -368,7 +368,7 @@ func (dsc *DaemonSetsController) snapshot(ds *extensions.DaemonSet, revision int return nil, getErr } if currDS.Status.CollisionCount == nil { - currDS.Status.CollisionCount = new(int64) + currDS.Status.CollisionCount = new(int32) } *currDS.Status.CollisionCount++ _, updateErr := dsc.kubeClient.ExtensionsV1beta1().DaemonSets(ds.Namespace).UpdateStatus(currDS) diff --git a/pkg/controller/deployment/deployment_controller.go b/pkg/controller/deployment/deployment_controller.go index 5dc5a8a773e53..610a69c7cbae3 100644 --- a/pkg/controller/deployment/deployment_controller.go +++ b/pkg/controller/deployment/deployment_controller.go @@ -620,14 +620,6 @@ func (dc *DeploymentController) syncDeployment(key string) error { return err } - _, err = dc.hasFailed(d, rsList, podMap) - if err != nil { - return err - } - // TODO: Automatically rollback here if we failed above. Locate the last complete - // revision and populate the rollback spec with it. - // See https://github.com/kubernetes/kubernetes/issues/23211. - if d.Spec.Paused { return dc.sync(d, rsList, podMap) } diff --git a/pkg/controller/deployment/progress.go b/pkg/controller/deployment/progress.go index 3a1425be6c9b9..1062d7edc7ac4 100644 --- a/pkg/controller/deployment/progress.go +++ b/pkg/controller/deployment/progress.go @@ -25,54 +25,9 @@ import ( "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" - "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/controller/deployment/util" ) -// hasFailed determines if a deployment has failed or not by estimating its progress. -// Progress for a deployment is considered when a new replica set is created or adopted, -// and when new pods scale up or old pods scale down. Progress is not estimated for paused -// deployments or when users don't really care about it ie. progressDeadlineSeconds is not -// specified. -func (dc *DeploymentController) hasFailed(d *extensions.Deployment, rsList []*extensions.ReplicaSet, podMap map[types.UID]*v1.PodList) (bool, error) { - if d.Spec.ProgressDeadlineSeconds == nil || d.Spec.RollbackTo != nil || d.Spec.Paused { - return false, nil - } - - newRS, oldRSs, err := dc.getAllReplicaSetsAndSyncRevision(d, rsList, podMap, false) - if err != nil { - return false, err - } - - // There is a template change so we don't need to check for any progress right now. - if newRS == nil { - return false, nil - } - - // Look at the status of the deployment - if there is already a NewRSAvailableReason - // then we don't need to estimate any progress. This is needed in order to avoid - // estimating progress for scaling events after a rollout has finished. - cond := util.GetDeploymentCondition(d.Status, extensions.DeploymentProgressing) - if cond != nil && cond.Reason == util.NewRSAvailableReason { - return false, nil - } - - // TODO: Look for permanent failures here. - // See https://github.com/kubernetes/kubernetes/issues/18568 - - allRSs := append(oldRSs, newRS) - newStatus := calculateStatus(allRSs, newRS, d) - - // If the deployment is complete or it is progressing, there is no need to check if it - // has timed out. - if util.DeploymentComplete(d, &newStatus) || util.DeploymentProgressing(d, &newStatus) { - return false, nil - } - - // Check if the deployment has timed out. - return util.DeploymentTimedOut(d, &newStatus), nil -} - // syncRolloutStatus updates the status of a deployment during a rollout. There are // cases this helper will run that cannot be prevented from the scaling detection, // for example a resync of the deployment after it was scaled up. In those cases, diff --git a/pkg/controller/deployment/sync.go b/pkg/controller/deployment/sync.go index a1a85a5a8f708..3731f21d32979 100644 --- a/pkg/controller/deployment/sync.go +++ b/pkg/controller/deployment/sync.go @@ -160,7 +160,7 @@ func (dc *DeploymentController) rsAndPodsWithHashKeySynced(d *extensions.Deploym // 1. Add hash label to the rs's pod template, and make sure the controller sees this update so that no orphaned pods will be created // 2. Add hash label to all pods this rs owns, wait until replicaset controller reports rs.Status.FullyLabeledReplicas equal to the desired number of replicas // 3. Add hash label to the rs's label and selector -func (dc *DeploymentController) addHashKeyToRSAndPods(rs *extensions.ReplicaSet, podList *v1.PodList, collisionCount *int64) (*extensions.ReplicaSet, error) { +func (dc *DeploymentController) addHashKeyToRSAndPods(rs *extensions.ReplicaSet, podList *v1.PodList, collisionCount *int32) (*extensions.ReplicaSet, error) { // If the rs already has the new hash label in its selector, it's done syncing if labelsutil.SelectorHasLabel(rs.Spec.Selector, extensions.DefaultDeploymentUniqueLabelKey) { return rs, nil @@ -349,7 +349,7 @@ func (dc *DeploymentController) getNewReplicaSet(d *extensions.Deployment, rsLis // and requeue the Deployment. if !isEqual { if d.Status.CollisionCount == nil { - d.Status.CollisionCount = new(int64) + d.Status.CollisionCount = new(int32) } preCollisionCount := *d.Status.CollisionCount *d.Status.CollisionCount++ diff --git a/pkg/controller/deployment/util/replicaset_util.go b/pkg/controller/deployment/util/replicaset_util.go index f2a2fad1cf222..0b7c0d36917a7 100644 --- a/pkg/controller/deployment/util/replicaset_util.go +++ b/pkg/controller/deployment/util/replicaset_util.go @@ -70,7 +70,7 @@ func UpdateRSWithRetries(rsClient unversionedextensions.ReplicaSetInterface, rsL } // GetReplicaSetHash returns the pod template hash of a ReplicaSet's pod template space -func GetReplicaSetHash(rs *extensions.ReplicaSet, uniquifier *int64) (string, error) { +func GetReplicaSetHash(rs *extensions.ReplicaSet, uniquifier *int32) (string, error) { template, err := scheme.Scheme.DeepCopy(rs.Spec.Template) if err != nil { return "", err diff --git a/pkg/controller/disruption/disruption.go b/pkg/controller/disruption/disruption.go index 0f27683d98276..a6495728b54da 100644 --- a/pkg/controller/disruption/disruption.go +++ b/pkg/controller/disruption/disruption.go @@ -107,7 +107,7 @@ type controllerAndScale struct { // podControllerFinder is a function type that maps a pod to a list of // controllers and their scale. -type podControllerFinder func(*v1.Pod) ([]controllerAndScale, error) +type podControllerFinder func(*v1.Pod) (*controllerAndScale, error) func NewDisruptionController( podInformer coreinformers.PodInformer, @@ -170,8 +170,8 @@ func NewDisruptionController( // and we may well need further tweaks just to be able to access scale // subresources. func (dc *DisruptionController) finders() []podControllerFinder { - return []podControllerFinder{dc.getPodReplicationControllers, dc.getPodDeployments, dc.getPodReplicaSets, - dc.getPodStatefulSets} + return []podControllerFinder{dc.getPodReplicationController, dc.getPodDeployment, dc.getPodReplicaSet, + dc.getPodStatefulSet} } var ( @@ -181,9 +181,8 @@ var ( controllerKindDep = v1beta1.SchemeGroupVersion.WithKind("Deployment") ) -// getPodReplicaSets finds replicasets which have no matching deployments. -func (dc *DisruptionController) getPodReplicaSets(pod *v1.Pod) ([]controllerAndScale, error) { - var casSlice []controllerAndScale +// getPodReplicaSet finds a replicaset which has no matching deployments. +func (dc *DisruptionController) getPodReplicaSet(pod *v1.Pod) (*controllerAndScale, error) { controllerRef := metav1.GetControllerOf(pod) if controllerRef == nil { return nil, nil @@ -204,13 +203,11 @@ func (dc *DisruptionController) getPodReplicaSets(pod *v1.Pod) ([]controllerAndS // Skip RS if it's controlled by a Deployment. return nil, nil } - casSlice = append(casSlice, controllerAndScale{rs.UID, *(rs.Spec.Replicas)}) - return casSlice, nil + return &controllerAndScale{rs.UID, *(rs.Spec.Replicas)}, nil } // getPodStatefulSet returns the statefulset managing the given pod. -func (dc *DisruptionController) getPodStatefulSets(pod *v1.Pod) ([]controllerAndScale, error) { - var casSlice []controllerAndScale +func (dc *DisruptionController) getPodStatefulSet(pod *v1.Pod) (*controllerAndScale, error) { controllerRef := metav1.GetControllerOf(pod) if controllerRef == nil { return nil, nil @@ -227,13 +224,11 @@ func (dc *DisruptionController) getPodStatefulSets(pod *v1.Pod) ([]controllerAnd return nil, nil } - casSlice = append(casSlice, controllerAndScale{ss.UID, *(ss.Spec.Replicas)}) - return casSlice, nil + return &controllerAndScale{ss.UID, *(ss.Spec.Replicas)}, nil } // getPodDeployments finds deployments for any replicasets which are being managed by deployments. -func (dc *DisruptionController) getPodDeployments(pod *v1.Pod) ([]controllerAndScale, error) { - var casSlice []controllerAndScale +func (dc *DisruptionController) getPodDeployment(pod *v1.Pod) (*controllerAndScale, error) { controllerRef := metav1.GetControllerOf(pod) if controllerRef == nil { return nil, nil @@ -264,12 +259,10 @@ func (dc *DisruptionController) getPodDeployments(pod *v1.Pod) ([]controllerAndS if deployment.UID != controllerRef.UID { return nil, nil } - casSlice = append(casSlice, controllerAndScale{deployment.UID, *(deployment.Spec.Replicas)}) - return casSlice, nil + return &controllerAndScale{deployment.UID, *(deployment.Spec.Replicas)}, nil } -func (dc *DisruptionController) getPodReplicationControllers(pod *v1.Pod) ([]controllerAndScale, error) { - var casSlice []controllerAndScale +func (dc *DisruptionController) getPodReplicationController(pod *v1.Pod) (*controllerAndScale, error) { controllerRef := metav1.GetControllerOf(pod) if controllerRef == nil { return nil, nil @@ -285,8 +278,7 @@ func (dc *DisruptionController) getPodReplicationControllers(pod *v1.Pod) ([]con if rc.UID != controllerRef.UID { return nil, nil } - casSlice = append(casSlice, controllerAndScale{rc.UID, *(rc.Spec.Replicas)}) - return casSlice, nil + return &controllerAndScale{rc.UID, *(rc.Spec.Replicas)}, nil } func (dc *DisruptionController) Run(stopCh <-chan struct{}) { @@ -590,30 +582,26 @@ func (dc *DisruptionController) getExpectedScale(pdb *policy.PodDisruptionBudget // A mapping from controllers to their scale. controllerScale := map[types.UID]int32{} - // 1. Find the controller(s) for each pod. If any pod has 0 controllers, - // that's an error. If any pod has more than 1 controller, that's also an - // error. + // 1. Find the controller for each pod. If any pod has 0 controllers, + // that's an error. With ControllerRef, a pod can only have 1 controller. for _, pod := range pods { - controllerCount := 0 + foundController := false for _, finder := range dc.finders() { - var controllers []controllerAndScale - controllers, err = finder(pod) + var controllerNScale *controllerAndScale + controllerNScale, err = finder(pod) if err != nil { return } - for _, controller := range controllers { - controllerScale[controller.UID] = controller.scale - controllerCount++ + if controllerNScale != nil { + controllerScale[controllerNScale.UID] = controllerNScale.scale + foundController = true + break } } - if controllerCount == 0 { + if !foundController { err = fmt.Errorf("found no controllers for pod %q", pod.Name) dc.recorder.Event(pdb, v1.EventTypeWarning, "NoControllers", err.Error()) return - } else if controllerCount > 1 { - err = fmt.Errorf("pod %q has %v>1 controllers", pod.Name, controllerCount) - dc.recorder.Event(pdb, v1.EventTypeWarning, "TooManyControllers", err.Error()) - return } } diff --git a/pkg/controller/endpoint/BUILD b/pkg/controller/endpoint/BUILD index 3202f12e557f6..f18fde133cd15 100644 --- a/pkg/controller/endpoint/BUILD +++ b/pkg/controller/endpoint/BUILD @@ -50,6 +50,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", diff --git a/pkg/controller/endpoint/endpoints_controller.go b/pkg/controller/endpoint/endpoints_controller.go index f0b0a4e912903..59b5de6953b27 100644 --- a/pkg/controller/endpoint/endpoints_controller.go +++ b/pkg/controller/endpoint/endpoints_controller.go @@ -202,6 +202,56 @@ func (e *EndpointController) addPod(obj interface{}) { } } +func podToEndpointAddress(pod *v1.Pod) *v1.EndpointAddress { + return &v1.EndpointAddress{ + IP: pod.Status.PodIP, + NodeName: &pod.Spec.NodeName, + TargetRef: &v1.ObjectReference{ + Kind: "Pod", + Namespace: pod.ObjectMeta.Namespace, + Name: pod.ObjectMeta.Name, + UID: pod.ObjectMeta.UID, + ResourceVersion: pod.ObjectMeta.ResourceVersion, + }} +} + +func podChanged(oldPod, newPod *v1.Pod) bool { + // If the pod's readiness has changed, the associated endpoint address + // will move from the unready endpoints set to the ready endpoints. + // So for the purposes of an endpoint, a readiness change on a pod + // means we have a changed pod. + if podutil.IsPodReady(oldPod) != podutil.IsPodReady(newPod) { + return true + } + // Convert the pod to an EndpointAddress, clear inert fields, + // and see if they are the same. + newEndpointAddress := podToEndpointAddress(newPod) + oldEndpointAddress := podToEndpointAddress(oldPod) + // Ignore the ResourceVersion because it changes + // with every pod update. This allows the comparison to + // show equality if all other relevant fields match. + newEndpointAddress.TargetRef.ResourceVersion = "" + oldEndpointAddress.TargetRef.ResourceVersion = "" + if reflect.DeepEqual(newEndpointAddress, oldEndpointAddress) { + // The pod has not changed in any way that impacts the endpoints + return false + } + return true +} + +func determineNeededServiceUpdates(oldServices, services sets.String, podChanged bool) sets.String { + if podChanged { + // if the labels and pod changed, all services need to be updated + services = services.Union(oldServices) + } else { + // if only the labels changed, services not common to + // both the new and old service set (i.e the disjunctive union) + // need to be updated + services = services.Difference(oldServices).Union(oldServices.Difference(services)) + } + return services +} + // When a pod is updated, figure out what services it used to be a member of // and what services it will be a member of, and enqueue the union of these. // old and cur must be *v1.Pod types. @@ -213,22 +263,37 @@ func (e *EndpointController) updatePod(old, cur interface{}) { // Two different versions of the same pod will always have different RVs. return } + + podChangedFlag := podChanged(oldPod, newPod) + + // Check if the pod labels have changed, indicating a possibe + // change in the service membership + labelsChanged := false + if !reflect.DeepEqual(newPod.Labels, oldPod.Labels) || + !hostNameAndDomainAreEqual(newPod, oldPod) { + labelsChanged = true + } + + // If both the pod and labels are unchanged, no update is needed + if !podChangedFlag && !labelsChanged { + return + } + services, err := e.getPodServiceMemberships(newPod) if err != nil { utilruntime.HandleError(fmt.Errorf("Unable to get pod %v/%v's service memberships: %v", newPod.Namespace, newPod.Name, err)) return } - // Only need to get the old services if the labels changed. - if !reflect.DeepEqual(newPod.Labels, oldPod.Labels) || - !hostNameAndDomainAreEqual(newPod, oldPod) { + if labelsChanged { oldServices, err := e.getPodServiceMemberships(oldPod) if err != nil { utilruntime.HandleError(fmt.Errorf("Unable to get pod %v/%v's service memberships: %v", oldPod.Namespace, oldPod.Name, err)) return } - services = services.Union(oldServices) + services = determineNeededServiceUpdates(oldServices, services, podChangedFlag) } + for key := range services { e.queue.Add(key) } @@ -376,16 +441,7 @@ func (e *EndpointController) syncService(key string) error { continue } - epa := v1.EndpointAddress{ - IP: pod.Status.PodIP, - NodeName: &pod.Spec.NodeName, - TargetRef: &v1.ObjectReference{ - Kind: "Pod", - Namespace: pod.ObjectMeta.Namespace, - Name: pod.ObjectMeta.Name, - UID: pod.ObjectMeta.UID, - ResourceVersion: pod.ObjectMeta.ResourceVersion, - }} + epa := *podToEndpointAddress(pod) hostname := pod.Spec.Hostname if len(hostname) > 0 && pod.Spec.Subdomain == service.Name && service.Namespace == pod.Namespace { diff --git a/pkg/controller/endpoint/endpoints_controller_test.go b/pkg/controller/endpoint/endpoints_controller_test.go index ad4007ca3c7d6..496aec14be08d 100644 --- a/pkg/controller/endpoint/endpoints_controller_test.go +++ b/pkg/controller/endpoint/endpoints_controller_test.go @@ -27,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" @@ -956,3 +957,149 @@ func TestShouldPodBeInEndpoints(t *testing.T) { } } } + +func TestPodToEndpointAddress(t *testing.T) { + podStore := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc) + ns := "test" + addPods(podStore, ns, 1, 1, 0) + pods := podStore.List() + if len(pods) != 1 { + t.Errorf("podStore size: expected: %d, got: %d", 1, len(pods)) + return + } + pod := pods[0].(*v1.Pod) + epa := podToEndpointAddress(pod) + if epa.IP != pod.Status.PodIP { + t.Errorf("IP: expected: %s, got: %s", pod.Status.PodIP, epa.IP) + } + if *(epa.NodeName) != pod.Spec.NodeName { + t.Errorf("NodeName: expected: %s, got: %s", pod.Spec.NodeName, *(epa.NodeName)) + } + if epa.TargetRef.Kind != "Pod" { + t.Errorf("TargetRef.Kind: expected: %s, got: %s", "Pod", epa.TargetRef.Kind) + } + if epa.TargetRef.Namespace != pod.ObjectMeta.Namespace { + t.Errorf("TargetRef.Kind: expected: %s, got: %s", pod.ObjectMeta.Namespace, epa.TargetRef.Namespace) + } + if epa.TargetRef.Name != pod.ObjectMeta.Name { + t.Errorf("TargetRef.Kind: expected: %s, got: %s", pod.ObjectMeta.Name, epa.TargetRef.Name) + } + if epa.TargetRef.UID != pod.ObjectMeta.UID { + t.Errorf("TargetRef.Kind: expected: %s, got: %s", pod.ObjectMeta.UID, epa.TargetRef.UID) + } + if epa.TargetRef.ResourceVersion != pod.ObjectMeta.ResourceVersion { + t.Errorf("TargetRef.Kind: expected: %s, got: %s", pod.ObjectMeta.ResourceVersion, epa.TargetRef.ResourceVersion) + } +} + +func TestPodChanged(t *testing.T) { + podStore := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc) + ns := "test" + addPods(podStore, ns, 1, 1, 0) + pods := podStore.List() + if len(pods) != 1 { + t.Errorf("podStore size: expected: %d, got: %d", 1, len(pods)) + return + } + oldPod := pods[0].(*v1.Pod) + newPod := oldPod.DeepCopy() + + if podChanged(oldPod, newPod) { + t.Errorf("Expected pod to be unchanged for copied pod") + } + + newPod.Spec.NodeName = "changed" + if !podChanged(oldPod, newPod) { + t.Errorf("Expected pod to be changed for pod with NodeName changed") + } + newPod.Spec.NodeName = oldPod.Spec.NodeName + + newPod.ObjectMeta.ResourceVersion = "changed" + if podChanged(oldPod, newPod) { + t.Errorf("Expected pod to be unchanged for pod with only ResourceVersion changed") + } + newPod.ObjectMeta.ResourceVersion = oldPod.ObjectMeta.ResourceVersion + + newPod.Status.PodIP = "1.2.3.1" + if !podChanged(oldPod, newPod) { + t.Errorf("Expected pod to be changed with pod IP address change") + } + newPod.Status.PodIP = oldPod.Status.PodIP + + newPod.ObjectMeta.Name = "wrong-name" + if !podChanged(oldPod, newPod) { + t.Errorf("Expected pod to be changed with pod name change") + } + newPod.ObjectMeta.Name = oldPod.ObjectMeta.Name + + saveConditions := oldPod.Status.Conditions + oldPod.Status.Conditions = nil + if !podChanged(oldPod, newPod) { + t.Errorf("Expected pod to be changed with pod readiness change") + } + oldPod.Status.Conditions = saveConditions +} + +func TestDetermineNeededServiceUpdates(t *testing.T) { + testCases := []struct { + name string + a sets.String + b sets.String + union sets.String + xor sets.String + }{ + { + name: "no services changed", + a: sets.NewString("a", "b", "c"), + b: sets.NewString("a", "b", "c"), + xor: sets.NewString(), + union: sets.NewString("a", "b", "c"), + }, + { + name: "all old services removed, new services added", + a: sets.NewString("a", "b", "c"), + b: sets.NewString("d", "e", "f"), + xor: sets.NewString("a", "b", "c", "d", "e", "f"), + union: sets.NewString("a", "b", "c", "d", "e", "f"), + }, + { + name: "all old services removed, no new services added", + a: sets.NewString("a", "b", "c"), + b: sets.NewString(), + xor: sets.NewString("a", "b", "c"), + union: sets.NewString("a", "b", "c"), + }, + { + name: "no old services, but new services added", + a: sets.NewString(), + b: sets.NewString("a", "b", "c"), + xor: sets.NewString("a", "b", "c"), + union: sets.NewString("a", "b", "c"), + }, + { + name: "one service removed, one service added, two unchanged", + a: sets.NewString("a", "b", "c"), + b: sets.NewString("b", "c", "d"), + xor: sets.NewString("a", "d"), + union: sets.NewString("a", "b", "c", "d"), + }, + { + name: "no services", + a: sets.NewString(), + b: sets.NewString(), + xor: sets.NewString(), + union: sets.NewString(), + }, + } + for _, testCase := range testCases { + retval := determineNeededServiceUpdates(testCase.a, testCase.b, false) + if !retval.Equal(testCase.xor) { + t.Errorf("%s (with podChanged=false): expected: %v got: %v", testCase.name, testCase.xor.List(), retval.List()) + } + + retval = determineNeededServiceUpdates(testCase.a, testCase.b, true) + if !retval.Equal(testCase.union) { + t.Errorf("%s (with podChanged=true): expected: %v got: %v", testCase.name, testCase.union.List(), retval.List()) + } + } +} diff --git a/pkg/controller/history/controller_history.go b/pkg/controller/history/controller_history.go index e4201c65bc969..16934d6cd4efc 100644 --- a/pkg/controller/history/controller_history.go +++ b/pkg/controller/history/controller_history.go @@ -54,15 +54,17 @@ func ControllerRevisionName(prefix string, hash uint32) string { return fmt.Sprintf("%s-%d", prefix, hash) } -// NewControllerRevision returns the a ControllerRevision with a ControllerRef pointing parent and indicating that +// NewControllerRevision returns a ControllerRevision with a ControllerRef pointing to parent and indicating that // parent is of parentKind. The ControllerRevision has labels matching selector, contains Data equal to data, and -// has a Revision equal to revision. If the returned error is nil, the returned ControllerRevision is valid. If the +// has a Revision equal to revision. The collisionCount is used when creating the name of the ControllerRevision +// so the name is likely unique. If the returned error is nil, the returned ControllerRevision is valid. If the // returned error is not nil, the returned ControllerRevision is invalid for use. func NewControllerRevision(parent metav1.Object, parentKind schema.GroupVersionKind, selector labels.Selector, data runtime.RawExtension, - revision int64) (*apps.ControllerRevision, error) { + revision int64, + collisionCount *int32) (*apps.ControllerRevision, error) { labelMap, err := labels.ConvertSelectorToLabelsMap(selector.String()) if err != nil { return nil, err @@ -86,7 +88,7 @@ func NewControllerRevision(parent metav1.Object, Data: data, Revision: revision, } - hash := HashControllerRevision(cr, nil) + hash := HashControllerRevision(cr, collisionCount) cr.Name = ControllerRevisionName(parent.GetName(), hash) cr.Labels[ControllerRevisionHashLabel] = strconv.FormatInt(int64(hash), 10) return cr, nil @@ -94,7 +96,7 @@ func NewControllerRevision(parent metav1.Object, // HashControllerRevision hashes the contents of revision's Data using FNV hashing. If probe is not nil, the byte value // of probe is added written to the hash as well. -func HashControllerRevision(revision *apps.ControllerRevision, probe *uint32) uint32 { +func HashControllerRevision(revision *apps.ControllerRevision, probe *int32) uint32 { hf := fnv.New32() if len(revision.Data.Raw) > 0 { hf.Write(revision.Data.Raw) @@ -177,11 +179,13 @@ type Interface interface { // returned error is not nil, the returned slice is not valid. ListControllerRevisions(parent metav1.Object, selector labels.Selector) ([]*apps.ControllerRevision, error) // CreateControllerRevision attempts to create the revision as owned by parent via a ControllerRef. If name - // collision occurs, a unique identifier is added to the hash of the revision and it is renamed using - // ControllerRevisionName. Implementations may cease to attempt to retry creation after some number of attempts - // and return an error. If the returned error is not nil, creation failed. If the returned error is nil, the - // returned ControllerRevision has been created. - CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision) (*apps.ControllerRevision, error) + // collision occurs, collisionCount (incremented each time collision occurs except for the first time) is + // added to the hash of the revision and it is renamed using ControllerRevisionName. Implementations may + // cease to attempt to retry creation after some number of attempts and return an error. If the returned + // error is not nil, creation failed. If the returned error is nil, the returned ControllerRevision has been + // created. + // Callers must make sure that collisionCount is not nil. An error is returned if it is. + CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) // DeleteControllerRevision attempts to delete revision. If the returned error is not nil, deletion has failed. DeleteControllerRevision(revision *apps.ControllerRevision) error // UpdateControllerRevision updates revision such that its Revision is equal to newRevision. Implementations @@ -233,9 +237,10 @@ func (rh *realHistory) ListControllerRevisions(parent metav1.Object, selector la return owned, err } -func (rh *realHistory) CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision) (*apps.ControllerRevision, error) { - // Initialize the probe to 0 - probe := uint32(0) +func (rh *realHistory) CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { + if collisionCount == nil { + return nil, fmt.Errorf("collisionCount should not be nil") + } // Clone the input any, err := scheme.Scheme.DeepCopy(revision) @@ -246,18 +251,12 @@ func (rh *realHistory) CreateControllerRevision(parent metav1.Object, revision * // Continue to attempt to create the revision updating the name with a new hash on each iteration for { - var hash uint32 - // The first attempt uses no probe to resolve collisions - if probe > 0 { - hash = HashControllerRevision(revision, &probe) - } else { - hash = HashControllerRevision(revision, nil) - } + hash := HashControllerRevision(revision, collisionCount) // Update the revisions name and labels clone.Name = ControllerRevisionName(parent.GetName(), hash) created, err := rh.client.AppsV1beta1().ControllerRevisions(parent.GetNamespace()).Create(clone) if errors.IsAlreadyExists(err) { - probe++ + *collisionCount++ continue } return created, err @@ -370,9 +369,10 @@ func (fh *fakeHistory) addRevision(revision *apps.ControllerRevision) (*apps.Con return revision, fh.indexer.Update(revision) } -func (fh *fakeHistory) CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision) (*apps.ControllerRevision, error) { - // Initialize the probe to 0 - probe := uint32(0) +func (fh *fakeHistory) CreateControllerRevision(parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { + if collisionCount == nil { + return nil, fmt.Errorf("collisionCount should not be nil") + } // Clone the input any, err := scheme.Scheme.DeepCopy(revision) @@ -384,18 +384,12 @@ func (fh *fakeHistory) CreateControllerRevision(parent metav1.Object, revision * // Continue to attempt to create the revision updating the name with a new hash on each iteration for { - var hash uint32 - // The first attempt uses no probe to resolve collisions - if probe > 0 { - hash = HashControllerRevision(revision, &probe) - } else { - hash = HashControllerRevision(revision, nil) - } + hash := HashControllerRevision(revision, collisionCount) // Update the revisions name and labels clone.Name = ControllerRevisionName(parent.GetName(), hash) created, err := fh.addRevision(clone) if errors.IsAlreadyExists(err) { - probe++ + *collisionCount++ continue } return created, err diff --git a/pkg/controller/history/controller_history_test.go b/pkg/controller/history/controller_history_test.go index ab5845eb34257..5e9c1eadf151e 100644 --- a/pkg/controller/history/controller_history_test.go +++ b/pkg/controller/history/controller_history_test.go @@ -85,22 +85,22 @@ func TestRealHistory_ListControllerRevisions(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, nil) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, nil) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, nil) if err != nil { t.Fatal(err) } ss2Rev1.Namespace = ss2.Namespace - ss1Orphan, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 3) + ss1Orphan, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 3, nil) if err != nil { t.Fatal(err) } @@ -186,22 +186,22 @@ func TestFakeHistory_ListControllerRevisions(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, nil) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, nil) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, nil) if err != nil { t.Fatal(err) } ss2Rev1.Namespace = ss2.Namespace - ss1Orphan, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 3) + ss1Orphan, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 3, nil) if err != nil { t.Fatal(err) } @@ -257,33 +257,53 @@ func TestRealHistory_CreateControllerRevision(t *testing.T) { testFn := func(test *testcase, t *testing.T) { client := fake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) - stop := make(chan struct{}) defer close(stop) informerFactory.Start(stop) informer := informerFactory.Apps().V1beta1().ControllerRevisions() informerFactory.WaitForCacheSync(stop) history := NewHistory(client, informer.Lister()) + + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } } - created, err := history.CreateControllerRevision(test.parent, test.revision) + // Clear collisionCount before creating the test revision + collisionCount = 0 + created, err := history.CreateControllerRevision(test.parent, test.revision, &collisionCount) if err != nil { t.Errorf("%s: %s", test.name, err) } - if test.rename && created.Name == test.revision.Name { - t.Errorf("%s: wanted rename got %s %s", test.name, created.Name, test.revision.Name) + if test.rename { + if created.Name == test.revision.Name { + t.Errorf("%s: wanted rename got %s %s", test.name, created.Name, test.revision.Name) + } + expectedName := ControllerRevisionName(test.parent.GetName(), HashControllerRevision(test.revision, &collisionCount)) + if created.Name != expectedName { + t.Errorf("%s: on name collision wanted new name %s got %s", test.name, expectedName, created.Name) + } + + // Second name collision should have incremented collisionCount to 2 + _, err = history.CreateControllerRevision(test.parent, test.revision, &collisionCount) + if err != nil { + t.Errorf("%s: %s", test.name, err) + } + if collisionCount != 2 { + t.Errorf("%s: on second name collision wanted collisionCount 1 got %d", test.name, collisionCount) + } } if !test.rename && created.Name != test.revision.Name { t.Errorf("%s: wanted %s got %s", test.name, test.revision.Name, created.Name) } } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) ss2 := newStatefulSet(3, "ss2", types.UID("ss2"), map[string]string{"goo": "car"}) + ss2.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) @@ -292,17 +312,17 @@ func TestRealHistory_CreateControllerRevision(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -374,26 +394,47 @@ func TestFakeHistory_CreateControllerRevision(t *testing.T) { informer := informerFactory.Apps().V1beta1().ControllerRevisions() informerFactory.WaitForCacheSync(stop) history := NewFakeHistory(informer) + + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } } - created, err := history.CreateControllerRevision(test.parent, test.revision) + // Clear collisionCount before creating the test revision + collisionCount = 0 + created, err := history.CreateControllerRevision(test.parent, test.revision, &collisionCount) if err != nil { t.Errorf("%s: %s", test.name, err) } - if test.rename && created.Name == test.revision.Name { - t.Errorf("%s: wanted rename got %s %s", test.name, created.Name, test.revision.Name) + if test.rename { + if created.Name == test.revision.Name { + t.Errorf("%s: wanted rename got %s %s", test.name, created.Name, test.revision.Name) + } + expectedName := ControllerRevisionName(test.parent.GetName(), HashControllerRevision(test.revision, &collisionCount)) + if created.Name != expectedName { + t.Errorf("%s: on name collision wanted new name %s got %s", test.name, expectedName, created.Name) + } + + // Second name collision should have incremented collisionCount to 2 + _, err = history.CreateControllerRevision(test.parent, test.revision, &collisionCount) + if err != nil { + t.Errorf("%s: %s", test.name, err) + } + if collisionCount != 2 { + t.Errorf("%s: on second name collision wanted collisionCount 1 got %d", test.name, collisionCount) + } } if !test.rename && created.Name != test.revision.Name { t.Errorf("%s: wanted %s got %s", test.name, test.revision.Name, created.Name) } } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) ss2 := newStatefulSet(3, "ss2", types.UID("ss2"), map[string]string{"goo": "car"}) + ss2.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) @@ -402,17 +443,17 @@ func TestFakeHistory_CreateControllerRevision(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -511,8 +552,9 @@ func TestRealHistory_UpdateControllerRevision(t *testing.T) { informer := informerFactory.Apps().V1beta1().ControllerRevisions() informerFactory.WaitForCacheSync(stop) history := NewHistory(client, informer.Lister()) + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } @@ -532,17 +574,18 @@ func TestRealHistory_UpdateControllerRevision(t *testing.T) { } } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -641,8 +684,9 @@ func TestFakeHistory_UpdateControllerRevision(t *testing.T) { informer := informerFactory.Apps().V1beta1().ControllerRevisions() informerFactory.WaitForCacheSync(stop) history := NewFakeHistory(informer) + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } @@ -659,17 +703,18 @@ func TestFakeHistory_UpdateControllerRevision(t *testing.T) { } } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -731,8 +776,9 @@ func TestRealHistory_DeleteControllerRevision(t *testing.T) { informer := informerFactory.Apps().V1beta1().ControllerRevisions() informerFactory.WaitForCacheSync(stop) history := NewHistory(client, informer.Lister()) + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } @@ -746,7 +792,9 @@ func TestRealHistory_DeleteControllerRevision(t *testing.T) { } } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) ss2 := newStatefulSet(3, "ss2", types.UID("ss2"), map[string]string{"goo": "car"}) + ss2.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) @@ -755,22 +803,22 @@ func TestRealHistory_DeleteControllerRevision(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } ss2Rev1.Namespace = ss2.Namespace - ss2Rev2, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 2) + ss2Rev2, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 2, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -839,8 +887,9 @@ func TestFakeHistory_DeleteControllerRevision(t *testing.T) { informer := informerFactory.Apps().V1beta1().ControllerRevisions() informerFactory.WaitForCacheSync(stop) history := NewFakeHistory(informer) + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } @@ -854,7 +903,9 @@ func TestFakeHistory_DeleteControllerRevision(t *testing.T) { } } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) ss2 := newStatefulSet(3, "ss2", types.UID("ss2"), map[string]string{"goo": "car"}) + ss2.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) @@ -863,22 +914,22 @@ func TestFakeHistory_DeleteControllerRevision(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } ss2Rev1.Namespace = ss2.Namespace - ss2Rev2, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 2) + ss2Rev2, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 2, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -982,8 +1033,9 @@ func TestRealHistory_AdoptControllerRevision(t *testing.T) { informerFactory.WaitForCacheSync(stop) history := NewHistory(client, informer.Lister()) + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } @@ -1001,7 +1053,9 @@ func TestRealHistory_AdoptControllerRevision(t *testing.T) { } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) ss2 := newStatefulSet(3, "ss2", types.UID("ss2"), map[string]string{"goo": "car"}) + ss2.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) @@ -1010,18 +1064,18 @@ func TestRealHistory_AdoptControllerRevision(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace ss1Rev2.OwnerReferences = []metav1.OwnerReference{} - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -1093,8 +1147,9 @@ func TestFakeHistory_AdoptControllerRevision(t *testing.T) { informerFactory.WaitForCacheSync(stop) history := NewFakeHistory(informer) + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } @@ -1112,7 +1167,9 @@ func TestFakeHistory_AdoptControllerRevision(t *testing.T) { } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) ss2 := newStatefulSet(3, "ss2", types.UID("ss2"), map[string]string{"goo": "car"}) + ss2.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) @@ -1121,18 +1178,18 @@ func TestFakeHistory_AdoptControllerRevision(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace ss1Rev2.OwnerReferences = []metav1.OwnerReference{} - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -1243,8 +1300,9 @@ func TestRealHistory_ReleaseControllerRevision(t *testing.T) { informerFactory.WaitForCacheSync(stop) history := NewHistory(client, informer.Lister()) + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } @@ -1276,18 +1334,18 @@ func TestRealHistory_ReleaseControllerRevision(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, nil) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, nil) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace ss1Rev2.OwnerReferences = []metav1.OwnerReference{} - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, nil) if err != nil { t.Fatal(err) } @@ -1371,8 +1429,9 @@ func TestFakeHistory_ReleaseControllerRevision(t *testing.T) { informer := informerFactory.Apps().V1beta1().ControllerRevisions() informerFactory.WaitForCacheSync(stop) history := NewFakeHistory(informer) + var collisionCount int32 for i := range test.existing { - _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision) + _, err := history.CreateControllerRevision(test.existing[i].parent, test.existing[i].revision, &collisionCount) if err != nil { t.Fatal(err) } @@ -1395,7 +1454,9 @@ func TestFakeHistory_ReleaseControllerRevision(t *testing.T) { } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) ss2 := newStatefulSet(3, "ss2", types.UID("ss2"), map[string]string{"goo": "car"}) + ss2.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) @@ -1404,18 +1465,18 @@ func TestFakeHistory_ReleaseControllerRevision(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace ss1Rev2.OwnerReferences = []metav1.OwnerReference{} - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -1499,7 +1560,9 @@ func TestFindEqualRevisions(t *testing.T) { } } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) ss2 := newStatefulSet(3, "ss2", types.UID("ss2"), map[string]string{"goo": "car"}) + ss2.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) @@ -1508,23 +1571,23 @@ func TestFindEqualRevisions(t *testing.T) { if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace ss1Rev2.OwnerReferences = []metav1.OwnerReference{} - ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1) + ss2Rev1, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 1, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } ss2Rev1.Namespace = ss2.Namespace - ss2Rev2, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 2) + ss2Rev2, err := NewControllerRevision(ss2, parentKind, sel2, rawTemplate(&ss2.Spec.Template), 2, ss2.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -1569,22 +1632,23 @@ func TestSortControllerRevisions(t *testing.T) { } } ss1 := newStatefulSet(3, "ss1", types.UID("ss1"), map[string]string{"foo": "bar"}) + ss1.Status.CollisionCount = new(int32) sel1, err := metav1.LabelSelectorAsSelector(ss1.Spec.Selector) if err != nil { t.Fatal(err) } - ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1) + ss1Rev1, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 1, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace - ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev2, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace - ss1Rev3, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2) + ss1Rev3, err := NewControllerRevision(ss1, parentKind, sel1, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } diff --git a/pkg/controller/job/job_controller.go b/pkg/controller/job/job_controller.go index 6ed99e36d63d1..d1e24fc5ac8e0 100644 --- a/pkg/controller/job/job_controller.go +++ b/pkg/controller/job/job_controller.go @@ -82,10 +82,10 @@ func NewJobController(podInformer coreinformers.PodInformer, jobInformer batchin eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(glog.Infof) // TODO: remove the wrapper when every clients have moved to use the clientset. - eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.Core().RESTClient()).Events("")}) + eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")}) - if kubeClient != nil && kubeClient.Core().RESTClient().GetRateLimiter() != nil { - metrics.RegisterMetricAndTrackRateLimiterUsage("job_controller", kubeClient.Core().RESTClient().GetRateLimiter()) + if kubeClient != nil && kubeClient.CoreV1().RESTClient().GetRateLimiter() != nil { + metrics.RegisterMetricAndTrackRateLimiterUsage("job_controller", kubeClient.CoreV1().RESTClient().GetRateLimiter()) } jm := &JobController{ @@ -100,12 +100,8 @@ func NewJobController(podInformer coreinformers.PodInformer, jobInformer batchin } jobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: jm.enqueueController, - UpdateFunc: func(old, cur interface{}) { - if job := cur.(*batch.Job); !IsJobFinished(job) { - jm.enqueueController(job) - } - }, + AddFunc: jm.enqueueController, + UpdateFunc: jm.updateJob, DeleteFunc: jm.enqueueController, }) jm.jobLister = jobInformer.Lister() @@ -306,6 +302,35 @@ func (jm *JobController) deletePod(obj interface{}) { jm.enqueueController(job) } +func (jm *JobController) updateJob(old, cur interface{}) { + oldJob := cur.(*batch.Job) + curJob := cur.(*batch.Job) + + // never return error + key, err := controller.KeyFunc(curJob) + if err != nil { + return + } + jm.queue.Add(key) + // check if need to add a new rsync for ActiveDeadlineSeconds + if curJob.Status.StartTime != nil { + curADS := curJob.Spec.ActiveDeadlineSeconds + if curADS == nil { + return + } + oldADS := oldJob.Spec.ActiveDeadlineSeconds + if oldADS == nil || *oldADS != *curADS { + now := metav1.Now() + start := curJob.Status.StartTime.Time + passed := now.Time.Sub(start) + total := time.Duration(*curADS) * time.Second + // AddAfter will handle total < passed + jm.queue.AddAfter(key, total-passed) + glog.V(4).Infof("job ActiveDeadlineSeconds updated, will rsync after %d seconds", total-passed) + } + } +} + // obj could be an *batch.Job, or a DeletionFinalStateUnknown marker item. func (jm *JobController) enqueueController(obj interface{}) { key, err := controller.KeyFunc(obj) @@ -420,9 +445,16 @@ func (jm *JobController) syncJob(key string) error { active := int32(len(activePods)) succeeded, failed := getStatus(pods) conditions := len(job.Status.Conditions) + // job first start if job.Status.StartTime == nil { now := metav1.Now() job.Status.StartTime = &now + // enqueue a sync to check if job past ActiveDeadlineSeconds + if job.Spec.ActiveDeadlineSeconds != nil { + glog.V(4).Infof("Job %s have ActiveDeadlineSeconds will sync after %d seconds", + key, *job.Spec.ActiveDeadlineSeconds) + jm.queue.AddAfter(key, time.Duration(*job.Spec.ActiveDeadlineSeconds)*time.Second) + } } // if job was finished previously, we don't want to redo the termination if IsJobFinished(&job) { @@ -685,8 +717,8 @@ func (o byCreationTimestamp) Len() int { return len(o) } func (o byCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o byCreationTimestamp) Less(i, j int) bool { - if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { return o[i].Name < o[j].Name } - return o[i].CreationTimestamp.Before(o[j].CreationTimestamp) + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) } diff --git a/pkg/controller/namespace/namespace_controller.go b/pkg/controller/namespace/namespace_controller.go index f066182b06b9a..c3bdd28be7a1e 100644 --- a/pkg/controller/namespace/namespace_controller.go +++ b/pkg/controller/namespace/namespace_controller.go @@ -183,14 +183,16 @@ func (nm *NamespaceController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer nm.queue.ShutDown() + glog.Infof("Starting namespace controller") + defer glog.Infof("Shutting down namespace controller") + if !controller.WaitForCacheSync("namespace", stopCh, nm.listerSynced) { return } - glog.V(5).Info("Starting workers") + glog.V(5).Info("Starting workers of namespace controller") for i := 0; i < workers; i++ { go wait.Until(nm.worker, time.Second, stopCh) } <-stopCh - glog.V(1).Infof("Shutting down") } diff --git a/pkg/controller/node/node_controller.go b/pkg/controller/node/node_controller.go index fa89efbbd3b36..e34d0c85be36c 100644 --- a/pkg/controller/node/node_controller.go +++ b/pkg/controller/node/node_controller.go @@ -134,10 +134,10 @@ type NodeController struct { // is just created, e.g. cluster bootstrap or node creation, we give a longer grace period. nodeStartupGracePeriod time.Duration // per Node map storing last observed Status together with a local time when it was observed. + nodeStatusMap map[string]nodeStatusData // This timestamp is to be used instead of LastProbeTime stored in Condition. We do this // to aviod the problem with time skew across the cluster. - nodeStatusMap map[string]nodeStatusData - now func() metav1.Time + now func() metav1.Time // Lock to access evictor workers evictorLock sync.Mutex // workers that evicts pods from unresponsive nodes. diff --git a/pkg/controller/podgc/gc_controller.go b/pkg/controller/podgc/gc_controller.go index c3e7e074ff46b..76179b736b2e7 100644 --- a/pkg/controller/podgc/gc_controller.go +++ b/pkg/controller/podgc/gc_controller.go @@ -193,8 +193,8 @@ func (o byCreationTimestamp) Len() int { return len(o) } func (o byCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o byCreationTimestamp) Less(i, j int) bool { - if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { return o[i].Name < o[j].Name } - return o[i].CreationTimestamp.Before(o[j].CreationTimestamp) + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) } diff --git a/pkg/controller/replication/replication_controller_utils.go b/pkg/controller/replication/replication_controller_utils.go index 943679b4fee1c..625317becb390 100644 --- a/pkg/controller/replication/replication_controller_utils.go +++ b/pkg/controller/replication/replication_controller_utils.go @@ -86,10 +86,10 @@ func (o OverlappingControllers) Len() int { return len(o) } func (o OverlappingControllers) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o OverlappingControllers) Less(i, j int) bool { - if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { return o[i].Name < o[j].Name } - return o[i].CreationTimestamp.Before(o[j].CreationTimestamp) + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) } func calculateStatus(rc *v1.ReplicationController, filteredPods []*v1.Pod, manageReplicasErr error) v1.ReplicationControllerStatus { diff --git a/pkg/controller/service/service_controller.go b/pkg/controller/service/service_controller.go index 52f3c64998a69..e1f76ce597d03 100644 --- a/pkg/controller/service/service_controller.go +++ b/pkg/controller/service/service_controller.go @@ -223,7 +223,14 @@ func (s *ServiceController) init() error { // indicating whether processing should be retried; zero means no-retry; otherwise // we should retry in that Duration. func (s *ServiceController) processServiceUpdate(cachedService *cachedService, service *v1.Service, key string) (error, time.Duration) { - + if cachedService.state != nil { + if cachedService.state.UID != service.UID { + err, retry := s.processLoadBalancerDelete(cachedService, key) + if err != nil { + return err, retry + } + } + } // cache the service, we need the info for service deletion cachedService.state = service err, retry := s.createLoadBalancerIfNeeded(key, service) @@ -280,13 +287,12 @@ func (s *ServiceController) createLoadBalancerIfNeeded(key string, service *v1.S // TODO: We could do a dry-run here if wanted to avoid the spurious cloud-calls & events when we restart - // The load balancer doesn't exist yet, so create it. - s.eventRecorder.Event(service, v1.EventTypeNormal, "CreatingLoadBalancer", "Creating load balancer") - newState, err = s.createLoadBalancer(service) + s.eventRecorder.Event(service, v1.EventTypeNormal, "EnsuringLoadBalancer", "Ensuring load balancer") + newState, err = s.ensureLoadBalancer(service) if err != nil { - return fmt.Errorf("Failed to create load balancer for service %s: %v", key, err), retryable + return fmt.Errorf("Failed to ensure load balancer for service %s: %v", key, err), retryable } - s.eventRecorder.Event(service, v1.EventTypeNormal, "CreatedLoadBalancer", "Created load balancer") + s.eventRecorder.Event(service, v1.EventTypeNormal, "EnsuredLoadBalancer", "Ensured load balancer") } // Write the state if changed @@ -340,7 +346,7 @@ func (s *ServiceController) persistUpdate(service *v1.Service) error { return err } -func (s *ServiceController) createLoadBalancer(service *v1.Service) (*v1.LoadBalancerStatus, error) { +func (s *ServiceController) ensureLoadBalancer(service *v1.Service) (*v1.LoadBalancerStatus, error) { nodes, err := s.nodeLister.ListWithPredicate(getNodeConditionPredicate()) if err != nil { return nil, err @@ -765,6 +771,10 @@ func (s *ServiceController) processServiceDeletion(key string) (error, time.Dura if !ok { return fmt.Errorf("Service %s not in cache even though the watcher thought it was. Ignoring the deletion.", key), doNotRetry } + return s.processLoadBalancerDelete(cachedService, key) +} + +func (s *ServiceController) processLoadBalancerDelete(cachedService *cachedService, key string) (error, time.Duration) { service := cachedService.state // delete load balancer info only if the service type is LoadBalancer if !wantsLoadBalancer(service) { diff --git a/pkg/controller/serviceaccount/OWNERS b/pkg/controller/serviceaccount/OWNERS index 8e6172a01c674..a678215e98405 100755 --- a/pkg/controller/serviceaccount/OWNERS +++ b/pkg/controller/serviceaccount/OWNERS @@ -4,3 +4,4 @@ approvers: reviewers: - liggitt - deads2k +- enj diff --git a/pkg/controller/statefulset/BUILD b/pkg/controller/statefulset/BUILD index 2f77880500e62..60f9301ff35e1 100644 --- a/pkg/controller/statefulset/BUILD +++ b/pkg/controller/statefulset/BUILD @@ -70,6 +70,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/informers/apps/v1beta1:go_default_library", "//vendor/k8s.io/client-go/informers/core/v1:go_default_library", diff --git a/pkg/controller/statefulset/stateful_pod_control_test.go b/pkg/controller/statefulset/stateful_pod_control_test.go index 1d4a3d423da44..9d397930a7d56 100644 --- a/pkg/controller/statefulset/stateful_pod_control_test.go +++ b/pkg/controller/statefulset/stateful_pod_control_test.go @@ -393,34 +393,6 @@ func TestStatefulPodControlUpdatePodConflictSuccess(t *testing.T) { } } -func TestStatefulPodControlUpdatePodConflictFailure(t *testing.T) { - recorder := record.NewFakeRecorder(10) - set := newStatefulSet(3) - pod := newStatefulSetPod(set, 0) - fakeClient := &fake.Clientset{} - indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) - updatedPod := newStatefulSetPod(set, 0) - updatedPod.Spec.Hostname = "wrong" - indexer.Add(updatedPod) - podLister := corelisters.NewPodLister(indexer) - control := NewRealStatefulPodControl(fakeClient, nil, podLister, nil, recorder) - fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { - update := action.(core.UpdateAction) - return true, update.GetObject(), apierrors.NewConflict(action.GetResource().GroupResource(), pod.Name, errors.New("conflict")) - - }) - pod.Name = "goo-0" - if err := control.UpdateStatefulPod(set, pod); err == nil { - t.Error("Failed update did not return an error") - } - events := collectEvents(recorder.Events) - if eventCount := len(events); eventCount != 1 { - t.Errorf("Pod update failed: got %d events, but want 1", eventCount) - } else if !strings.Contains(events[0], v1.EventTypeWarning) { - t.Errorf("Found unexpected non-normal event %s", events[0]) - } -} - func TestStatefulPodControlDeletesStatefulPod(t *testing.T) { recorder := record.NewFakeRecorder(10) set := newStatefulSet(3) diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index 637236b8768d6..dd264790d9ef8 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -80,13 +80,13 @@ func (ssc *defaultStatefulSetControl) UpdateStatefulSet(set *apps.StatefulSet, p history.SortControllerRevisions(revisions) // get the current, and update revisions - currentRevision, updateRevision, err := ssc.getStatefulSetRevisions(set, revisions) + currentRevision, updateRevision, collisionCount, err := ssc.getStatefulSetRevisions(set, revisions) if err != nil { return err } // perform the main update function and get the status - status, err := ssc.updateStatefulSet(set, currentRevision, updateRevision, pods) + status, err := ssc.updateStatefulSet(set, currentRevision, updateRevision, collisionCount, pods) if err != nil { return err } @@ -174,21 +174,31 @@ func (ssc *defaultStatefulSetControl) truncateHistory( return nil } -// getStatefulSetRevisions returns the current and update ControllerRevisions for set. This method may create a new revision, -// or modify the Revision of an existing revision if an update to set is detected. This method expects that revisions -// is sorted when supplied. +// getStatefulSetRevisions returns the current and update ControllerRevisions for set. It also +// returns a collision count that records the number of name collisions set saw when creating +// new ControllerRevisions. This count is incremented on every name collision and is used in +// building the ControllerRevision names for name collision avoidance. This method may create +// a new revision, or modify the Revision of an existing revision if an update to set is detected. +// This method expects that revisions is sorted when supplied. func (ssc *defaultStatefulSetControl) getStatefulSetRevisions( set *apps.StatefulSet, - revisions []*apps.ControllerRevision) (*apps.ControllerRevision, *apps.ControllerRevision, error) { + revisions []*apps.ControllerRevision) (*apps.ControllerRevision, *apps.ControllerRevision, int32, error) { var currentRevision, updateRevision *apps.ControllerRevision revisionCount := len(revisions) history.SortControllerRevisions(revisions) + // Use a local copy of set.Status.CollisionCount to avoid modifying set.Status directly. + // This copy is returned so the value gets carried over to set.Status in updateStatefulSet. + var collisionCount int32 + if set.Status.CollisionCount != nil { + collisionCount = *set.Status.CollisionCount + } + // create a new revision from the current set - updateRevision, err := newRevision(set, nextRevision(revisions)) + updateRevision, err := newRevision(set, nextRevision(revisions), &collisionCount) if err != nil { - return nil, nil, err + return nil, nil, collisionCount, err } // find any equivalent revisions @@ -205,13 +215,13 @@ func (ssc *defaultStatefulSetControl) getStatefulSetRevisions( equalRevisions[equalCount-1], updateRevision.Revision) if err != nil { - return nil, nil, err + return nil, nil, collisionCount, err } } else { //if there is no equivalent revision we create a new one - updateRevision, err = ssc.controllerHistory.CreateControllerRevision(set, updateRevision) + updateRevision, err = ssc.controllerHistory.CreateControllerRevision(set, updateRevision, &collisionCount) if err != nil { - return nil, nil, err + return nil, nil, collisionCount, err } } @@ -227,7 +237,7 @@ func (ssc *defaultStatefulSetControl) getStatefulSetRevisions( currentRevision = updateRevision } - return currentRevision, updateRevision, nil + return currentRevision, updateRevision, collisionCount, nil } // updateStatefulSet performs the update function for a StatefulSet. This method creates, updates, and deletes Pods in @@ -243,13 +253,14 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( set *apps.StatefulSet, currentRevision *apps.ControllerRevision, updateRevision *apps.ControllerRevision, + collisionCount int32, pods []*v1.Pod) (*apps.StatefulSetStatus, error) { // get the current and update revisions of the set. - currentSet, err := applyRevision(set, currentRevision) + currentSet, err := ApplyRevision(set, currentRevision) if err != nil { return nil, err } - updateSet, err := applyRevision(set, updateRevision) + updateSet, err := ApplyRevision(set, updateRevision) if err != nil { return nil, err } @@ -260,6 +271,8 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( *status.ObservedGeneration = set.Generation status.CurrentRevision = currentRevision.Name status.UpdateRevision = updateRevision.Name + status.CollisionCount = new(int32) + *status.CollisionCount = collisionCount replicaCount := int(*set.Spec.Replicas) // slice that will contain all Pods such that 0 <= getOrdinal(pod) < set.Spec.Replicas diff --git a/pkg/controller/statefulset/stateful_set_control_test.go b/pkg/controller/statefulset/stateful_set_control_test.go index 3d0184968bd3f..bcb38badf45fe 100644 --- a/pkg/controller/statefulset/stateful_set_control_test.go +++ b/pkg/controller/statefulset/stateful_set_control_test.go @@ -465,14 +465,18 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) { informerFactory.Core().V1().Pods().Informer().HasSynced, informerFactory.Apps().V1beta1().ControllerRevisions().Informer().HasSynced, ) + test.set.Status.CollisionCount = new(int32) for i := range test.existing { - ssc.controllerHistory.CreateControllerRevision(test.set, test.existing[i]) + ssc.controllerHistory.CreateControllerRevision(test.set, test.existing[i], test.set.Status.CollisionCount) } revisions, err := ssc.ListRevisions(test.set) if err != nil { t.Fatal(err) } - current, update, err := ssc.getStatefulSetRevisions(test.set, revisions) + current, update, _, err := ssc.getStatefulSetRevisions(test.set, revisions) + if err != nil { + t.Fatalf("error getting statefulset revisions:%v", err) + } revisions, err = ssc.ListRevisions(test.set) if err != nil { t.Fatal(err) @@ -480,20 +484,20 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) { if len(revisions) != test.expectedCount { t.Errorf("%s: want %d revisions got %d", test.name, test.expectedCount, len(revisions)) } - if test.err && err != nil { + if test.err && err == nil { t.Errorf("%s: expected error", test.name) } if !test.err && !history.EqualRevision(current, test.expectedCurrent) { t.Errorf("%s: for current want %v got %v", test.name, test.expectedCurrent, current) } if !test.err && !history.EqualRevision(update, test.expectedUpdate) { - t.Errorf("%s: for current want %v got %v", test.name, test.expectedUpdate, update) + t.Errorf("%s: for update want %v got %v", test.name, test.expectedUpdate, update) } if !test.err && test.expectedCurrent != nil && current != nil && test.expectedCurrent.Revision != current.Revision { t.Errorf("%s: for current revision want %d got %d", test.name, test.expectedCurrent.Revision, current.Revision) } if !test.err && test.expectedUpdate != nil && update != nil && test.expectedUpdate.Revision != update.Revision { - t.Errorf("%s: for current revision want %d got %d", test.name, test.expectedUpdate.Revision, update.Revision) + t.Errorf("%s: for update revision want %d got %d", test.name, test.expectedUpdate.Revision, update.Revision) } } @@ -508,14 +512,17 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) { } set := newStatefulSet(3) + set.Status.CollisionCount = new(int32) rev0 := newRevisionOrDie(set, 1) set1 := copySet(set) set1.Spec.Template.Spec.Containers[0].Image = "foo" set1.Status.CurrentRevision = rev0.Name + set1.Status.CollisionCount = new(int32) rev1 := newRevisionOrDie(set1, 2) set2 := copySet(set1) set2.Spec.Template.Labels["new"] = "label" set2.Status.CurrentRevision = rev0.Name + set2.Status.CollisionCount = new(int32) rev2 := newRevisionOrDie(set2, 3) tests := []testcase{ { @@ -1274,7 +1281,7 @@ func TestStatefulSetControlRollback(t *testing.T) { t.Fatalf("%s: %s", test.name, err) } history.SortControllerRevisions(revisions) - set, err = applyRevision(set, revisions[0]) + set, err = ApplyRevision(set, revisions[0]) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -2097,7 +2104,7 @@ func updateStatefulSetControl(set *apps.StatefulSet, } func newRevisionOrDie(set *apps.StatefulSet, revision int64) *apps.ControllerRevision { - rev, err := newRevision(set, revision) + rev, err := newRevision(set, revision, set.Status.CollisionCount) if err != nil { panic(err) } diff --git a/pkg/controller/statefulset/stateful_set_test.go b/pkg/controller/statefulset/stateful_set_test.go index 58242002f1b1b..f43d7b7014525 100644 --- a/pkg/controller/statefulset/stateful_set_test.go +++ b/pkg/controller/statefulset/stateful_set_test.go @@ -17,7 +17,6 @@ limitations under the License. package statefulset import ( - "reflect" "sort" "testing" @@ -25,6 +24,7 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" @@ -521,16 +521,13 @@ func TestGetPodsForStatefulSetAdopt(t *testing.T) { if err != nil { t.Fatalf("getPodsForStatefulSet() error: %v", err) } - var got []string + got := sets.NewString() for _, pod := range pods { - got = append(got, pod.Name) + got.Insert(pod.Name) } - // pod2 should be claimed, pod3 and pod4 ignored - want := []string{pod1.Name, pod2.Name} - sort.Strings(got) - sort.Strings(want) - if !reflect.DeepEqual(got, want) { + want := sets.NewString(pod1.Name, pod2.Name) + if !got.Equal(want) { t.Errorf("getPodsForStatefulSet() = %v, want %v", got, want) } } @@ -561,16 +558,14 @@ func TestGetPodsForStatefulSetRelease(t *testing.T) { if err != nil { t.Fatalf("getPodsForStatefulSet() error: %v", err) } - var got []string + got := sets.NewString() for _, pod := range pods { - got = append(got, pod.Name) + got.Insert(pod.Name) } // Expect only pod1 (pod2 and pod3 should be released, pod4 ignored). - want := []string{pod1.Name} - sort.Strings(got) - sort.Strings(want) - if !reflect.DeepEqual(got, want) { + want := sets.NewString(pod1.Name) + if !got.Equal(want) { t.Errorf("getPodsForStatefulSet() = %v, want %v", got, want) } } @@ -620,6 +615,9 @@ func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetControl } for set.Status.ReadyReplicas < *set.Spec.Replicas { pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } ord := len(pods) - 1 pod := getPodAtOrdinal(pods, ord) if pods, err = spc.setPodPending(set, ord); err != nil { @@ -674,6 +672,9 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr ssc.enqueueStatefulSet(set) fakeWorker(ssc) pods, err = spc.addTerminatingPod(set, ord) + if err != nil { + return err + } pod = getPodAtOrdinal(pods, ord) ssc.updatePod(&prev, pod) fakeWorker(ssc) @@ -682,8 +683,15 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr fakeWorker(ssc) for set.Status.Replicas > *set.Spec.Replicas { pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + ord := len(pods) pods, err = spc.addTerminatingPod(set, ord) + if err != nil { + return err + } pod = getPodAtOrdinal(pods, ord) ssc.updatePod(&prev, pod) fakeWorker(ssc) diff --git a/pkg/controller/statefulset/stateful_set_utils.go b/pkg/controller/statefulset/stateful_set_utils.go index e210cc915d570..fb80888a95690 100644 --- a/pkg/controller/statefulset/stateful_set_utils.go +++ b/pkg/controller/statefulset/stateful_set_utils.go @@ -17,6 +17,7 @@ limitations under the License. package statefulset import ( + "bytes" "encoding/json" "fmt" "regexp" @@ -50,10 +51,10 @@ func (o overlappingStatefulSets) Len() int { return len(o) } func (o overlappingStatefulSets) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o overlappingStatefulSets) Less(i, j int) bool { - if o[i].CreationTimestamp.Equal(o[j].CreationTimestamp) { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { return o[i].Name < o[j].Name } - return o[i].CreationTimestamp.Before(o[j].CreationTimestamp) + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) } // statefulPodRegex is a regular expression that extracts the parent StatefulSet and ordinal from the Name of a Pod @@ -111,9 +112,7 @@ func identityMatches(set *apps.StatefulSet, pod *v1.Pod) bool { return ordinal >= 0 && set.Name == parent && pod.Name == getPodName(set, ordinal) && - pod.Namespace == set.Namespace && - pod.Spec.Hostname == pod.Name && - pod.Spec.Subdomain == set.Spec.ServiceName + pod.Namespace == set.Namespace } // storageMatches returns true if pod's Volumes cover the set of PersistentVolumeClaims @@ -181,12 +180,18 @@ func updateStorage(set *apps.StatefulSet, pod *v1.Pod) { pod.Spec.Volumes = newVolumes } +func initIdentity(set *apps.StatefulSet, pod *v1.Pod) { + updateIdentity(set, pod) + // Set these immutable fields only on initial Pod creation, not updates. + pod.Spec.Hostname = pod.Name + pod.Spec.Subdomain = set.Spec.ServiceName +} + // updateIdentity updates pod's name, hostname, and subdomain to conform to set's name and headless service. func updateIdentity(set *apps.StatefulSet, pod *v1.Pod) { pod.Name = getPodName(set, getOrdinal(pod)) pod.Namespace = set.Namespace - pod.Spec.Hostname = pod.Name - pod.Spec.Subdomain = set.Spec.ServiceName + } // isRunningAndReady returns true if pod is in the PodRunning Phase, if it has a condition of PodReady. @@ -240,7 +245,7 @@ func getPodRevision(pod *v1.Pod) string { func newStatefulSetPod(set *apps.StatefulSet, ordinal int) *v1.Pod { pod, _ := controller.GetPodFromTemplate(&set.Spec.Template, set, metav1.NewControllerRef(set, controllerKind)) pod.Name = getPodName(set, ordinal) - updateIdentity(set, pod) + initIdentity(set, pod) updateStorage(set, pod) return pod } @@ -262,6 +267,15 @@ func newVersionedStatefulSetPod(currentSet, updateSet *apps.StatefulSet, current return pod } +// Match check if the given StatefulSet's template matches the template stored in the given history. +func Match(ss *apps.StatefulSet, history *apps.ControllerRevision) (bool, error) { + patch, err := getPatch(ss) + if err != nil { + return false, err + } + return bytes.Equal(patch, history.Data.Raw), nil +} + // getPatch returns a strategic merge patch that can be applied to restore a StatefulSet to a // previous version. If the returned error is nil the patch is valid. The current state that we save is just the // PodSpecTemplate. We can modify this later to encompass more state (or less) and remain compatible with previously @@ -288,7 +302,7 @@ func getPatch(set *apps.StatefulSet) ([]byte, error) { // The Revision of the returned ControllerRevision is set to revision. If the returned error is nil, the returned // ControllerRevision is valid. StatefulSet revisions are stored as patches that re-apply the current state of set // to a new StatefulSet using a strategic merge patch to replace the saved state of the new StatefulSet. -func newRevision(set *apps.StatefulSet, revision int64) (*apps.ControllerRevision, error) { +func newRevision(set *apps.StatefulSet, revision int64, collisionCount *int32) (*apps.ControllerRevision, error) { patch, err := getPatch(set) if err != nil { return nil, err @@ -301,7 +315,8 @@ func newRevision(set *apps.StatefulSet, revision int64) (*apps.ControllerRevisio controllerKind, selector, runtime.RawExtension{Raw: patch}, - revision) + revision, + collisionCount) if err != nil { return nil, err } @@ -314,9 +329,9 @@ func newRevision(set *apps.StatefulSet, revision int64) (*apps.ControllerRevisio return cr, nil } -// applyRevision returns a new StatefulSet constructed by restoring the state in revision to set. If the returned error +// ApplyRevision returns a new StatefulSet constructed by restoring the state in revision to set. If the returned error // is nil, the returned StatefulSet is valid. -func applyRevision(set *apps.StatefulSet, revision *apps.ControllerRevision) (*apps.StatefulSet, error) { +func ApplyRevision(set *apps.StatefulSet, revision *apps.ControllerRevision) (*apps.StatefulSet, error) { obj, err := scheme.Scheme.DeepCopy(set) if err != nil { return nil, err diff --git a/pkg/controller/statefulset/stateful_set_utils_test.go b/pkg/controller/statefulset/stateful_set_utils_test.go index 6efe8c466f7be..3b5c10c99fc4c 100644 --- a/pkg/controller/statefulset/stateful_set_utils_test.go +++ b/pkg/controller/statefulset/stateful_set_utils_test.go @@ -78,16 +78,6 @@ func TestIdentityMatches(t *testing.T) { if identityMatches(set, pod) { t.Error("identity matches for a Pod with the wrong namespace") } - pod = newStatefulSetPod(set, 1) - pod.Spec.Hostname = "" - if identityMatches(set, pod) { - t.Error("identity matches for a Pod with no hostname") - } - pod = newStatefulSetPod(set, 1) - pod.Spec.Subdomain = "" - if identityMatches(set, pod) { - t.Error("identity matches for a Pod with no subdomain") - } } func TestStorageMatches(t *testing.T) { @@ -137,24 +127,6 @@ func TestUpdateIdentity(t *testing.T) { if !identityMatches(set, pod) { t.Error("updateIdentity failed to update the Pods namespace") } - pod = newStatefulSetPod(set, 1) - pod.Spec.Hostname = "" - if identityMatches(set, pod) { - t.Error("identity matches for a Pod with no hostname") - } - updateIdentity(set, pod) - if !identityMatches(set, pod) { - t.Error("updateIdentity failed to update the Pod's hostname") - } - pod = newStatefulSetPod(set, 1) - pod.Spec.Subdomain = "" - if identityMatches(set, pod) { - t.Error("identity matches for a Pod with no subdomain") - } - updateIdentity(set, pod) - if !identityMatches(set, pod) { - t.Error("updateIdentity failed to update the Pod's subdomain") - } } func TestUpdateStorage(t *testing.T) { @@ -274,7 +246,8 @@ func TestNewPodControllerRef(t *testing.T) { func TestCreateApplyRevision(t *testing.T) { set := newStatefulSet(1) - revision, err := newRevision(set, 1) + set.Status.CollisionCount = new(int32) + revision, err := newRevision(set, 1, set.Status.CollisionCount) if err != nil { t.Fatal(err) } @@ -285,11 +258,11 @@ func TestCreateApplyRevision(t *testing.T) { key := "foo" expectedValue := "bar" set.Annotations[key] = expectedValue - restoredSet, err := applyRevision(set, revision) + restoredSet, err := ApplyRevision(set, revision) if err != nil { t.Fatal(err) } - restoredRevision, err := newRevision(restoredSet, 2) + restoredRevision, err := newRevision(restoredSet, 2, restoredSet.Status.CollisionCount) if err != nil { t.Fatal(err) } diff --git a/pkg/controller/volume/attachdetach/BUILD b/pkg/controller/volume/attachdetach/BUILD index db1e5fcccb2d3..b3d129b7abed3 100644 --- a/pkg/controller/volume/attachdetach/BUILD +++ b/pkg/controller/volume/attachdetach/BUILD @@ -45,6 +45,7 @@ go_test( "//pkg/controller:go_default_library", "//pkg/controller/volume/attachdetach/cache:go_default_library", "//pkg/controller/volume/attachdetach/testing:go_default_library", + "//pkg/volume:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index 6c206e504e414..a608fbf94d7df 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -97,6 +97,7 @@ func NewAttachDetachController( pvInformer coreinformers.PersistentVolumeInformer, cloud cloudprovider.Interface, plugins []volume.VolumePlugin, + prober volume.DynamicPluginProber, disableReconciliationSync bool, reconcilerSyncDuration time.Duration, timerConfig TimerConfig) (AttachDetachController, error) { @@ -127,7 +128,7 @@ func NewAttachDetachController( cloud: cloud, } - if err := adc.volumePluginMgr.InitPlugins(plugins, adc); err != nil { + if err := adc.volumePluginMgr.InitPlugins(plugins, prober, adc); err != nil { return nil, fmt.Errorf("Could not initialize volume plugins for Attach/Detach Controller: %+v", err) } @@ -539,7 +540,7 @@ func (adc *attachDetachController) GetCloudProvider() cloudprovider.Interface { return adc.cloud } -func (adc *attachDetachController) GetMounter() mount.Interface { +func (adc *attachDetachController) GetMounter(pluginName string) mount.Interface { return nil } @@ -571,6 +572,10 @@ func (adc *attachDetachController) GetConfigMapFunc() func(namespace, name strin } } +func (adc *attachDetachController) GetExec(pluginName string) mount.Exec { + return mount.NewOsExec() +} + func (adc *attachDetachController) addNodeToDswp(node *v1.Node, nodeName types.NodeName) { if _, exists := node.Annotations[volumehelper.ControllerManagedAttachAnnotation]; exists { keepTerminatedPodVolumes := false diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go index b63103bf2f8cd..b0e12589b16c8 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" controllervolumetesting "k8s.io/kubernetes/pkg/controller/volume/attachdetach/testing" + "k8s.io/kubernetes/pkg/volume" ) func Test_NewAttachDetachController_Positive(t *testing.T) { @@ -45,6 +46,7 @@ func Test_NewAttachDetachController_Positive(t *testing.T) { informerFactory.Core().V1().PersistentVolumes(), nil, /* cloud */ nil, /* plugins */ + nil, /* prober */ false, 5*time.Second, DefaultTimerConfig) @@ -79,8 +81,9 @@ func Test_AttachDetachControllerStateOfWolrdPopulators_Positive(t *testing.T) { // Act plugins := controllervolumetesting.CreateTestPlugin() + var prober volume.DynamicPluginProber = nil // TODO (#51147) inject mock - if err := adc.volumePluginMgr.InitPlugins(plugins, adc); err != nil { + if err := adc.volumePluginMgr.InitPlugins(plugins, prober, adc); err != nil { t.Fatalf("Could not initialize volume plugins for Attach/Detach Controller: %+v", err) } @@ -141,6 +144,7 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1) //informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1) plugins := controllervolumetesting.CreateTestPlugin() + var prober volume.DynamicPluginProber = nil // TODO (#51147) inject mock nodeInformer := informerFactory.Core().V1().Nodes().Informer() podInformer := informerFactory.Core().V1().Pods().Informer() var podsNum, extraPodsNum, nodesNum, i int @@ -212,6 +216,7 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 informerFactory.Core().V1().PersistentVolumes(), nil, /* cloud */ plugins, + prober, false, 1*time.Second, DefaultTimerConfig) diff --git a/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go b/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go index 5e96b4c433bf4..4ee4e26c9b429 100644 --- a/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go +++ b/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go @@ -125,10 +125,6 @@ type ActualStateOfWorld interface { // GetNodesToUpdateStatusFor returns the map of nodeNames to nodeToUpdateStatusFor GetNodesToUpdateStatusFor() map[types.NodeName]nodeToUpdateStatusFor - - // Removes the given node from the record of attach updates. The node's entire - // volumesToReportAsAttached list is removed. - RemoveNodeFromAttachUpdates(nodeName types.NodeName) error } // AttachedVolume represents a volume that is attached to a node. @@ -264,19 +260,6 @@ func (asw *actualStateOfWorld) AddVolumeToReportAsAttached( asw.addVolumeToReportAsAttached(volumeName, nodeName) } -func (asw *actualStateOfWorld) RemoveNodeFromAttachUpdates(nodeName types.NodeName) error { - asw.Lock() - defer asw.Unlock() - - _, nodeToUpdateExists := asw.nodesToUpdateStatusFor[nodeName] - if nodeToUpdateExists { - delete(asw.nodesToUpdateStatusFor, nodeName) - return nil - } - return fmt.Errorf("node %q does not exist in volumesToReportAsAttached list", - nodeName) -} - func (asw *actualStateOfWorld) AddVolumeNode( uniqueName v1.UniqueVolumeName, volumeSpec *volume.Spec, nodeName types.NodeName, devicePath string) (v1.UniqueVolumeName, error) { asw.Lock() diff --git a/pkg/controller/volume/attachdetach/cache/actual_state_of_world_test.go b/pkg/controller/volume/attachdetach/cache/actual_state_of_world_test.go index dcc9fd837b9c6..22a4e3df728d2 100644 --- a/pkg/controller/volume/attachdetach/cache/actual_state_of_world_test.go +++ b/pkg/controller/volume/attachdetach/cache/actual_state_of_world_test.go @@ -1165,89 +1165,6 @@ func Test_updateNodeStatusUpdateNeededError(t *testing.T) { } } -// Test_RemoveNodeFromAttachUpdates_Positive expects an entire node entry to be removed -// from nodesToUpdateStatusFor -func Test_RemoveNodeFromAttachUpdates_Positive(t *testing.T) { - // Arrange - volumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) - asw := &actualStateOfWorld{ - attachedVolumes: make(map[v1.UniqueVolumeName]attachedVolume), - nodesToUpdateStatusFor: make(map[types.NodeName]nodeToUpdateStatusFor), - volumePluginMgr: volumePluginMgr, - } - nodeName := types.NodeName("node-1") - nodeToUpdate := nodeToUpdateStatusFor{ - nodeName: nodeName, - statusUpdateNeeded: true, - volumesToReportAsAttached: make(map[v1.UniqueVolumeName]v1.UniqueVolumeName), - } - asw.nodesToUpdateStatusFor[nodeName] = nodeToUpdate - - // Act - err := asw.RemoveNodeFromAttachUpdates(nodeName) - - // Assert - if err != nil { - t.Fatalf("RemoveNodeFromAttachUpdates should not return error, but got: %v", err) - } - if len(asw.nodesToUpdateStatusFor) > 0 { - t.Fatal("nodesToUpdateStatusFor should be empty as its only entry has been deleted.") - } -} - -// Test_RemoveNodeFromAttachUpdates_Negative_NodeDoesNotExist expects an error to be thrown -// when nodeName is not in nodesToUpdateStatusFor. -func Test_RemoveNodeFromAttachUpdates_Negative_NodeDoesNotExist(t *testing.T) { - // Arrange - volumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) - asw := &actualStateOfWorld{ - attachedVolumes: make(map[v1.UniqueVolumeName]attachedVolume), - nodesToUpdateStatusFor: make(map[types.NodeName]nodeToUpdateStatusFor), - volumePluginMgr: volumePluginMgr, - } - nodeName := types.NodeName("node-1") - nodeToUpdate := nodeToUpdateStatusFor{ - nodeName: nodeName, - statusUpdateNeeded: true, - volumesToReportAsAttached: make(map[v1.UniqueVolumeName]v1.UniqueVolumeName), - } - asw.nodesToUpdateStatusFor[nodeName] = nodeToUpdate - - // Act - err := asw.RemoveNodeFromAttachUpdates("node-2") - - // Assert - if err == nil { - t.Fatal("RemoveNodeFromAttachUpdates should return an error as the nodeName doesn't exist.") - } - if len(asw.nodesToUpdateStatusFor) != 1 { - t.Fatal("The length of nodesToUpdateStatusFor should not change because no operation was performed.") - } -} - -// Test_RemoveNodeFromAttachUpdates_Negative_Empty expects an error to be thrown -// when a nodesToUpdateStatusFor is empty. -func Test_RemoveNodeFromAttachUpdates_Negative_Empty(t *testing.T) { - // Arrange - volumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) - asw := &actualStateOfWorld{ - attachedVolumes: make(map[v1.UniqueVolumeName]attachedVolume), - nodesToUpdateStatusFor: make(map[types.NodeName]nodeToUpdateStatusFor), - volumePluginMgr: volumePluginMgr, - } - - // Act - err := asw.RemoveNodeFromAttachUpdates("node-1") - - // Assert - if err == nil { - t.Fatal("RemoveNodeFromAttachUpdates should return an error as nodeToUpdateStatusFor is empty.") - } - if len(asw.nodesToUpdateStatusFor) != 0 { - t.Fatal("The length of nodesToUpdateStatusFor should be 0.") - } -} - func verifyAttachedVolume( t *testing.T, attachedVolumes []AttachedVolume, diff --git a/pkg/controller/volume/attachdetach/reconciler/reconciler.go b/pkg/controller/volume/attachdetach/reconciler/reconciler.go index d70a59db1fb1a..26133bb3bcd5e 100644 --- a/pkg/controller/volume/attachdetach/reconciler/reconciler.go +++ b/pkg/controller/volume/attachdetach/reconciler/reconciler.go @@ -148,6 +148,11 @@ func (rc *reconciler) isMultiAttachForbidden(volumeSpec *volume.Spec) bool { // Only if this volume is a persistent volume, we have reliable information on wether it's allowed or not to // multi-attach. We trust in the individual volume implementations to not allow unsupported access modes if volumeSpec.PersistentVolume != nil { + // Check for persistent volume types which do not fail when trying to multi-attach + if volumeSpec.PersistentVolume.Spec.VsphereVolume != nil { + return false + } + if len(volumeSpec.PersistentVolume.Spec.AccessModes) == 0 { // No access mode specified so we don't know for sure. Let the attacher fail if needed return false diff --git a/pkg/controller/volume/attachdetach/statusupdater/node_status_updater.go b/pkg/controller/volume/attachdetach/statusupdater/node_status_updater.go index acfc2b97bbc68..2ebb1706e99c7 100644 --- a/pkg/controller/volume/attachdetach/statusupdater/node_status_updater.go +++ b/pkg/controller/volume/attachdetach/statusupdater/node_status_updater.go @@ -68,13 +68,11 @@ func (nsu *nodeStatusUpdater) UpdateNodeStatuses() error { nodeObj, err := nsu.nodeLister.Get(string(nodeName)) if errors.IsNotFound(err) { // If node does not exist, its status cannot be updated. - // Remove the node entry from the collection of attach updates, preventing the - // status updater from unnecessarily updating the node. + // Do nothing so that there is no retry until node is created. glog.V(2).Infof( "Could not update node status. Failed to find node %q in NodeInformer cache. Error: '%v'", nodeName, err) - nsu.actualStateOfWorld.RemoveNodeFromAttachUpdates(nodeName) continue } else if err != nil { // For all other errors, log error and reset flag statusUpdateNeeded diff --git a/pkg/controller/volume/attachdetach/testing/testvolumespec.go b/pkg/controller/volume/attachdetach/testing/testvolumespec.go index a77eddfed4e49..a20f4507421b5 100644 --- a/pkg/controller/volume/attachdetach/testing/testvolumespec.go +++ b/pkg/controller/volume/attachdetach/testing/testvolumespec.go @@ -377,7 +377,7 @@ func (attacher *testPluginAttacher) VolumesAreAttached(specs []*volume.Spec, nod return nil, nil } -func (attacher *testPluginAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { +func (attacher *testPluginAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) { attacher.pluginLock.Lock() defer attacher.pluginLock.Unlock() if spec == nil { diff --git a/pkg/controller/volume/persistentvolume/BUILD b/pkg/controller/volume/persistentvolume/BUILD index aa6b165efa842..ebeb3defe4d6f 100644 --- a/pkg/controller/volume/persistentvolume/BUILD +++ b/pkg/controller/volume/persistentvolume/BUILD @@ -24,6 +24,7 @@ go_library( "//pkg/util/io:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", + "//pkg/volume/util:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/storage/v1:go_default_library", @@ -61,12 +62,14 @@ go_test( ], library = ":go_default_library", deps = [ + "//pkg/api:go_default_library", "//pkg/api/testapi:go_default_library", "//pkg/controller:go_default_library", "//pkg/volume:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/storage/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/pkg/controller/volume/persistentvolume/framework_test.go b/pkg/controller/volume/persistentvolume/framework_test.go index 64c75adb5321b..9c53ef3891fbe 100644 --- a/pkg/controller/volume/persistentvolume/framework_test.go +++ b/pkg/controller/volume/persistentvolume/framework_test.go @@ -827,7 +827,7 @@ func wrapTestWithPluginCalls(expectedRecycleCalls, expectedDeleteCalls []error, deleteCalls: expectedDeleteCalls, provisionCalls: expectedProvisionCalls, } - ctrl.volumePluginMgr.InitPlugins([]vol.VolumePlugin{plugin}, ctrl) + ctrl.volumePluginMgr.InitPlugins([]vol.VolumePlugin{plugin}, nil /* prober */, ctrl) return toWrap(ctrl, reactor, test) } } diff --git a/pkg/controller/volume/persistentvolume/provision_test.go b/pkg/controller/volume/persistentvolume/provision_test.go index 8465136074c94..ef7fa7e082ac7 100644 --- a/pkg/controller/volume/persistentvolume/provision_test.go +++ b/pkg/controller/volume/persistentvolume/provision_test.go @@ -22,7 +22,9 @@ import ( "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/api" ) var class1Parameters = map[string]string{ @@ -360,6 +362,36 @@ func TestProvisionSync(t *testing.T) { []string{"Warning ProvisioningFailed"}, noerrors, wrapTestWithProvisionCalls([]provisionCall{}, testSyncClaim), }, + { + // Provision success - first save of a PV to API server fails (API + // server has written the object to etcd, but crashed before sending + // 200 OK response to the controller). Controller retries and the + // second save of the PV returns "AlreadyExists" because the PV + // object already is in the API server. + // + "11-19 - provisioned volume saved but API server crashed", + novolumes, + // We don't actually simulate API server saving the object and + // crashing afterwards, Create() just returns error without saving + // the volume in this test. So the set of expected volumes at the + // end of the test is empty. + novolumes, + newClaimArray("claim11-19", "uid11-19", "1Gi", "", v1.ClaimPending, &classGold), + newClaimArray("claim11-19", "uid11-19", "1Gi", "", v1.ClaimPending, &classGold, annStorageProvisioner), + noevents, + []reactorError{ + // Inject errors to simulate crashed API server during + // kubeclient.PersistentVolumes.Create() + {"create", "persistentvolumes", errors.New("Mock creation error1")}, + {"create", "persistentvolumes", apierrs.NewAlreadyExists(api.Resource("persistentvolumes"), "")}, + }, + wrapTestWithPluginCalls( + nil, // recycle calls + nil, // delete calls - if Delete was called the test would fail + []provisionCall{provision1Success}, + testSyncClaim, + ), + }, } runSyncTests(t, tests, storageClasses) } diff --git a/pkg/controller/volume/persistentvolume/pv_controller.go b/pkg/controller/volume/persistentvolume/pv_controller.go index 16ad055d83a02..69b8b627871c1 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller.go +++ b/pkg/controller/volume/persistentvolume/pv_controller.go @@ -24,6 +24,7 @@ import ( "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -39,6 +40,7 @@ import ( "k8s.io/kubernetes/pkg/util/goroutinemap" "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" vol "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" "github.com/golang/glog" ) @@ -625,14 +627,19 @@ func (ctrl *PersistentVolumeController) updateClaimStatus(claim *v1.PersistentVo dirty = true } - volumeCap, ok := volume.Spec.Capacity[v1.ResourceStorage] - if !ok { - return nil, fmt.Errorf("PersistentVolume %q is without a storage capacity", volume.Name) - } - claimCap, ok := claim.Status.Capacity[v1.ResourceStorage] - if !ok || volumeCap.Cmp(claimCap) != 0 { - claimClone.Status.Capacity = volume.Spec.Capacity - dirty = true + // Update Capacity if the claim is becoming Bound, not if it was already. + // A discrepancy can be intentional to mean that the PVC filesystem size + // doesn't match the PV block device size, so don't clobber it + if claim.Status.Phase != phase { + volumeCap, ok := volume.Spec.Capacity[v1.ResourceStorage] + if !ok { + return nil, fmt.Errorf("PersistentVolume %q is without a storage capacity", volume.Name) + } + claimCap, ok := claim.Status.Capacity[v1.ResourceStorage] + if !ok || volumeCap.Cmp(claimCap) != 0 { + claimClone.Status.Capacity = volume.Spec.Capacity + dirty = true + } } } @@ -1216,7 +1223,10 @@ func (ctrl *PersistentVolumeController) doDeleteVolume(volume *v1.PersistentVolu return false, fmt.Errorf("Failed to create deleter for volume %q: %v", volume.Name, err) } - if err = deleter.Delete(); err != nil { + opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_delete") + err = deleter.Delete() + opComplete(err) + if err != nil { // Deleter failed return false, err } @@ -1326,7 +1336,9 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa return } + opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_provision") volume, err = provisioner.Provision() + opComplete(err) if err != nil { strerr := fmt.Sprintf("Failed to provision volume with StorageClass %q: %v", storageClass.Name, err) glog.V(2).Infof("failed to provision volume for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err) @@ -1353,14 +1365,19 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa for i := 0; i < ctrl.createProvisionedPVRetryCount; i++ { glog.V(4).Infof("provisionClaimOperation [%s]: trying to save volume %s", claimToClaimKey(claim), volume.Name) var newVol *v1.PersistentVolume - if newVol, err = ctrl.kubeClient.Core().PersistentVolumes().Create(volume); err == nil { + if newVol, err = ctrl.kubeClient.Core().PersistentVolumes().Create(volume); err == nil || apierrs.IsAlreadyExists(err) { // Save succeeded. - glog.V(3).Infof("volume %q for claim %q saved", volume.Name, claimToClaimKey(claim)) + if err != nil { + glog.V(3).Infof("volume %q for claim %q already exists, reusing", volume.Name, claimToClaimKey(claim)) + err = nil + } else { + glog.V(3).Infof("volume %q for claim %q saved", volume.Name, claimToClaimKey(claim)) - _, updateErr := ctrl.storeVolumeUpdate(newVol) - if updateErr != nil { - // We will get an "volume added" event soon, this is not a big error - glog.V(4).Infof("provisionClaimOperation [%s]: cannot update internal cache: %v", volume.Name, updateErr) + _, updateErr := ctrl.storeVolumeUpdate(newVol) + if updateErr != nil { + // We will get an "volume added" event soon, this is not a big error + glog.V(4).Infof("provisionClaimOperation [%s]: cannot update internal cache: %v", volume.Name, updateErr) + } } break } diff --git a/pkg/controller/volume/persistentvolume/pv_controller_base.go b/pkg/controller/volume/persistentvolume/pv_controller_base.go index cfbe26401aeb2..a4c6d918d1f17 100644 --- a/pkg/controller/volume/persistentvolume/pv_controller_base.go +++ b/pkg/controller/volume/persistentvolume/pv_controller_base.go @@ -90,7 +90,8 @@ func NewController(p ControllerParameters) (*PersistentVolumeController, error) resyncPeriod: p.SyncPeriod, } - if err := controller.volumePluginMgr.InitPlugins(p.VolumePlugins, controller); err != nil { + // Prober is nil because PV is not aware of Flexvolume. + if err := controller.volumePluginMgr.InitPlugins(p.VolumePlugins, nil /* prober */, controller); err != nil { return nil, fmt.Errorf("Could not initialize volume plugins for PersistentVolume Controller: %v", err) } diff --git a/pkg/controller/volume/persistentvolume/volume_host.go b/pkg/controller/volume/persistentvolume/volume_host.go index e4c402f006853..8f182edc6b981 100644 --- a/pkg/controller/volume/persistentvolume/volume_host.go +++ b/pkg/controller/volume/persistentvolume/volume_host.go @@ -61,7 +61,7 @@ func (ctrl *PersistentVolumeController) GetCloudProvider() cloudprovider.Interfa return ctrl.cloud } -func (ctrl *PersistentVolumeController) GetMounter() mount.Interface { +func (ctrl *PersistentVolumeController) GetMounter(pluginName string) mount.Interface { return nil } @@ -93,6 +93,10 @@ func (adc *PersistentVolumeController) GetConfigMapFunc() func(namespace, name s } } +func (adc *PersistentVolumeController) GetExec(pluginName string) mount.Exec { + return mount.NewOsExec() +} + func (ctrl *PersistentVolumeController) GetNodeLabels() (map[string]string, error) { return nil, fmt.Errorf("GetNodeLabels() unsupported in PersistentVolumeController") } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 696e84c7d08ce..98713fe35ab47 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -164,4 +164,5 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta}, genericfeatures.AdvancedAuditing: {Default: false, PreRelease: utilfeature.Alpha}, TaintNodesByCondition: {Default: false, PreRelease: utilfeature.Alpha}, + genericfeatures.Initializers: {Default: false, PreRelease: utilfeature.Alpha}, } diff --git a/pkg/generated/bindata.go b/pkg/generated/bindata.go index 3b46cac4b2913..acae0361745f1 100644 --- a/pkg/generated/bindata.go +++ b/pkg/generated/bindata.go @@ -8,6 +8,8 @@ // translations/kubectl/en_US/LC_MESSAGES/k8s.po // translations/kubectl/fr_FR/LC_MESSAGES/k8s.mo // translations/kubectl/fr_FR/LC_MESSAGES/k8s.po +// translations/kubectl/it_IT/LC_MESSAGES/k8s.mo +// translations/kubectl/it_IT/LC_MESSAGES/k8s.po // translations/kubectl/ja_JP/LC_MESSAGES/k8s.mo // translations/kubectl/ja_JP/LC_MESSAGES/k8s.po // translations/kubectl/template.pot @@ -7289,6 +7291,2922 @@ func translationsKubectlFr_frLc_messagesK8sPo() (*asset, error) { return a, nil } +var _translationsKubectlIt_itLc_messagesK8sMo = []byte("\xde\x12\x04\x95\x00\x00\x00\x00\xd6\x00\x00\x00\x1c\x00\x00\x00\xcc\x06\x00\x00%\x01\x00\x00|\r\x00\x00\x00\x00\x00\x00\x10\x12\x00\x00\xdc\x00\x00\x00\x11\x12\x00\x00\xb6\x00\x00\x00\xee\x12\x00\x00\v\x02\x00\x00\xa5\x13\x00\x00\x1f\x01\x00\x00\xb1\x15\x00\x00z\x00\x00\x00\xd1\x16\x00\x00_\x02\x00\x00L\x17\x00\x00|\x01\x00\x00\xac\x19\x00\x00\x8f\x01\x00\x00)\x1b\x00\x00k\x01\x00\x00\xb9\x1c\x00\x00k\x01\x00\x00%\x1e\x00\x00>\x01\x00\x00\x91\x1f\x00\x00\x03\x02\x00\x00\xd0 \x00\x00o\x01\x00\x00\xd4\"\x00\x00H\x05\x00\x00D$\x00\x00g\x02\x00\x00\x8d)\x00\x00\x1b\x02\x00\x00\xf5+\x00\x00q\x01\x00\x00\x11.\x00\x00\xa8\x01\x00\x00\x83/\x00\x00\xd4\x01\x00\x00,1\x00\x00\x02\x02\x00\x00\x013\x00\x00\xb4\x00\x00\x00\x045\x00\x00\xb7\x02\x00\x00\xb95\x00\x00\x92\x03\x00\x00q8\x00\x00\xbf\x01\x00\x00\x04<\x00\x00=\x00\x00\x00\xc4=\x00\x00;\x00\x00\x00\x02>\x00\x00\xcd\x02\x00\x00>>\x00\x00<\x00\x00\x00\fA\x00\x00P\x00\x00\x00IA\x00\x00S\x00\x00\x00\x9aA\x00\x00<\x00\x00\x00\xeeA\x00\x00\xfa\x01\x00\x00+B\x00\x00\xda\x01\x00\x00&D\x00\x00T\x01\x00\x00\x01F\x00\x00\xfb\x00\x00\x00VG\x00\x00\xa5\x01\x00\x00RH\x00\x00\x18\x00\x00\x00\xf8I\x00\x00<\x00\x00\x00\x11J\x00\x00=\x00\x00\x00NJ\x00\x00\xc6\x00\x00\x00\x8cJ\x00\x00.\x00\x00\x00SK\x00\x00g\x00\x00\x00\x82K\x00\x00Q\x00\x00\x00\xeaK\x00\x00R\x00\x00\x00\x00\x00\x009\xcb\x00\x00\xe0\x00\x00\x00x\xcb\x00\x00/\x00\x00\x00Y\xcc\x00\x00h\x00\x00\x00\x89\xcc\x00\x00S\x00\x00\x00\xf2\xcc\x00\x00T\x00\x00\x00F\xcd\x00\x00(\x00\x00\x00\x9b\xcd\x00\x005\x00\x00\x00\xc4\xcd\x00\x00\x82\x00\x00\x00\xfa\xcd\x00\x00\x98\x01\x00\x00}\xce\x00\x00\x93\x00\x00\x00\x16\xd0\x00\x00\x1f\x01\x00\x00\xaa\xd0\x00\x00\xec\x00\x00\x00\xca\xd1\x00\x00+\x00\x00\x00\xb7\xd2\x00\x001\x00\x00\x00\xe3\xd2\x00\x00\x88\x01\x00\x00\x15\xd3\x00\x00'\x01\x00\x00\x9e\xd4\x00\x00\xff\x01\x00\x00\xc6\xd5\x00\x00=\x01\x00\x00\xc6\xd7\x00\x00\xc2\x01\x00\x00\x04\xd9\x00\x00\xdb\x00\x00\x00\xc7\xda\x00\x00\xa9\x00\x00\x00\xa3\xdb\x00\x00^\x00\x00\x00M\xdc\x00\x00N\x02\x00\x00\xac\xdc\x00\x00u\x00\x00\x00\xfb\xde\x00\x00}\x00\x00\x00q\xdf\x00\x00)\x01\x00\x00\xef\xdf\x00\x00\x8b\x00\x00\x00\x19\xe1\x00\x00}\x00\x00\x00\xa5\xe1\x00\x00\x06\x01\x00\x00#\xe2\x00\x00\x84\x00\x00\x00*\xe3\x00\x00s\x00\x00\x00\xaf\xe3\x00\x00\xf0\x01\x00\x00#\xe4\x00\x00\x13\x04\x00\x00\x14\xe6\x00\x00;\x00\x00\x00(\xea\x00\x008\x00\x00\x00d\xea\x00\x002\x00\x00\x00\x9d\xea\x00\x009\x00\x00\x00\xd0\xea\x00\x00\xd5\x02\x00\x00\n\xeb\x00\x00\xc8\x00\x00\x00\xe0\xed\x00\x00p\x00\x00\x00\xa9\xee\x00\x00]\x00\x00\x00\x1a\xef\x00\x00l\x00\x00\x00x\xef\x00\x00\xba\x00\x00\x00\xe5\xef\x00\x00B\x00\x00\x00\xa0\xf0\x00\x00\xca\x00\x00\x00\xe3\xf0\x00\x00\xb5\x00\x00\x00\xae\xf1\x00\x00\xe6\x00\x00\x00d\xf2\x00\x007\x00\x00\x00K\xf3\x00\x00.\x00\x00\x00\x83\xf3\x00\x00r\x00\x00\x00\xb2\xf3\x00\x00$\x00\x00\x00%\xf4\x00\x00<\x00\x00\x00J\xf4\x00\x00\x86\x00\x00\x00\x87\xf4\x00\x00:\x00\x00\x00\x0e\xf5\x00\x003\x00\x00\x00I\xf5\x00\x00\xc6\x00\x00\x00}\xf5\x00\x00=\x00\x00\x00D\xf6\x00\x000\x00\x00\x00\x82\xf6\x00\x009\x00\x00\x00\xb3\xf6\x00\x00 \x00\x00\x00\xed\xf6\x00\x00\x1a\x00\x00\x00\x0e\xf7\x00\x009\x00\x00\x00)\xf7\x00\x00\x12\x00\x00\x00c\xf7\x00\x00\x1b\x00\x00\x00v\xf7\x00\x00H\x00\x00\x00\x92\xf7\x00\x00-\x00\x00\x00\xdb\xf7\x00\x00)\x00\x00\x00\t\xf8\x00\x006\x00\x00\x003\xf8\x00\x00'\x00\x00\x00j\xf8\x00\x00&\x00\x00\x00\x92\xf8\x00\x003\x00\x00\x00\xb9\xf8\x00\x00E\x00\x00\x00\xed\xf8\x00\x004\x00\x00\x003\xf9\x00\x005\x00\x00\x00h\xf9\x00\x007\x00\x00\x00\x9e\xf9\x00\x00\x1e\x00\x00\x00\xd6\xf9\x00\x00g\x00\x00\x00\xf5\xf9\x00\x00-\x00\x00\x00]\xfa\x00\x00-\x00\x00\x00\x8b\xfa\x00\x00+\x00\x00\x00\xb9\xfa\x00\x00A\x00\x00\x00\xe5\xfa\x00\x00\x1b\x00\x00\x00'\xfb\x00\x007\x00\x00\x00C\xfb\x00\x007\x00\x00\x00{\xfb\x00\x00/\x00\x00\x00\xb3\xfb\x00\x00#\x00\x00\x00\xe3\xfb\x00\x00(\x00\x00\x00\a\xfc\x00\x00P\x00\x00\x000\xfc\x00\x00\x1d\x00\x00\x00\x81\xfc\x00\x00\x1d\x00\x00\x00\x9f\xfc\x00\x00\x1c\x00\x00\x00\xbd\xfc\x00\x00,\x00\x00\x00\xda\xfc\x00\x00F\x00\x00\x00\a\xfd\x00\x00!\x00\x00\x00N\xfd\x00\x00\x1c\x00\x00\x00p\xfd\x00\x00#\x00\x00\x00\x8d\xfd\x00\x00\x88\x00\x00\x00\xb1\xfd\x00\x00(\x00\x00\x00:\xfe\x00\x00\x1b\x00\x00\x00c\xfe\x00\x00u\x00\x00\x00\u007f\xfe\x00\x00e\x00\x00\x00\xf5\xfe\x00\x00\xb4\x00\x00\x00[\xff\x00\x00\xab\x00\x00\x00\x10\x00\x01\x00\xc0\x00\x00\x00\xbc\x00\x01\x00\x1e\x00\x00\x00}\x01\x01\x00)\x00\x00\x00\x9c\x01\x01\x00-\x00\x00\x00\xc6\x01\x01\x00$\x00\x00\x00\xf4\x01\x01\x00&\x00\x00\x00\x19\x02\x01\x00\x1a\x00\x00\x00@\x02\x01\x00e\x00\x00\x00[\x02\x01\x00\x93\x00\x00\x00\xc1\x02\x01\x00M\x00\x00\x00U\x03\x01\x00g\x00\x00\x00\xa3\x03\x01\x003\x00\x00\x00\v\x04\x01\x007\x00\x00\x00?\x04\x01\x00D\x00\x00\x00w\x04\x01\x00@\x00\x00\x00\xbc\x04\x01\x00\x84\x00\x00\x00\xfd\x04\x01\x009\x00\x00\x00\x82\x05\x01\x001\x00\x00\x00\xbc\x05\x01\x00$\x00\x00\x00\xee\x05\x01\x00+\x00\x00\x00\x13\x06\x01\x00\x1f\x00\x00\x00?\x06\x01\x00$\x00\x00\x00_\x06\x01\x00+\x00\x00\x00\x84\x06\x01\x00*\x00\x00\x00\xb0\x06\x01\x00+\x00\x00\x00\xdb\x06\x01\x00V\x00\x00\x00\a\a\x01\x000\x00\x00\x00^\a\x01\x00s\x00\x00\x00\x8f\a\x01\x00%\x00\x00\x00\x03\b\x01\x00&\x00\x00\x00)\b\x01\x00&\x00\x00\x00P\b\x01\x00%\x00\x00\x00w\b\x01\x00/\x00\x00\x00\x9d\b\x01\x000\x00\x00\x00\xcd\b\x01\x00B\x00\x00\x00\xfe\b\x01\x00\x1b\x00\x00\x00A\t\x01\x00\x19\x00\x00\x00]\t\x01\x00i\x00\x00\x00w\t\x01\x00(\x00\x00\x00\xe1\t\x01\x00\x8f\x00\x00\x00\n\n\x01\x00\xa3\x00\x00\x00\x9a\n\x01\x00P\x00\x00\x00>\v\x01\x00#\x00\x00\x00\x8f\v\x01\x00i\x00\x00\x00\xb3\v\x01\x00\x85\x00\x00\x00\x1d\f\x01\x00M\x00\x00\x00\xa3\f\x01\x00\x03\x01\x00\x00\xf1\f\x01\x00i\x00\x00\x00\xf5\r\x01\x00P\x00\x00\x00_\x0e\x01\x00X\x00\x00\x00\xb0\x0e\x01\x00u\x00\x00\x00\t\x0f\x01\x00\xed\x00\x00\x00\u007f\x0f\x01\x00\xf8\x00\x00\x00m\x10\x01\x00H\x01\x00\x00f\x11\x01\x00\x19\x00\x00\x00\xaf\x12\x01\x00^\x00\x00\x00\xc9\x12\x01\x00\x1d\x00\x00\x00(\x13\x01\x00,\x00\x00\x00F\x13\x01\x00=\x00\x00\x00s\x13\x01\x00$\x00\x00\x00\xb1\x13\x01\x00C\x00\x00\x00\xd6\x13\x01\x00\x1f\x00\x00\x00\x1a\x14\x01\x00\x1d\x00\x00\x00:\x14\x01\x00$\x00\x00\x00X\x14\x01\x004\x00\x00\x00}\x14\x01\x00V\x00\x00\x00\xb2\x14\x01\x00 \x00\x00\x00\t\x15\x01\x00\x82\x00\x00\x00*\x15\x01\x00\x16\x00\x00\x00\xad\x15\x01\x00\x19\x00\x00\x00\xc4\x15\x01\x002\x00\x00\x00\xde\x15\x01\x00\x01\x00\x00\x00~\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xaa\x00\x00\x00 \x00\x00\x00\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00r\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00\x00\x00\x00\x00X\x00\x00\x00R\x00\x00\x00\v\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x17\x00\x00\x00\xca\x00\x00\x00\x0e\x00\x00\x00e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd4\x00\x00\x00K\x00\x00\x00T\x00\x00\x00\x82\x00\x00\x000\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xab\x00\x00\x00H\x00\x00\x00\xbe\x00\x00\x00\x9f\x00\x00\x00}\x00\x00\x00\xac\x00\x00\x00O\x00\x00\x00\x00\x00\x00\x00\x93\x00\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\xb4\x00\x00\x00\x88\x00\x00\x00]\x00\x00\x00\xbf\x00\x00\x00c\x00\x00\x00g\x00\x00\x003\x00\x00\x00\x86\x00\x00\x00\x0f\x00\x00\x00Q\x00\x00\x00f\x00\x00\x00\xb9\x00\x00\x00v\x00\x00\x00\x00\x00\x00\x00\x89\x00\x00\x00_\x00\x00\x00\x00\x00\x00\x00u\x00\x00\x00;\x00\x00\x00\x00\x00\x00\x00A\x00\x00\x00\x85\x00\x00\x00\xc5\x00\x00\x00\x9b\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00W\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x8a\x00\x00\x00\xb3\x00\x00\x00\x81\x00\x00\x00p\x00\x00\x00\xcf\x00\x00\x00\xd6\x00\x00\x00\xa0\x00\x00\x00\x00\x00\x00\x00/\x00\x00\x00\x99\x00\x00\x00\xc8\x00\x00\x00n\x00\x00\x00\x00\x00\x00\x00w\x00\x00\x00\x00\x00\x00\x00\xc4\x00\x00\x00\x11\x00\x00\x00`\x00\x00\x00\x00\x00\x00\x00\xaf\x00\x00\x00\t\x00\x00\x00z\x00\x00\x00\xa1\x00\x00\x00\xc1\x00\x00\x00^\x00\x00\x00\xbc\x00\x00\x00\x87\x00\x00\x00,\x00\x00\x00q\x00\x00\x00\x00\x00\x00\x00=\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc9\x00\x00\x00\x9a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x12\x00\x00\x00\xbd\x00\x00\x00\x00\x00\x00\x00\x92\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00\xb0\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00M\x00\x00\x00\xce\x00\x00\x00\x1e\x00\x00\x00I\x00\x00\x00\x00\x00\x00\x00Y\x00\x00\x00a\x00\x00\x00@\x00\x00\x00\x1f\x00\x00\x00\u007f\x00\x00\x00S\x00\x00\x00\xa2\x00\x00\x00\n\x00\x00\x00>\x00\x00\x00l\x00\x00\x00\x8f\x00\x00\x00\x05\x00\x00\x00\x15\x00\x00\x00%\x00\x00\x00\"\x00\x00\x00\x90\x00\x00\x00\xa9\x00\x00\x00\x84\x00\x00\x00\xa7\x00\x00\x00\xa6\x00\x00\x00.\x00\x00\x00\b\x00\x00\x00\\\x00\x00\x00\xa3\x00\x00\x00\xd3\x00\x00\x00!\x00\x00\x00&\x00\x00\x00Z\x00\x00\x00\xb6\x00\x00\x00\x97\x00\x00\x00F\x00\x00\x00\x00\x00\x00\x00\x8c\x00\x00\x00\x94\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\xcb\x00\x00\x00L\x00\x00\x00\xbb\x00\x00\x00*\x00\x00\x00+\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x83\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00\x00\x9d\x00\x00\x00\a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00#\x00\x00\x00V\x00\x00\x00\xa8\x00\x00\x007\x00\x00\x00\x00\x00\x00\x00\xae\x00\x00\x00\x1b\x00\x00\x00h\x00\x00\x00\xb5\x00\x00\x00\xb7\x00\x00\x00\xd1\x00\x00\x00\x00\x00\x00\x00\x9e\x00\x00\x00d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x005\x00\x00\x00k\x00\x00\x00{\x00\x00\x006\x00\x00\x00\xad\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00G\x00\x00\x00\xa5\x00\x00\x00\xd2\x00\x00\x00t\x00\x00\x00\xc7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x95\x00\x00\x00b\x00\x00\x00\xa4\x00\x00\x00\x8d\x00\x00\x00\x8e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc2\x00\x00\x00J\x00\x00\x00\xc6\x00\x00\x00\xcd\x00\x00\x00i\x00\x00\x00\x00\x00\x00\x00\x98\x00\x00\x00\x00\x00\x00\x00\x8b\x00\x00\x00\xb8\x00\x00\x00?\x00\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\xb1\x00\x00\x00\xcc\x00\x00\x00\x00\x00\x00\x00C\x00\x00\x00\x00\x00\x00\x00B\x00\x00\x00\x9c\x00\x00\x00\x00\x00\x00\x001\x00\x00\x00\xd5\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00o\x00\x00\x00\x04\x00\x00\x00x\x00\x00\x00\x00\x00\x00\x004\x00\x00\x00:\x00\x00\x00N\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x002\x00\x00\x00j\x00\x00\x008\x00\x00\x00P\x00\x00\x00|\x00\x00\x00\x19\x00\x00\x00\x00\x00\x00\x00\xb2\x00\x00\x00\x96\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00y\x00\x00\x00\xba\x00\x00\x00)\x00\x00\x00\x00\x00\x00\x00[\x00\x00\x00\x18\x00\x00\x00\x91\x00\x00\x00E\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00s\x00\x00\x00\xd0\x00\x00\x009\x00\x00\x00'\x00\x00\x00\x16\x00\x00\x00U\x00\x00\x00<\x00\x00\x00m\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00D\x00\x00\x00\x00\n\t\t # Create a ClusterRoleBinding for user1, user2, and group1 using the cluster-admin ClusterRole\n\t\t kubectl create clusterrolebinding cluster-admin --clusterrole=cluster-admin --user=user1 --user=user2 --group=group1\x00\n\t\t # Create a RoleBinding for user1, user2, and group1 using the admin ClusterRole\n\t\t kubectl create rolebinding admin --clusterrole=admin --user=user1 --user=user2 --group=group1\x00\n\t\t # Create a new configmap named my-config based on folder bar\n\t\t kubectl create configmap my-config --from-file=path/to/bar\n\n\t\t # Create a new configmap named my-config with specified keys instead of file basenames on disk\n\t\t kubectl create configmap my-config --from-file=key1=/path/to/bar/file1.txt --from-file=key2=/path/to/bar/file2.txt\n\n\t\t # Create a new configmap named my-config with key1=config1 and key2=config2\n\t\t kubectl create configmap my-config --from-literal=key1=config1 --from-literal=key2=config2\x00\n\t\t # If you don't already have a .dockercfg file, you can create a dockercfg secret directly by using:\n\t\t kubectl create secret docker-registry my-secret --docker-server=DOCKER_REGISTRY_SERVER --docker-username=DOCKER_USER --docker-password=DOCKER_PASSWORD --docker-email=DOCKER_EMAIL\x00\n\t\t # Show metrics for all nodes\n\t\t kubectl top node\n\n\t\t # Show metrics for a given node\n\t\t kubectl top node NODE_NAME\x00\n\t\t# Apply the configuration in pod.json to a pod.\n\t\tkubectl apply -f ./pod.json\n\n\t\t# Apply the JSON passed into stdin to a pod.\n\t\tcat pod.json | kubectl apply -f -\n\n\t\t# Note: --prune is still in Alpha\n\t\t# Apply the configuration in manifest.yaml that matches label app=nginx and delete all the other resources that are not in the file and match label app=nginx.\n\t\tkubectl apply --prune -f manifest.yaml -l app=nginx\n\n\t\t# Apply the configuration in manifest.yaml and delete all the other configmaps that are not in the file.\n\t\tkubectl apply --prune -f manifest.yaml --all --prune-whitelist=core/v1/ConfigMap\x00\n\t\t# Auto scale a deployment \"foo\", with the number of pods between 2 and 10, target CPU utilization specified so a default autoscaling policy will be used:\n\t\tkubectl autoscale deployment foo --min=2 --max=10\n\n\t\t# Auto scale a replication controller \"foo\", with the number of pods between 1 and 5, target CPU utilization at 80%:\n\t\tkubectl autoscale rc foo --max=5 --cpu-percent=80\x00\n\t\t# Convert 'pod.yaml' to latest version and print to stdout.\n\t\tkubectl convert -f pod.yaml\n\n\t\t# Convert the live state of the resource specified by 'pod.yaml' to the latest version\n\t\t# and print to stdout in json format.\n\t\tkubectl convert -f pod.yaml --local -o json\n\n\t\t# Convert all files under current directory to latest version and create them all.\n\t\tkubectl convert -f . | kubectl create -f -\x00\n\t\t# Create a ClusterRole named \"pod-reader\" that allows user to perform \"get\", \"watch\" and \"list\" on pods\n\t\tkubectl create clusterrole pod-reader --verb=get,list,watch --resource=pods\n\n\t\t# Create a ClusterRole named \"pod-reader\" with ResourceName specified\n\t\tkubectl create clusterrole pod-reader --verb=get,list,watch --resource=pods --resource-name=readablepod\x00\n\t\t# Create a Role named \"pod-reader\" that allows user to perform \"get\", \"watch\" and \"list\" on pods\n\t\tkubectl create role pod-reader --verb=get --verb=list --verb=watch --resource=pods\n\n\t\t# Create a Role named \"pod-reader\" with ResourceName specified\n\t\tkubectl create role pod-reader --verb=get --verg=list --verb=watch --resource=pods --resource-name=readablepod\x00\n\t\t# Create a new resourcequota named my-quota\n\t\tkubectl create quota my-quota --hard=cpu=1,memory=1G,pods=2,services=3,replicationcontrollers=2,resourcequotas=1,secrets=5,persistentvolumeclaims=10\n\n\t\t# Create a new resourcequota named best-effort\n\t\tkubectl create quota best-effort --hard=pods=100 --scopes=BestEffort\x00\n\t\t# Create a pod disruption budget named my-pdb that will select all pods with the app=rails label\n\t\t# and require at least one of them being available at any point in time.\n\t\tkubectl create poddisruptionbudget my-pdb --selector=app=rails --min-available=1\n\n\t\t# Create a pod disruption budget named my-pdb that will select all pods with the app=nginx label\n\t\t# and require at least half of the pods selected to be available at any point in time.\n\t\tkubectl create pdb my-pdb --selector=app=nginx --min-available=50%\x00\n\t\t# Create a pod using the data in pod.json.\n\t\tkubectl create -f ./pod.json\n\n\t\t# Create a pod based on the JSON passed into stdin.\n\t\tcat pod.json | kubectl create -f -\n\n\t\t# Edit the data in docker-registry.yaml in JSON using the v1 API format then create the resource using the edited data.\n\t\tkubectl create -f docker-registry.yaml --edit --output-version=v1 -o json\x00\n\t\t# Create a service for a replicated nginx, which serves on port 80 and connects to the containers on port 8000.\n\t\tkubectl expose rc nginx --port=80 --target-port=8000\n\n\t\t# Create a service for a replication controller identified by type and name specified in \"nginx-controller.yaml\", which serves on port 80 and connects to the containers on port 8000.\n\t\tkubectl expose -f nginx-controller.yaml --port=80 --target-port=8000\n\n\t\t# Create a service for a pod valid-pod, which serves on port 444 with the name \"frontend\"\n\t\tkubectl expose pod valid-pod --port=444 --name=frontend\n\n\t\t# Create a second service based on the above service, exposing the container port 8443 as port 443 with the name \"nginx-https\"\n\t\tkubectl expose service nginx --port=443 --target-port=8443 --name=nginx-https\n\n\t\t# Create a service for a replicated streaming application on port 4100 balancing UDP traffic and named 'video-stream'.\n\t\tkubectl expose rc streamer --port=4100 --protocol=udp --name=video-stream\n\n\t\t# Create a service for a replicated nginx using replica set, which serves on port 80 and connects to the containers on port 8000.\n\t\tkubectl expose rs nginx --port=80 --target-port=8000\n\n\t\t# Create a service for an nginx deployment, which serves on port 80 and connects to the containers on port 8000.\n\t\tkubectl expose deployment nginx --port=80 --target-port=8000\x00\n\t\t# Delete a pod using the type and name specified in pod.json.\n\t\tkubectl delete -f ./pod.json\n\n\t\t# Delete a pod based on the type and name in the JSON passed into stdin.\n\t\tcat pod.json | kubectl delete -f -\n\n\t\t# Delete pods and services with same names \"baz\" and \"foo\"\n\t\tkubectl delete pod,service baz foo\n\n\t\t# Delete pods and services with label name=myLabel.\n\t\tkubectl delete pods,services -l name=myLabel\n\n\t\t# Delete a pod with minimal delay\n\t\tkubectl delete pod foo --now\n\n\t\t# Force delete a pod on a dead node\n\t\tkubectl delete pod foo --grace-period=0 --force\n\n\t\t# Delete all pods\n\t\tkubectl delete pods --all\x00\n\t\t# Describe a node\n\t\tkubectl describe nodes kubernetes-node-emt8.c.myproject.internal\n\n\t\t# Describe a pod\n\t\tkubectl describe pods/nginx\n\n\t\t# Describe a pod identified by type and name in \"pod.json\"\n\t\tkubectl describe -f pod.json\n\n\t\t# Describe all pods\n\t\tkubectl describe pods\n\n\t\t# Describe pods by label name=myLabel\n\t\tkubectl describe po -l name=myLabel\n\n\t\t# Describe all pods managed by the 'frontend' replication controller (rc-created pods\n\t\t# get the name of the rc as a prefix in the pod the name).\n\t\tkubectl describe pods frontend\x00\n\t\t# Drain node \"foo\", even if there are pods not managed by a ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet on it.\n\t\t$ kubectl drain foo --force\n\n\t\t# As above, but abort if there are pods not managed by a ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet, and use a grace period of 15 minutes.\n\t\t$ kubectl drain foo --grace-period=900\x00\n\t\t# Edit the service named 'docker-registry':\n\t\tkubectl edit svc/docker-registry\n\n\t\t# Use an alternative editor\n\t\tKUBE_EDITOR=\"nano\" kubectl edit svc/docker-registry\n\n\t\t# Edit the job 'myjob' in JSON using the v1 API format:\n\t\tkubectl edit job.v1.batch/myjob -o json\n\n\t\t# Edit the deployment 'mydeployment' in YAML and save the modified config in its annotation:\n\t\tkubectl edit deployment/mydeployment -o yaml --save-config\x00\n\t\t# Get output from running 'date' from pod 123456-7890, using the first container by default\n\t\tkubectl exec 123456-7890 date\n\n\t\t# Get output from running 'date' in ruby-container from pod 123456-7890\n\t\tkubectl exec 123456-7890 -c ruby-container date\n\n\t\t# Switch to raw terminal mode, sends stdin to 'bash' in ruby-container from pod 123456-7890\n\t\t# and sends stdout/stderr from 'bash' back to the client\n\t\tkubectl exec 123456-7890 -c ruby-container -i -t -- bash -il\x00\n\t\t# Get output from running pod 123456-7890, using the first container by default\n\t\tkubectl attach 123456-7890\n\n\t\t# Get output from ruby-container from pod 123456-7890\n\t\tkubectl attach 123456-7890 -c ruby-container\n\n\t\t# Switch to raw terminal mode, sends stdin to 'bash' in ruby-container from pod 123456-7890\n\t\t# and sends stdout/stderr from 'bash' back to the client\n\t\tkubectl attach 123456-7890 -c ruby-container -i -t\n\n\t\t# Get output from the first pod of a ReplicaSet named nginx\n\t\tkubectl attach rs/nginx\n\t\t\x00\n\t\t# Get the documentation of the resource and its fields\n\t\tkubectl explain pods\n\n\t\t# Get the documentation of a specific field of a resource\n\t\tkubectl explain pods.spec.containers\x00\n\t\t# Install bash completion on a Mac using homebrew\n\t\tbrew install bash-completion\n\t\tprintf \"\n# Bash completion support\nsource $(brew --prefix)/etc/bash_completion\n\" >> $HOME/.bash_profile\n\t\tsource $HOME/.bash_profile\n\n\t\t# Load the kubectl completion code for bash into the current shell\n\t\tsource <(kubectl completion bash)\n\n\t\t# Write bash completion code to a file and source if from .bash_profile\n\t\tkubectl completion bash > ~/.kube/completion.bash.inc\n\t\tprintf \"\n# Kubectl shell completion\nsource '$HOME/.kube/completion.bash.inc'\n\" >> $HOME/.bash_profile\n\t\tsource $HOME/.bash_profile\n\n\t\t# Load the kubectl completion code for zsh[1] into the current shell\n\t\tsource <(kubectl completion zsh)\x00\n\t\t# List all pods in ps output format.\n\t\tkubectl get pods\n\n\t\t# List all pods in ps output format with more information (such as node name).\n\t\tkubectl get pods -o wide\n\n\t\t# List a single replication controller with specified NAME in ps output format.\n\t\tkubectl get replicationcontroller web\n\n\t\t# List a single pod in JSON output format.\n\t\tkubectl get -o json pod web-pod-13je7\n\n\t\t# List a pod identified by type and name specified in \"pod.yaml\" in JSON output format.\n\t\tkubectl get -f pod.yaml -o json\n\n\t\t# Return only the phase value of the specified pod.\n\t\tkubectl get -o template pod/web-pod-13je7 --template={{.status.phase}}\n\n\t\t# List all replication controllers and services together in ps output format.\n\t\tkubectl get rc,services\n\n\t\t# List one or more resources by their type and names.\n\t\tkubectl get rc/web service/frontend pods/web-pod-13je7\n\n\t\t# List all resources with different types.\n\t\tkubectl get all\x00\n\t\t# Listen on ports 5000 and 6000 locally, forwarding data to/from ports 5000 and 6000 in the pod\n\t\tkubectl port-forward mypod 5000 6000\n\n\t\t# Listen on port 8888 locally, forwarding to 5000 in the pod\n\t\tkubectl port-forward mypod 8888:5000\n\n\t\t# Listen on a random port locally, forwarding to 5000 in the pod\n\t\tkubectl port-forward mypod :5000\n\n\t\t# Listen on a random port locally, forwarding to 5000 in the pod\n\t\tkubectl port-forward mypod 0:5000\x00\n\t\t# Mark node \"foo\" as schedulable.\n\t\t$ kubectl uncordon foo\x00\n\t\t# Mark node \"foo\" as unschedulable.\n\t\tkubectl cordon foo\x00\n\t\t# Partially update a node using strategic merge patch\n\t\tkubectl patch node k8s-node-1 -p '{\"spec\":{\"unschedulable\":true}}'\n\n\t\t# Partially update a node identified by the type and name specified in \"node.json\" using strategic merge patch\n\t\tkubectl patch -f node.json -p '{\"spec\":{\"unschedulable\":true}}'\n\n\t\t# Update a container's image; spec.containers[*].name is required because it's a merge key\n\t\tkubectl patch pod valid-pod -p '{\"spec\":{\"containers\":[{\"name\":\"kubernetes-serve-hostname\",\"image\":\"new image\"}]}}'\n\n\t\t# Update a container's image using a json patch with positional arrays\n\t\tkubectl patch pod valid-pod --type='json' -p='[{\"op\": \"replace\", \"path\": \"/spec/containers/0/image\", \"value\":\"new image\"}]'\x00\n\t\t# Print flags inherited by all commands\n\t\tkubectl options\x00\n\t\t# Print the address of the master and cluster services\n\t\tkubectl cluster-info\x00\n\t\t# Print the client and server versions for the current context\n\t\tkubectl version\x00\n\t\t# Print the supported API versions\n\t\tkubectl api-versions\x00\n\t\t# Scale a replicaset named 'foo' to 3.\n\t\tkubectl scale --replicas=3 rs/foo\n\n\t\t# Scale a resource identified by type and name specified in \"foo.yaml\" to 3.\n\t\tkubectl scale --replicas=3 -f foo.yaml\n\n\t\t# If the deployment named mysql's current size is 2, scale mysql to 3.\n\t\tkubectl scale --current-replicas=2 --replicas=3 deployment/mysql\n\n\t\t# Scale multiple replication controllers.\n\t\tkubectl scale --replicas=5 rc/foo rc/bar rc/baz\n\n\t\t# Scale job named 'cron' to 3.\n\t\tkubectl scale --replicas=3 job/cron\x00\n\t\t# Set the last-applied-configuration of a resource to match the contents of a file.\n\t\tkubectl apply set-last-applied -f deploy.yaml\n\n\t\t# Execute set-last-applied against each configuration file in a directory.\n\t\tkubectl apply set-last-applied -f path/\n\n\t\t# Set the last-applied-configuration of a resource to match the contents of a file, will create the annotation if it does not already exist.\n\t\tkubectl apply set-last-applied -f deploy.yaml --create-annotation=true\n\t\t\x00\n\t\t# Shut down foo.\n\t\tkubectl stop replicationcontroller foo\n\n\t\t# Stop pods and services with label name=myLabel.\n\t\tkubectl stop pods,services -l name=myLabel\n\n\t\t# Shut down the service defined in service.json\n\t\tkubectl stop -f service.json\n\n\t\t# Shut down all resources in the path/to/resources directory\n\t\tkubectl stop -f path/to/resources\x00\n\t\t# View the last-applied-configuration annotations by type/name in YAML.\n\t\tkubectl apply view-last-applied deployment/nginx\n\n\t\t# View the last-applied-configuration annotations by file in JSON\n\t\tkubectl apply view-last-applied -f deploy.yaml -o json\x00\n\t\tApply a configuration to a resource by filename or stdin.\n\t\tThis resource will be created if it doesn't exist yet.\n\t\tTo use 'apply', always create the resource initially with either 'apply' or 'create --save-config'.\n\n\t\tJSON and YAML formats are accepted.\n\n\t\tAlpha Disclaimer: the --prune functionality is not yet complete. Do not use unless you are aware of what the current state is. See https://issues.k8s.io/34274.\x00\n\t\tCreate a ClusterRole.\x00\n\t\tCreate a ClusterRoleBinding for a particular ClusterRole.\x00\n\t\tCreate a RoleBinding for a particular Role or ClusterRole.\x00\n\t\tCreate a TLS secret from the given public/private key pair.\n\n\t\tThe public/private key pair must exist before hand. The public key certificate must be .PEM encoded and match the given private key.\x00\n\t\tCreate a namespace with the specified name.\x00\n\t\tCreate a pod disruption budget with the specified name, selector, and desired minimum available pods\x00\n\t\tCreate a resource by filename or stdin.\n\n\t\tJSON and YAML formats are accepted.\x00\n\t\tCreate a resourcequota with the specified name, hard limits and optional scopes\x00\n\t\tCreate a role with single rule.\x00\n\t\tCreate a service account with the specified name.\x00\n\t\tCreate and run a particular image, possibly replicated.\n\n\t\tCreates a deployment or job to manage the created container(s).\x00\n\t\tCreates an autoscaler that automatically chooses and sets the number of pods that run in a kubernetes cluster.\n\n\t\tLooks up a Deployment, ReplicaSet, or ReplicationController by name and creates an autoscaler that uses the given resource as a reference.\n\t\tAn autoscaler can automatically increase or decrease number of pods deployed within the system as needed.\x00\n\t\tDisplay Resource (CPU/Memory/Storage) usage of nodes.\n\n\t\tThe top-node command allows you to see the resource consumption of nodes.\x00\n\t\tDisplay Resource (CPU/Memory/Storage) usage of pods.\n\n\t\tThe 'top pod' command allows you to see the resource consumption of pods.\n\n\t\tDue to the metrics pipeline delay, they may be unavailable for a few minutes\n\t\tsince pod creation.\x00\n\t\tDisplay Resource (CPU/Memory/Storage) usage.\n\n\t\tThe top command allows you to see the resource consumption for nodes or pods.\n\n\t\tThis command requires Heapster to be correctly configured and working on the server. \x00\n\t\tMark node as schedulable.\x00\n\t\tMark node as unschedulable.\x00\n\t\tSet a new size for a Deployment, ReplicaSet, Replication Controller, or Job.\n\n\t\tScale also allows users to specify one or more preconditions for the scale action.\n\n\t\tIf --current-replicas or --resource-version is specified, it is validated before the\n\t\tscale is attempted, and it is guaranteed that the precondition holds true when the\n\t\tscale is sent to the server.\x00\n\t\tSet the latest last-applied-configuration annotations by setting it to match the contents of a file.\n\t\tThis results in the last-applied-configuration being updated as though 'kubectl apply -f ' was run,\n\t\twithout updating any other parts of the object.\x00\n\t\tTo proxy all of the kubernetes api and nothing else, use:\n\n\t\t $ kubectl proxy --api-prefix=/\n\n\t\tTo proxy only part of the kubernetes api and also some static files:\n\n\t\t $ kubectl proxy --www=/my/files --www-prefix=/static/ --api-prefix=/api/\n\n\t\tThe above lets you 'curl localhost:8001/api/v1/pods'.\n\n\t\tTo proxy the entire kubernetes api at a different root, use:\n\n\t\t $ kubectl proxy --api-prefix=/custom/\n\n\t\tThe above lets you 'curl localhost:8001/custom/api/v1/pods'\x00\n\t\tUpdate field(s) of a resource using strategic merge patch\n\n\t\tJSON and YAML formats are accepted.\n\n\t\tPlease refer to the models in https://htmlpreview.github.io/?https://github.com/kubernetes/kubernetes/blob/HEAD/docs/api-reference/v1/definitions.html to find if a field is mutable.\x00\n\t\tUpdate the labels on a resource.\n\n\t\t* A label must begin with a letter or number, and may contain letters, numbers, hyphens, dots, and underscores, up to %[1]d characters.\n\t\t* If --overwrite is true, then existing labels can be overwritten, otherwise attempting to overwrite a label will result in an error.\n\t\t* If --resource-version is specified, then updates will use this resource version, otherwise the existing resource-version will be used.\x00\n\t\tView the latest last-applied-configuration annotations by type/name or file.\n\n\t\tThe default output will be printed to stdout in YAML format. One can use -o option\n\t\tto change output format.\x00\n\t # Create a new TLS secret named tls-secret with the given key pair:\n\t kubectl create secret tls tls-secret --cert=path/to/tls.cert --key=path/to/tls.key\x00\n\t # Create a new namespace named my-namespace\n\t kubectl create namespace my-namespace\x00\n\t # Create a new secret named my-secret with keys for each file in folder bar\n\t kubectl create secret generic my-secret --from-file=path/to/bar\n\n\t # Create a new secret named my-secret with specified keys instead of names on disk\n\t kubectl create secret generic my-secret --from-file=ssh-privatekey=~/.ssh/id_rsa --from-file=ssh-publickey=~/.ssh/id_rsa.pub\n\n\t # Create a new secret named my-secret with key1=supersecret and key2=topsecret\n\t kubectl create secret generic my-secret --from-literal=key1=supersecret --from-literal=key2=topsecret\x00\n\t # Create a new service account named my-service-account\n\t kubectl create serviceaccount my-service-account\x00\n\t# Create a new ExternalName service named my-ns \n\tkubectl create service externalname my-ns --external-name bar.com\x00\n\tCreate an ExternalName service with the specified name.\n\n\tExternalName service references to an external DNS address instead of\n\tonly pods, which will allow application authors to reference services\n\tthat exist off platform, on other clusters, or locally.\x00\n\tHelp provides help for any command in the application.\n\tSimply type kubectl help [path to command] for full details.\x00\n # Create a new LoadBalancer service named my-lbs\n kubectl create service loadbalancer my-lbs --tcp=5678:8080\x00\n # Create a new clusterIP service named my-cs\n kubectl create service clusterip my-cs --tcp=5678:8080\n\n # Create a new clusterIP service named my-cs (in headless mode)\n kubectl create service clusterip my-cs --clusterip=\"None\"\x00\n # Create a new deployment named my-dep that runs the busybox image.\n kubectl create deployment my-dep --image=busybox\x00\n # Create a new nodeport service named my-ns\n kubectl create service nodeport my-ns --tcp=5678:8080\x00\n # Dump current cluster state to stdout\n kubectl cluster-info dump\n\n # Dump current cluster state to /path/to/cluster-state\n kubectl cluster-info dump --output-directory=/path/to/cluster-state\n\n # Dump all namespaces to stdout\n kubectl cluster-info dump --all-namespaces\n\n # Dump a set of namespaces to /path/to/cluster-state\n kubectl cluster-info dump --namespaces default,kube-system --output-directory=/path/to/cluster-state\x00\n # Update pod 'foo' with the annotation 'description' and the value 'my frontend'.\n # If the same annotation is set multiple times, only the last value will be applied\n kubectl annotate pods foo description='my frontend'\n\n # Update a pod identified by type and name in \"pod.json\"\n kubectl annotate -f pod.json description='my frontend'\n\n # Update pod 'foo' with the annotation 'description' and the value 'my frontend running nginx', overwriting any existing value.\n kubectl annotate --overwrite pods foo description='my frontend running nginx'\n\n # Update all pods in the namespace\n kubectl annotate pods --all description='my frontend running nginx'\n\n # Update pod 'foo' only if the resource is unchanged from version 1.\n kubectl annotate pods foo description='my frontend running nginx' --resource-version=1\n\n # Update pod 'foo' by removing an annotation named 'description' if it exists.\n # Does not require the --overwrite flag.\n kubectl annotate pods foo description-\x00\n Create a LoadBalancer service with the specified name.\x00\n Create a clusterIP service with the specified name.\x00\n Create a deployment with the specified name.\x00\n Create a nodeport service with the specified name.\x00\n Dumps cluster info out suitable for debugging and diagnosing cluster problems. By default, dumps everything to\n stdout. You can optionally specify a directory with --output-directory. If you specify a directory, kubernetes will\n build a set of files in that directory. By default only dumps things in the 'kube-system' namespace, but you can\n switch to a different namespace with the --namespaces flag, or specify --all-namespaces to dump all namespaces.\n\n The command also dumps the logs of all of the pods in the cluster, these logs are dumped into different directories\n based on namespace and pod name.\x00\n Display addresses of the master and services with label kubernetes.io/cluster-service=true\n To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.\x00A comma-delimited set of quota scopes that must all match each object tracked by the quota.\x00A comma-delimited set of resource=quantity pairs that define a hard limit.\x00A label selector to use for this budget. Only equality-based selector requirements are supported.\x00A label selector to use for this service. Only equality-based selector requirements are supported. If empty (the default) infer the selector from the replication controller or replica set.)\x00A schedule in the Cron format the job should be run with.\x00Additional external IP address (not managed by Kubernetes) to accept for the service. If this IP is routed to a node, the service can be accessed by this IP in addition to its generated service IP.\x00An inline JSON override for the generated object. If this is non-empty, it is used to override the generated object. Requires that the object supply a valid apiVersion field.\x00An inline JSON override for the generated service object. If this is non-empty, it is used to override the generated object. Requires that the object supply a valid apiVersion field. Only used if --expose is true.\x00Apply a configuration to a resource by filename or stdin\x00Approve a certificate signing request\x00Assign your own ClusterIP or set to 'None' for a 'headless' service (no loadbalancing).\x00Attach to a running container\x00Auto-scale a Deployment, ReplicaSet, or ReplicationController\x00ClusterIP to be assigned to the service. Leave empty to auto-allocate, or set to 'None' to create a headless service.\x00ClusterRole this ClusterRoleBinding should reference\x00ClusterRole this RoleBinding should reference\x00Container name which will have its image upgraded. Only relevant when --image is specified, ignored otherwise. Required when using --image on a multi-container pod\x00Convert config files between different API versions\x00Copy files and directories to and from containers.\x00Create a ClusterRoleBinding for a particular ClusterRole\x00Create a LoadBalancer service.\x00Create a NodePort service.\x00Create a RoleBinding for a particular Role or ClusterRole\x00Create a TLS secret\x00Create a clusterIP service.\x00Create a configmap from a local file, directory or literal value\x00Create a deployment with the specified name.\x00Create a namespace with the specified name\x00Create a pod disruption budget with the specified name.\x00Create a quota with the specified name.\x00Create a resource by filename or stdin\x00Create a secret for use with a Docker registry\x00Create a secret from a local file, directory or literal value\x00Create a secret using specified subcommand\x00Create a service account with the specified name\x00Create a service using specified subcommand.\x00Create an ExternalName service.\x00Delete resources by filenames, stdin, resources and names, or by resources and label selector\x00Delete the specified cluster from the kubeconfig\x00Delete the specified context from the kubeconfig\x00Deny a certificate signing request\x00Deprecated: Gracefully shut down a resource by name or filename\x00Describe one or many contexts\x00Display Resource (CPU/Memory) usage of nodes\x00Display Resource (CPU/Memory) usage of pods\x00Display Resource (CPU/Memory) usage.\x00Display cluster info\x00Display clusters defined in the kubeconfig\x00Display merged kubeconfig settings or a specified kubeconfig file\x00Display one or many resources\x00Displays the current-context\x00Documentation of resources\x00Drain node in preparation for maintenance\x00Dump lots of relevant info for debugging and diagnosis\x00Edit a resource on the server\x00Email for Docker registry\x00Execute a command in a container\x00Explicit policy for when to pull container images. Required when --image is same as existing image, ignored otherwise.\x00Forward one or more local ports to a pod\x00Help about any command\x00IP to assign to the Load Balancer. If empty, an ephemeral IP will be created and used (cloud-provider specific).\x00If non-empty, set the session affinity for the service to this; legal values: 'None', 'ClientIP'\x00If non-empty, the annotation update will only succeed if this is the current resource-version for the object. Only valid when specifying a single resource.\x00If non-empty, the labels update will only succeed if this is the current resource-version for the object. Only valid when specifying a single resource.\x00Image to use for upgrading the replication controller. Must be distinct from the existing image (either new image or new image tag). Can not be used with --filename/-f\x00Manage a deployment rollout\x00Mark node as schedulable\x00Mark node as unschedulable\x00Mark the provided resource as paused\x00Modify certificate resources.\x00Modify kubeconfig files\x00Name or number for the port on the container that the service should direct traffic to. Optional.\x00Only return logs after a specific date (RFC3339). Defaults to all logs. Only one of since-time / since may be used.\x00Output shell completion code for the specified shell (bash or zsh)\x00Output the formatted object with the given group version (for ex: 'extensions/v1beta1').)\x00Password for Docker registry authentication\x00Path to PEM encoded public key certificate.\x00Path to private key associated with given certificate.\x00Perform a rolling update of the given ReplicationController\x00Precondition for resource version. Requires that the current resource version match this value in order to scale.\x00Print the client and server version information\x00Print the list of flags inherited by all commands\x00Print the logs for a container in a pod\x00Replace a resource by filename or stdin\x00Resume a paused resource\x00Role this RoleBinding should reference\x00Run a particular image on the cluster\x00Run a proxy to the Kubernetes API server\x00Server location for Docker registry\x00Set a new size for a Deployment, ReplicaSet, Replication Controller, or Job\x00Set specific features on objects\x00Set the last-applied-configuration annotation on a live object to match the contents of a file.\x00Set the selector on a resource\x00Sets a cluster entry in kubeconfig\x00Sets a context entry in kubeconfig\x00Sets a user entry in kubeconfig\x00Sets an individual value in a kubeconfig file\x00Sets the current-context in a kubeconfig file\x00Show details of a specific resource or group of resources\x00Show the status of the rollout\x00Synonym for --target-port\x00Take a replication controller, service, deployment or pod and expose it as a new Kubernetes Service\x00The image for the container to run.\x00The image pull policy for the container. If left empty, this value will not be specified by the client and defaulted by the server\x00The key to use to differentiate between two different controllers, default 'deployment'. Only relevant when --image is specified, ignored otherwise\x00The minimum number or percentage of available pods this budget requires.\x00The name for the newly created object.\x00The name for the newly created object. If not specified, the name of the input resource will be used.\x00The name of the API generator to use, see http://kubernetes.io/docs/user-guide/kubectl-conventions/#generators for a list.\x00The name of the API generator to use. Currently there is only 1 generator.\x00The name of the API generator to use. There are 2 generators: 'service/v1' and 'service/v2'. The only difference between them is that service port in v1 is named 'default', while it is left unnamed in v2. Default is 'service/v2'.\x00The name of the generator to use for creating a service. Only used if --expose is true\x00The network protocol for the service to be created. Default is 'TCP'.\x00The port that the service should serve on. Copied from the resource being exposed, if unspecified\x00The port that this container exposes. If --expose is true, this is also the port used by the service that is created.\x00The resource requirement limits for this container. For example, 'cpu=200m,memory=512Mi'. Note that server side components may assign limits depending on the server configuration, such as limit ranges.\x00The resource requirement requests for this container. For example, 'cpu=100m,memory=256Mi'. Note that server side components may assign requests depending on the server configuration, such as limit ranges.\x00The restart policy for this Pod. Legal values [Always, OnFailure, Never]. If set to 'Always' a deployment is created, if set to 'OnFailure' a job is created, if set to 'Never', a regular pod is created. For the latter two --replicas must be 1. Default 'Always', for CronJobs `Never`.\x00The type of secret to create\x00Type for this service: ClusterIP, NodePort, or LoadBalancer. Default is 'ClusterIP'.\x00Undo a previous rollout\x00Unsets an individual value in a kubeconfig file\x00Update field(s) of a resource using strategic merge patch\x00Update image of a pod template\x00Update resource requests/limits on objects with pod templates\x00Update the annotations on a resource\x00Update the labels on a resource\x00Update the taints on one or more nodes\x00Username for Docker registry authentication\x00View latest last-applied-configuration annotations of a resource/object\x00View rollout history\x00Where to output the files. If empty or '-' uses stdout, otherwise creates a directory hierarchy in that directory\x00dummy restart flag)\x00external name of service\x00kubectl controls the Kubernetes cluster manager\x00Project-Id-Version: kubernetes\nReport-Msgid-Bugs-To: EMAIL\nPOT-Creation-Date: 2017-03-14 21:32-0700\nPO-Revision-Date: 2017-08-04 17:54+0200\nLanguage: it_IT\nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\nPlural-Forms: nplurals=2; plural=(n != 1);\nLast-Translator: Luca Berton \nLanguage-Team: Luca Berton \nX-Generator: Poedit 1.8.7.1\nX-Poedit-SourceCharset: UTF-8\n\x00\n\t\t # Creare un ClusterRoleBinding per user1, user2 e group1 utilizzando il cluster-admin ClusterRole\n\t\t kubectl create clusterrolebinding cluster-admin --clusterrole=cluster-admin --user=user1 --user=user2 --group=group1\x00\n\t\t # Crea un RoleBinding per user1, user2, and group1 utilizzando l'admin ClusterRole\n\t\t kubectl create rolebinding admin --clusterrole=admin --user=user1 --user=user2 --group=group1\x00\n\t\t # Crea un nuovo configmap denominato my-config in base alla cartella bar\n\t\t kubectl create configmap my-config --from-file=path/to/bar\n\n\t\t # Crea un nuovo configmap denominato my-config con le chiavi specificate anzich\u00e9 i nomi dei file su disco\n\t\t kubectl create configmap my-config --from-file=key1=/path/to/bar/file1.txt --from-file=key2=/path/to/bar/file2.txt\n\n\t\t # Crea un nuovo configmap denominato my-config con key1 = config1 e key2 = config2\n\t\t kubectl create configmap my-config --from-literal=key1=config1 --from-literal=key2=config2\x00\n\t\t # Se non si dispone ancora di un file .dockercfg, \u00e8 possibile creare un secret dockercfg direttamente utilizzando:\n\t\t kubectl create secret docker-registry my-secret --docker-server=DOCKER_REGISTRY_SERVER --docker-username=DOCKER_USER --docker-password=DOCKER_PASSWORD --docker-email=DOCKER_EMAIL\x00\n\t\t # Mostra metriche per tutti i nodi\n\t\t kubectl top node\n\n\t\t # Mostra metriche per un determinato nodo\n\t\t kubectl top node NODE_NAME\x00\n\t\t# Applica la configurazione pod.json a un pod.\n\t\tkubectl apply -f ./pod.json\n\n\t\t# Applicare il JSON passato in stdin a un pod.\n\t\tcat pod.json | kubectl apply -f -\n\n\t\t# Nota: --prune \u00e8 ancora in in Alpha\n\t\t# Applica la configurazione manifest.yaml che corrisponde alla label app = nginx ed elimina tutte le altre risorse che non sono nel file e nella label corrispondente app = nginx.\n\t\tkubectl apply --prune -f manifest.yaml -l app=nginx\n\n\t\t# Applica la configurazione manifest.yaml ed elimina tutti gli altri configmaps non presenti nel file.\n\t\tkubectl apply --prune -f manifest.yaml --all --prune-whitelist=core/v1/ConfigMap\x00\n\t\t# Auto scale un deployment \"foo\", con il numero di pod compresi tra 2 e 10, utilizzo della CPU target specificato in modo da utilizzare una politica di autoscaling predefinita:\n\t\tkubectl autoscale deployment foo --min=2 --max=10\n\n\t\t# Auto scale un controller di replica \"foo\", con il numero di pod compresi tra 1 e 5, utilizzo dell'utilizzo della CPU a 80%:\n\t\tkubectl autoscale rc foo --max=5 --cpu-percent=80\x00\n\t\t# Converte 'pod.yaml' alla versione pi\u00f9 recente e stampa in stdout.\n\t\tkubectl convert -f pod.yaml\n\n\t\t# Converte lo stato live della risorsa specificata da 'pod.yaml' nella versione pi\u00f9 recente.\n\t\t# e stampa in stdout nel formato json.\n\t\tkubectl convert -f pod.yaml --local -o json\n\n\t\t# Converte tutti i file nella directory corrente alla versione pi\u00f9 recente e li crea tutti.\n\t\tkubectl convert -f . | kubectl create -f -\x00\n\t\t# Crea un ClusterRole denominato \"pod-reader\" che consente all'utente di eseguire \"get\", \"watch\" e \"list\" sui pod\n\t\tkubectl create clusterrole pod-reader --verb=get,list,watch --resource=pods\n\n\t\t# Crea un ClusterRole denominato \"pod-reader\" con ResourceName specificato\n\t\tkubectl create clusterrole pod-reader --verb=get,list,watch --resource=pods --resource-name=readablepod\x00\n\t\t# Crea un ruolo denominato \"pod-reader\" che consente all'utente di eseguire \"get\", \"watch\" e \"list\" sui pod\n\t\tkubectl create role pod-reader --verb=get --verb=list --verb=watch --resource=pods\n\n\t\t# Crea un ruolo denominato \"pod-reader\" con ResourceName specificato\n\t\tkubectl create role pod-reader --verb=get --verg=list --verb=watch --resource=pods --resource-name=readablepod\x00\n\t\t# Crea una nuova resourcequota chiamata my-quota\n\t\tkubectl create quota my-quota --hard=cpu=1,memory=1G,pods=2,services=3,replicationcontrollers=2,resourcequotas=1,secrets=5,persistentvolumeclaims=10\n\n\t\t# Creare una nuova resourcequota denominata best-effort\n\t\tkubectl create quota best-effort --hard=pods=100 --scopes=BestEffort\x00\n\t\t# Crea un pod disruption budget chiamato my-pdb che seleziona tutti i pod con label app = rail\n\t\t# e richiede che almeno uno di essi sia disponibile in qualsiasi momento.\n\t\tkubectl create poddisruptionbudget my-pdb --selector=app=rails --min-available=1\n\n\t\t# Crea un pod disruption budget con nome my-pdb che seleziona tutti i pod con label app = nginx \n\t\t# e richiede che almeno la met\u00e0 dei pod selezionati sia disponibile in qualsiasi momento.\n\t\tkubectl create pdb my-pdb --selector=app=nginx --min-available=50%\x00\n\t\t# Crea un pod utilizzando i dati in pod.json.\n\t\tkubectl create -f ./pod.json\n\n\t\t# Crea un pod basato sul JSON passato in stdin.\n\t\tcat pod.json | kubectl create -f -\n\n\t\t# Modifica i dati in docker-registry.yaml in JSON utilizzando il formato API v1 quindi creare la risorsa utilizzando i dati modificati.\n\t\tkubectl create -f docker-registry.yaml --edit --output-version=v1 -o json\x00\n\t\t# Crea un servizio per un nginx replicato, che serve nella porta 80 e si collega ai container sulla porta 8000.\n\t\tkubectl expose rc nginx --port=80 --target-port=8000\n\n\t\t# Crea un servizio per un controller di replica identificato per tipo e nome specificato in \"nginx-controller.yaml\", che serve nella porta 80 e si collega ai container sulla porta 8000.\n\t\tkubectl expose -f nginx-controller.yaml --port=80 --target-port=8000\n\n\t\t# Crea un servizio per un pod valid-pod, che serve nella porta 444 con il nome \"frontend\"\n\t\tkubectl expose pod valid-pod --port=444 --name=frontend\n\n\t\t# Crea un secondo servizio basato sul servizio sopra, esponendo la porta container 8443 come porta 443 con il nome \"nginx-https\"\n\t\tkubectl expose service nginx --port=443 --target-port=8443 --name=nginx-https\n\n\t\t# Crea un servizio per un'applicazione di replica in porta 4100 che bilanci il traffico UDP e denominato \"video stream\".\n\t\tkubectl expose rc streamer --port=4100 --protocol=udp --name=video-stream\n\n\t\t# Crea un servizio per un nginx replicato utilizzando l'insieme di replica, che serve nella porta 80 e si collega ai contenitori sulla porta 8000.\n\t\tkubectl expose rs nginx --port=80 --target-port=8000\n\n\t\t# Crea un servizio per una distribuzione di nginx, che serve nella porta 80 e si collega ai contenitori della porta 8000.\n\t\tkubectl expose deployment nginx --port=80 --target-port=8000\x00\n\t\t# Elimina un pod utilizzando il tipo e il nome specificati in pod.json.\n\t\tkubectl delete -f ./pod.json\n\n\t\t# Elimina un pod in base al tipo e al nome del JSON passato in stdin.\n\t\tcat pod.json | kubectl delete -f -\n\n\t\t# Elimina i baccelli ei servizi con gli stessi nomi \"baz\" e \"foo\"\n\t\tkubectl delete pod,service baz foo\n\n\t\t# Elimina i baccelli ei servizi con il nome dell'etichetta = myLabel.\n\t\tkubectl delete pods,services -l name=myLabel\n\n\t\t# Eliminare un pod con un ritardo minimo\n\t\tkubectl delete pod foo --now\n\n\t\t# Forza elimina un pod in un nodo morto\n\t\tkubectl delete pod foo --grace-period=0 --force\n\n\t\t# Elimina tutti i pod\n\t\tkubectl delete pods --all\x00\n\t\t# Descrive un nodo\n\t\tkubectl describe nodes kubernetes-node-emt8.c.myproject.internal\n\n\t\t# Descrive un pod\n\t\tkubectl describe pods/nginx\n\n\t\t# Descrive un pod identificato da tipo e nome in \"pod.json\"\n\t\tkubectl describe -f pod.json\n\n\t\t# Descrive tutti i pod\n\t\tkubectl describe pods\n\n\t\t# Descrive i pod con label name=myLabel\n\t\tkubectl describe po -l name=myLabel\n\n\t\t# Descrivere tutti i pod gestiti dal controller di replica \"frontend\" (rc-created pods\n\t\t# ottiene il nome del rc come un prefisso del nome pod).\n\t\tkubectl describe pods frontend\x00\n\t\t# Drain node \"foo\", anche se ci sono i baccelli non gestiti da ReplicationController, ReplicaSet, Job, DaemonSet o StatefulSet su di esso.\n\t\t$ kubectl drain foo --force\n\n\t\t# Come sopra, ma interrompere se ci sono i baccelli non gestiti da ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet, e utilizzare un periodo di grazia di 15 minuti.\n\t\t$ kubectl drain foo --grace-period=900\x00\n\t\t# Modifica il servizio denominato 'docker-registry':\n\t\tkubectl edit svc/docker-registry\n\n\t\t# Usa un editor alternativo\n\t\tKUBE_EDITOR=\"nano\" kubectl edit svc/docker-registry\n\n\t\t# Modifica il lavoro 'myjob' in JSON utilizzando il formato API v1:\n\t\tkubectl edit job.v1.batch/myjob -o json\n\n\t\t# Modifica la distribuzione 'mydeployment' in YAML e salvare la configurazione modificata nella sua annotazione:\n\t\tkubectl edit deployment/mydeployment -o yaml --save-config\x00\n\t\t# Ottieni l'output dalla 'data' di esecuzione del pod 123456-7890, utilizzando il primo contenitore per impostazione predefinita\n\t\tkubectl exec 123456-7890 date\n\n\t\t# Ottieni l'output dalla data di esecuzione in ruby-container del pod 123456-7890\n\t\tkubectl exec 123456-7890 -c ruby-container date\n\n\t\t# Passare alla modalit\u00e0 raw terminal, invia stdin a 'bash' in ruby-container del pod 123456-7890\n\t\t# and sends stdout/stderr from 'bash' back to the client\n\t\tkubectl exec 123456-7890 -c ruby-container -i -t -- bash -il\x00\n\t\t# Ottieni l'output dal pod 123456-7890 in esecuzione, utilizzando il primo contenitore per impostazione predefinita\n\t\tkubectl attach 123456-7890\n\n\t\t# Ottieni l'output dal ruby-container del pod 123456-7890\n\t\tkubectl attach 123456-7890 -c ruby-container\n\n\t\t# Passa alla modalit\u00e0 raw terminal, invia stdin a 'bash' in ruby-container del pod 123456-7890\n\t\t# e invia stdout/stderr da 'bash' al client\n\t\tkubectl attach 123456-7890 -c ruby-container -i -t\n\n\t\t# Ottieni l'output dal primo pod di una ReplicaSet denominata nginx\n\t\tkubectl attach rs/nginx\n\t\t\x00\n\t\t# Ottieni la documentazione della risorsa e i relativi campi\n\t\tkubectl explain pods\n\n\t\t# Ottieni la documentazione di un campo specifico di una risorsa\n\t\tkubectl explain pods.spec.containers\x00\n\t\t# Installa il completamento di bash su un Mac utilizzando homebrew\n\t\tbrew install bash-completion\n\t\tprintf \"\n# Bash completion support\nsource $(brew --prefix)/etc/bash_completion\n\" >> $HOME/.bash_profile\n\t\tsource $HOME/.bash_profile\n\n\t\t# Carica il codice di completamento kubectl per bash nella shell corrente\n\t\tsource <(kubectl completion bash)\n\n\t\t# Scrive il codice di completamento bash in un file e lo carica da .bash_profile\n\t\tkubectl completion bash > ~/.kube/completion.bash.inc\n\t\tprintf \"\n# Kubectl shell completion\nsource '$HOME/.kube/completion.bash.inc'\n\" >> $HOME/.bash_profile\n\t\tsource $HOME/.bash_profile\n\n\t\t# Carica il codice di completamento kubectl per zsh [1] nella shell corrente\n\t\tsource <(kubectl completion zsh)\x00\n\t\t# Elenca tutti i pod in formato output ps.\n\t\tkubectl get pods\n\n\t\t# Elenca tutti i pod in formato output ps con maggiori informazioni (ad esempio il nome del nodo).\n\t\tkubectl get pods -o wide\n\n\t\t# Elenca un controller di replica singolo con NAME specificato nel formato di output ps.\n\t\tkubectl get replicationcontroller web\n\n\t\t# Elenca un singolo pod nel formato di uscita JSON.\n\t\tkubectl get -o json pod web-pod-13je7\n\n\t\t# Elenca un pod identificato per tipo e nome specificato in \"pod.yaml\" nel formato di uscita JSON.\n\t\tkubectl get -f pod.yaml -o json\n\n\t\t# Restituisce solo il valore di fase del pod specificato.\n\t\tkubectl get -o template pod/web-pod-13je7 --template={{.status.phase}}\n\n\t\t# Elenca tutti i controller e servizi di replica insieme in formato output ps.\n\t\tkubectl get rc,services\n\n\t\t# Elenca una o pi\u00f9 risorse per il tipo e per i nomi.\n\t\tkubectl get rc/web service/frontend pods/web-pod-13je7\n\n\t\t# Elenca tutte le risorse con tipi diversi.\n\t\tkubectl get all\x00\n\t\t# Ascolta localmente le porte 5000 e 6000, inoltrando i dati da/verso le porte 5000 e 6000 nel pod\n\t\tkubectl port-forward mypod 5000 6000\n\n\t\t# Ascolta localmente la porta 8888, inoltra a 5000 nel pod\n\t\tkubectl port-forward mypod 8888:5000\n\n\t\t# Ascolta localmente una porta casuale, inoltra a 5000 nel pod\n\t\tkubectl port-forward mypod :5000\n\n\t\t# Ascolta localmente una porta casuale, inoltra a 5000 nel pod\n\t\tkubectl port-forward mypod 0:5000\x00\n\t\t# Segna il nodo \"foo\" come programmabile.\n\t\t$ Kubectl uncordon foo\x00\n\t\t# Segna il nodo \"foo\" come non programmabile.\n\t\tkubectl cordon foo\x00\n\t\t# Aggiorna parzialmente un nodo utilizzando merge patch strategica\n\t\tkubectl patch node k8s-node-1 -p '{\"spec\":{\"unschedulable\":true}}'\n\n\t\t# Aggiorna parzialmente un nodo identificato dal tipo e dal nome specificato in \"node.json\" utilizzando merge patch strategica\n\t\tkubectl patch -f node.json -p '{\"spec\":{\"unschedulable\":true}}'\n\n\t\t# Aggiorna l'immagine di un contenitore; spec.containers [*]. name \u00e8 richiesto perch\u00e9 \u00e8 una chiave di fusione\n\t\tkubectl patch pod valid-pod -p '{\"spec\":{\"containers\":[{\"name\":\"kubernetes-serve-hostname\",\"image\":\"new image\"}]}}'\n\n\t\t# Aggiorna l'immagine di un contenitore utilizzando una patch json con array posizionali\n\t\tkubectl patch pod valid-pod --type='json' -p='[{\"op\": \"replace\", \"path\": \"/spec/containers/0/image\", \"value\":\"new image\"}]'\x00\n\t\t# Stampa i flag ereditati da tutti i comandi\n\t\tkubectl options\x00\n\t\t# Stampa l'indirizzo dei servizi master e cluster\n\t\tkubectl cluster-info\x00\n\t\t# Stampa le versioni client e server per il current context\n\t\tkubectl version\x00\n\t\t# Stampa le versioni API supportate\n\t\tkubectl api-versions\x00\n\t\t# Scala un replicaset denominato 'foo' a 3.\n\t\tkubectl scale --replicas=3 rs/foo\n\n\t\t# Scala una risorsa identificata per tipo e nome specificato in \"foo.yaml\" a 3.\n\t\tkubectl scale --replicas=3 -f foo.yaml\n\n\t\t# Se la distribuzione corrente di mysql \u00e8 2, scala mysql a 3.\n\t\tkubectl scale --current-replicas=2 --replicas=3 deployment/mysql\n\n\t\t# Scalare pi\u00f9 controllori di replica.\n\t\tkubectl scale --replicas=5 rc/foo rc/bar rc/baz\n\n\t\t# Scala il lavoro denominato 'cron' a 3.\n\t\tkubectl scale --replicas=3 job/cron\x00\n\t\t# Imposta l'ultima-configurazione-applicata di una risorsa che corrisponda al contenuto di un file.\n\t\tkubectl apply set-last-applied -f deploy.yaml\n\n\t\t# Esegue set-last-applied per ogni file di configurazione in una directory.\n\t\tkubectl apply set-last-applied -f path/\n\n\t\t# Imposta la configurazione dell'ultima applicazione di una risorsa che corrisponda al contenuto di un file, creer\u00e0 l'annotazione se non esiste gi\u00e0.\n\t\tkubectl apply set-last-applied -f deploy.yaml --create-annotation=true\n\t\t\x00\n\t\t# Spegni foo.\n\t\tkubectl stop replicationcontroller foo\n\n\t\t# Stop di tutti i pod e servizi con label name=myLabel.\n\t\tkubectl stop pods,services -l name=myLabel\n\n\t\t# Spegnere il servizio definito in service.json\n\t\tkubectl stop -f service.json\n\n\t\t# Spegnere tutte le resources in path/to/resources directory\n\t\tkubectl stop -f path/to/resources\x00\n\t\t# Visualizza le annotazioni dell'ultima-configurazione-applicata per tipo/nome in YAML.\n\t\tkubectl apply view-last-applied deployment/nginx\n\n\t\t# # Visualizza le annotazioni dell'ultima-configurazione-applicata per file in JSON.\n\t\tkubectl apply view-last-applied -f deploy.yaml -o json\x00\n\t\tApplicare una configurazione a una risorsa per nomefile o stdin.\n\t\tQuesta risorsa verr\u00e0 creata se non esiste ancora.\n\t\tPer utilizzare 'apply', creare sempre la risorsa inizialmente con 'apply' o 'create --save-config'.\n\n\t\tSono accettati i formati JSON e YAML.\n\n\t\tDisclaimer Alpha: la funzionalit\u00e0 --prune non \u00e8 ancora completa. Non utilizzare a meno che non si sia a conoscenza di quale sia lo stato attuale. Vedi https://issues.k8s.io/34274.\x00\n\t\nCrea un ClusterRole.\x00\n\t\tCrea un ClusterRoleBinding per un ClusterRole particolare.\x00\n\t\tCrea un RoleBinding per un particolare Ruolo o ClusterRole.\x00\n\t\tCrea un TLS secret dalla coppia di chiavi pubblica/privata.\n\n\t\tLa coppia di chiavi pubblica/privata deve esistere prima. Il certificato chiave pubblica deve essere .PEM codificato e corrispondere alla chiave privata data.\x00\n\t\tCreare un namespace con il nome specificato.\x00\n\t\tCrea un pod disruption budget con il nome specificato, selector e il numero minimo di pod disponibili\x00\n\t\tCrea una risorsa per nome file o stdin.\n\n\t\tSono accettati i formati JSON e YAML.\x00\n\t\tCrea una resourcequota con il nome specificato, hard limits e gli scope opzionali\x00\n\t\tCrea un ruolo con una singola regola.\x00\n\t\tCreare un service account con il nome specificato.\x00\n\t\tCrea ed esegue un'immagine particolare, eventualmente replicata.\n\n\t\tCrea un deployment o un job per gestire i container creati.\x00\n\t\tCrea un autoscaler che automaticamente sceglie e imposta il numero di pod che vengono eseguiti in un cluster di kubernets.\n\n\t\tEsegue una ricerca di un Deployment, ReplicaSet o ReplicationController per nome e crea un autoscaler che utilizza la risorsa indicata come riferimento.\n\t\tUn autoscaler pu\u00f2 aumentare o diminuire automaticamente il numero di pod distribuiti all'interno del sistema se necessario.\x00\n\t\tVisualizza l'utilizzo di risorse (CPU/Memoria/Storage) dei nodi.\n\n\t\tIl comando top-node consente di visualizzare il consumo di risorse dei nodi.\x00\n\t\tVisualizza l'utilizzo di risorse (CPU/Memoria/Storage) dei pod.\n\n\t\tIl comando \"top pod\" consente di visualizzare il consumo delle risorse dei pod.\n\n\t\tA causa del ritardo della pipeline metrica, potrebbero non essere disponibili per alcuni minuti\n\t\teal momento della creazione dei pod.\x00\n\t\tVisualizza l'utilizzo di risorse (CPU/Memoria/Storage).\n\n\t\tIl comando top consente di visualizzare il consumo di risorse per nodi o pod.\n\n\t\tQuesto comando richiede che Heapster sia configurato correttamente e che funzioni sul server.\x00\n\t\tContrassegna il nodo come programmabile.\x00\n\t\tContrassegnare il nodo come non programmabile.\x00\n\t\tImposta una nuova dimensione per Deployment, ReplicaSet, Replication Controller, o Job.\n\n\t\tScala consente anche agli utenti di specificare una o pi\u00f9 condizioni preliminari per l'azione della scala.\n\n\t\tSe --current-replicas o --resource-version sono specificate, viene convalidata prima di\n\t\ttentare scale, ed \u00e8 garantito che la precondizione vale quando\n\t\tscale viene inviata al server..\x00\n\t\tImposta le annotazioni dell'ultima-configurazione-applicata impostandola in modo che corrisponda al contenuto di un file.\n\t\tCi\u00f2 determina l'aggiornamento dell'ultima-configurazione-applicata come se 'kubectl apply -f ' fosse stato eseguito,\n\t\tsenza aggiornare altre parti dell'oggetto.\x00\n\t\tPer proxy tutti i kubernetes api e nient'altro, utilizzare:\n\n\t\t $ kubectl proxy --api-prefix=/\n\n\t\tPer proxy solo una parte dei kubernetes api e anche alcuni file static\n\n\t\t $ kubectl proxy --www=/my/files --www-prefix=/static/ --api-prefix=/api/\n\n\t\tQuanto sopra consente 'curl localhost:8001/api/v1/pods'.\n\n\t\tPer eseguire il proxy tutti i kubernetes api in una radice diversa, utilizzare:\n\n\t\t $ kubectl proxy --api-prefix=/custom/\n\n\t\tQuanto sopra ti permette 'curl localhost:8001/custom/api/v1/pods'\x00\n\t\tAggiorna i campi di una risorsa utilizzando la merge patch strategica\n\n\t\tSono accettati i formati JSON e YAML.\n\n\t\tSi prega di fare riferimento ai modelli in https://htmlpreview.github.io/?https://github.com/kubernetes/kubernetes/blob/HEAD/docs/api-reference/v1/definitions.html per trovare se un campo \u00e8 mutevole.\x00\n\t\tAggiorna le label di una risorsa.\n\n\t\t* A label must begin with a letter or number, and may contain letters, numbers, hyphens, dots, and underscores, up to %[1]d characters.\n\t\t* If --overwrite is true, then existing labels can be overwritten, otherwise attempting to overwrite a label will result in an error.\n\t\t* If --resource-version is specified, then updates will use this resource version, otherwise the existing resource-version will be used.\x00\n\t\tVisualizza le annotazioni dell'ultima-configurazione-applicata per tipo/nome o file.\n\n\t\tL'output predefinito verr\u00e0 stampato su stdout nel formato YAML. Si pu\u00f2 usare l'opzione -o\n\t\tPer cambiare il formato di output.\x00\n\t # Crea un nuovo secret TLS denominato tls-secret con la coppia di dati fornita:\n\t kubectl create secret tls tls-secret --cert=path/to/tls.cert --key=path/to/tls.key\x00\n\t # Crea un nuovo namespace denominato my-namespace\n\t kubectl create namespace my-namespace\x00\n\t # Crea un nuovo secret denominato my-secret con i tasti per ogni file nella barra delle cartelle\n\t kubectl create secret generic my-secret --from-file=path/to/bar\n\n\t # Crea un nuovo secret denominato my-secret con le chiavi specificate anzich\u00e9 i nomi sul disco\n\t kubectl create secret generic my-secret --from-file=ssh-privatekey=~/.ssh/id_rsa --from-file=ssh-publickey=~/.ssh/id_rsa.pub\n\n\t # Crea un nuovo secret denominato my-secret con key1 = supersecret e key2 = topsecret\n\t kubectl create secret generic my-secret --from-literal=key1=supersecret --from-literal=key2=topsecret\x00\n\t # Crea un nuovo service account denominato my-service-account\n\t kubectl create serviceaccount my-service-account\x00\n\t# Crea un nuovo servizio ExternalName denominato my-ns \n\tkubectl create service externalname my-ns --external-name bar.com\x00\n\tCrea un servizio ExternalName con il nome specificato.\n\n\tIl servizio ExternalName fa riferimento a un indirizzo DNS esterno \n\tsolo pod, che permetteranno agli autori delle applicazioni di utilizzare i servizi di riferimento\n\tche esistono fuori dalla piattaforma, su altri cluster, o localmente..\x00\n\tHelp fornisce assistenza per qualsiasi comando nell'applicazione.\n\tBasta digitare kubectl help [path to command] per i dettagli completi.\x00\n # Creare un nuovo servizio LoadBalancer denominato my-lbs\n kubectl create service loadbalancer my-lbs --tcp=5678:8080\x00\n # Creare un nuovo servizio clusterIP denominato my-cs\n kubectl create service clusterip my-cs --tcp=5678:8080\n\n # Creare un nuovo servizio clusterIP denominato my-cs (in modalit\u00e0 headless)\n kubectl create service clusterip my-cs --clusterip=\"None\"\x00\n # Crea una nuovo deployment chiamato my-dep che esegue l'immagine busybox.\n kubectl create deployment my-dep --image=busybox\x00\n # Creare un nuovo servizio nodeport denominato my-ns\n kubectl create service nodeport my-ns --tcp=5678:8080\x00\n # Dump dello stato corrente del cluster verso stdout\n kubectl cluster-info dump\n\n # Dump dello stato corrente del cluster verso /path/to/cluster-state\n kubectl cluster-info dump --output-directory=/path/to/cluster-state\n\n # Dump di tutti i namespaces verso stdout\n kubectl cluster-info dump --all-namespaces\n\n # Dump di un set di namespace verso /path/to/cluster-state\n kubectl cluster-info dump --namespaces default,kube-system --output-directory=/path/to/cluster-state\x00\n # Aggiorna il pod 'foo' con annotazione 'description'e il valore 'my frontend'.\n # Se la stessa annotazione \u00e8 impostata pi\u00f9 volte, verr\u00e0 applicato solo l'ultimo valore\n kubectl annotate pods foo description='my frontend'\n\n # Aggiorna un pod identificato per tipo e nome in \"pod.json\"\n kubectl annotate -f pod.json description='my frontend'\n\n # Aggiorna pod 'foo' con la annotazione 'description' e il valore 'my frontend running nginx', sovrascrivendo qualsiasi valore esistente.\n kubectl annotate --overwrite pods foo description='my frontend running nginx'\n\n # Aggiorna tutti i baccelli nel namespace\n kubectl annotate pods --all description='my frontend running nginx'\n\n # Aggiorna il pod 'foo' solo se la risorsa \u00e8 invariata dalla versione 1.\n kubectl annotate pods foo description='my frontend running nginx' --resource-version=1\n\n # Aggiorna il pod 'foo' rimuovendo un'annotazione denominata 'descrizione' se esiste.\n # Non richiede flag -overwrite.\n kubectl annotate pods foo description-\x00\n Crea un servizio LoadBalancer con il nome specificato.\x00\n Crea un servizio clusterIP con il nome specificato.\x00\n Creare un deployment con il nome specificato.\x00\n Creare un servizio nodeport con il nome specificato.\x00\n Dump delle informazioni di cluster idonee per il debug e la diagnostica di problemi di cluster. Per impostazione predefinita, tutto\n\u00a0\u00a0\u00a0\u00a0verso stdout. \u00c8 possibile specificare opzionalmente una directory con --output-directory. Se si specifica una directory, kubernetes \n creear\u00e0 un insieme di file in quella directory. Per impostazione predefinita, dumps solo i dati del namespace \"kube-system\", ma \u00e8\n possibile passare ad namespace diverso con il flag --namespaces o specificare --all-namespaces per il dump di tutti i namespace.\n\n\u00a0\u00a0\u00a0\u00a0 Il comando esegue dump anche dei log di tutti i pod del cluster, questi log vengono scaricati in directory differenti\n\u00a0\u00a0\u00a0\u00a0 basati sul namespace e sul nome del pod.\x00\n Visualizza gli indirizzi del master e dei servizi con label kubernetes.io/cluster-service=true\n\u00a0\u00a0Per ulteriore debug e diagnosticare i problemi di cluster, utilizzare 'kubectl cluster-info dump'.\x00Un insieme delimitato-da-virgole di quota scopes che devono corrispondere a ciascun oggetto gestito dalla quota.\x00Un insieme delimitato-da-virgola di coppie risorsa = quantit\u00e0 che definiscono un hard limit.\x00Un label selector da utilizzare per questo budget. Sono supportati solo i selettori equality-based selector.\x00Un selettore di label da utilizzare per questo servizio. Sono supportati solo equality-based selector. Se vuota (default) dedurre il selettore dal replication controller o replica set.)\x00Un calendario in formato Cron del lavoro che deve essere eseguito.\x00Indirizzo IP esterno aggiuntivo (non gestito da Kubernetes) da accettare per il servizio. Se questo IP viene indirizzato a un nodo, \u00e8 possibile accedere da questo IP in aggiunta al service IP generato.\x00Un override JSON inline per l'oggetto generato. Se questo non \u00e8 vuoto, viene utilizzato per ignorare l'oggetto generato. Richiede che l'oggetto fornisca un campo valido apiVersion.\x00Un override JSON inline per l'oggetto di servizio generato. Se questo non \u00e8 vuoto, viene utilizzato per ignorare l'oggetto generato. Richiede che l'oggetto fornisca un campo valido apiVersion. Utilizzato solo se --expose \u00e8 true.\x00Applica una configurazione risorsa per nomefile o stdin\x00Approva una richiesta di firma del certificato\x00Assegnare il proprio ClusterIP o impostare su 'None' per un servizio 'headless' (nessun bilanciamento del carico).\x00Collega a un container in esecuzione\x00Auto-scale a Deployment, ReplicaSet, o ReplicationController\x00ClusterIP da assegnare al servizio. Lasciare vuoto per allocare automaticamente o impostare su 'None' per creare un servizio headless.\x00ClusterRole a cui questo ClusterRoleBinding fa riferimento\x00ClusterRole a cui questo RoleBinding fa riferimento\x00Nome container che avr\u00e0 la sua immagine aggiornata. Soltanto rilevante quando --image \u00e8 specificato, altrimenti ignorato. Necessario quando si utilizza --image su un contenitore a pi\u00f9 contenitori\x00Convertire i file di configurazione tra diverse versioni APIs\x00Copiare file e directory da e verso i container.\x00Crea un ClusterRoleBinding per un ClusterRole particolare\x00Creare un servizio LoadBalancer.\x00Crea un servizio NodePort.\x00Crea un RoleBinding per un particolare Role o ClusterRole\x00Crea un secret TLS\x00Crea un servizio clusterIP.\x00Crea un configmap da un file locale, una directory o un valore letterale\x00Creare un deployment con il nome specificato.\x00Crea un namespace con il nome specificato\x00Crea un pod disruption budget con il nome specificato.\x00Crea una quota con il nome specificato.\x00Crea una risorsa per nome file o stdin\x00Crea un secret da utilizzare con un registro Docker\x00Crea un secret da un file locale, una directory o un valore letterale\x00Crea un secret utilizzando un subcommand specificato\x00Creare un account di servizio con il nome specificato\x00Crea un servizio utilizzando il subcommand specificato.\x00Crea un servizio ExternalName.\x00Elimina risorse selezionate per nomi di file, stdin, risorse e nomi, o per risorsa e selettore di label\x00Elimina il cluster specificato dal kubeconfig\x00Elimina il context specificato dal kubeconfig\x00Nega una richiesta di firma del certificato\x00Deprecated: spegne correttamente una risorsa per nome o nome file\x00Descrive uno o pi\u00f9 context\x00Visualizza l'utilizzo di risorse (CPU/Memoria) per nodo\x00Visualizza l'utilizzo di risorse (CPU/Memoria) per pod.\x00Visualizza l'utilizzo di risorse (CPU/Memoria).\x00Visualizza informazioni sul cluster\x00Mostra i cluster definiti nel kubeconfig\x00Visualizza le impostazioni merged di kubeconfig o un file kubeconfig specificato\x00Visualizza una o pi\u00f9 risorse\x00Visualizza il current-context\x00Documentazione delle risorse\x00Drain node in preparazione alla manutenzione\x00Dump di un sacco di informazioni pertinenti per il debug e la diagnosi\x00Modificare una risorsa sul server\x00Email per il registro Docker\x00Esegui un comando in un contenitore\x00Politica esplicita per il pull delle immagini container. Richiesto quando --image \u00e8 uguale all'immagine esistente, altrimenti ignorata.\x00Inoltra una o pi\u00f9 porte locali a un pod\x00Aiuto per qualsiasi comando\x00IP da assegnare al Load Balancer. Se vuota, un IP effimero verr\u00e0 creato e utilizzato (specifico per provider cloud).\x00Se non \u00e8 vuoto, impostare l'affinit\u00e0 di sessione per il servizio; Valori validi: 'None', 'ClientIP'\x00Se non \u00e8 vuoto, l'aggiornamento delle annotazioni avr\u00e0 successo solo se questa \u00e8 la resource-version corrente per l'oggetto. Valido solo quando si specifica una singola risorsa.\x00Se non vuoto, l'aggiornamento delle label avr\u00e0 successo solo se questa \u00e8 la resource-version corrente per l'oggetto. Valido solo quando si specifica una singola risorsa.\x00Immagine da utilizzare per aggiornare il replication controller. Deve essere diversa dall'immagine esistente (nuova immagine o nuovo tag immagine). Non pu\u00f2 essere utilizzata con --filename/-f\x00Gestisci un deployment rollout\x00Contrassegnare il nodo come programmabile\x00Contrassegnare il nodo come non programmabile\x00Imposta la risorsa indicata in pausa\x00Modificare le risorse del certificato.\x00Modifica i file kubeconfig\x00Nome o numero di porta nel container verso il quale il servizio deve dirigere il traffico. Opzionale.\x00Restituisce solo i log dopo una data specificata (RFC3339). Predefinito tutti i log. \u00c8 possibile utilizzare solo uno tra data-inizio/a-partire-da.\x00Codice di completamento shell di output per la shell specificata (bash o zsh)\x00Output dell'oggetto formattato con la versione del gruppo fornito (per esempio: 'extensions/v1beta1').)\x00Password per l'autenticazione al registro di Docker\x00Percorso certificato di chiave pubblica codificato PEM.\x00Percorso alla chiave privata associata a un certificato specificato.\x00Eseguire un rolling update del ReplicationController specificato\x00Prerequisito per la versione delle risorse. Richiede che la versione corrente delle risorse corrisponda a questo valore per scalare.\x00Stampa per client e server le informazioni sulla versione\x00Stampa l'elenco flag ereditati da tutti i comandi\x00Stampa i log per container in un pod\x00Sostituire una risorsa per nomefile o stdin\x00Riprendere una risorsa in pausa\x00Ruolo di riferimento per RoleBinding\x00Esegui una particolare immagine nel cluster\x00Eseguire un proxy al server Kubernetes API\x00Posizione del server per il Registro Docker\x00Imposta una nuova dimensione per Deployment, ReplicaSet, Replication Controller, o Job\x00Imposta caratteristiche specifiche sugli oggetti\x00Imposta l'annotazione dell'ultima-configurazione-applicata ad un oggetto live per abbinare il contenuto di un file.\x00Impostare il selettore di una risorsa\x00Imposta una voce cluster in kubeconfig\x00Imposta una voce context in kubeconfig\x00Imposta una voce utente in kubeconfig\x00Imposta un singolo valore in un file kubeconfig\x00Imposta il current-context in un file kubeconfig\x00Mostra i dettagli di una specifiche risorsa o un gruppo di risorse\x00Mostra lo stato del rollout\x00Sinonimo di --target-port\x00Prende un replication controller, service, deployment o un pod e lo espone come nuovo servizio Kubernetes\x00L'immagine per il container da eseguire.\x00La politica di pull dell'immagine per il container. Se lasciato vuoto, questo valore non verr\u00e0 specificato dal client e predefinito dal server\x00La chiave da utilizzare per distinguere tra due controller diversi, predefinito \"deployment\". Rilevante soltanto quando --image \u00e8 specificato, altrimenti ignorato\x00Il numero minimo o la percentuale di pod disponibili che questo budget richiede.\x00Il nome dell'oggetto appena creato.\x00Il nome dell'oggetto appena creato. Se non specificato, verr\u00e0 utilizzato il nome della risorsa di input.\x00Il nome del generatore API da utilizzare, si veda http://kubernetes.io/docs/user-guide/kubectl-conventions/#generators per un elenco.\x00Il nome del generatore API da utilizzare. Attualmente c'\u00e8 solo 1 generatore.\x00Il nome del generatore API da utilizzare. Ci sono 2 generatori: 'service/v1' e 'service/v2'. L'unica differenza tra loro \u00e8 che la porta di servizio in v1 \u00e8 denominata \"predefinita\", mentre viene lasciata unnamed in v2. Il valore predefinito \u00e8 'service/v2'.\x00Il nome del generatore da utilizzare per la creazione di un servizio. Utilizzato solo se --expose \u00e8 true\x00Il protocollo di rete per il servizio da creare. Il valore predefinito \u00e8 'TCP'.\x00La porta che il servizio deve servire. Copiato dalla risorsa esposta, se non specificata\x00La porta che questo contenitore espone. Se --expose \u00e8 true, questa \u00e8 anche la porta utilizzata dal servizio creato.\x00I limiti delle richieste di risorse per questo contenitore. Ad esempio, 'cpu=200m,memory=512Mi'. Si noti che i componenti lato server possono assegnare i limiti a seconda della configurazione del server, ad esempio intervalli di limiti.\x00La risorsa necessita di richieste di requisiti per questo pod. Ad esempio, 'cpu = 100m, memoria = 256Mi'. Si noti che i componenti lato server possono assegnare i requisiti a seconda della configurazione del server, ad esempio intervalli di limiti.\x00La politica di riavvio per questo Pod. Valori accettati [Always, OnFailure, Never]. Se impostato su 'Always' viene creato un deployment, se impostato su 'OnFailure' viene creato un job, se impostato su 'Never', viene creato un pod. Per questi ultimi due le - repliche devono essere 1. Predefinito 'Always', per CronJobs `Never`.\x00Tipo di segreto da creare\x00Digitare per questo servizio: ClusterIP, NodePort o LoadBalancer. Ppredefinito \u00e8 'ClusterIP'.\x00Annulla un precedente rollout\x00Annulla singolo valore in un file kubeconfig\x00Aggiornare campo/i risorsa utilizzando merge patch strategici\x00Aggiorna immagine di un pod template\x00Aggiorna richieste di risorse/limiti sugli oggetti con pod template\x00Aggiorna annotazioni di risorsa\x00Aggiorna label di una risorsa\x00Aggiorna i taints su uno o pi\u00f9 nodi\x00Nome utente per l'autenticazione nel registro Docker\x00Visualizza ultime annotazioni dell'ultima configurazione applicata per risorsa/oggetto\x00Visualizza la storia del rollout\x00Dove eseguire l'output dei file. Se vuota o '-' utilizza lo stdout, altrimenti crea una gerarchia di directory in quella directory\x00flag di riavvio finto)\x00nome esterno del servizio\x00Kubectl controlla il gestore cluster di Kubernetes\x00") + +func translationsKubectlIt_itLc_messagesK8sMoBytes() ([]byte, error) { + return _translationsKubectlIt_itLc_messagesK8sMo, nil +} + +func translationsKubectlIt_itLc_messagesK8sMo() (*asset, error) { + bytes, err := translationsKubectlIt_itLc_messagesK8sMoBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "translations/kubectl/it_IT/LC_MESSAGES/k8s.mo", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _translationsKubectlIt_itLc_messagesK8sPo = []byte(`# Italian translation. +# Copyright (C) 2017 +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR evolution85@gmail.com, 2017. +# +msgid "" +msgstr "" +"Project-Id-Version: kubernetes\n" +"Report-Msgid-Bugs-To: EMAIL\n" +"POT-Creation-Date: 2017-03-14 21:32-0700\n" +"PO-Revision-Date: 2017-08-04 17:54+0200\n" +"Language: it_IT\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"Last-Translator: Luca Berton \n" +"Language-Team: Luca Berton \n" +"X-Generator: Poedit 1.8.7.1\n" +"X-Poedit-SourceCharset: UTF-8\n" + +#: pkg/kubectl/cmd/create_clusterrolebinding.go:35 +msgid "" +"\n" +"\t\t # Create a ClusterRoleBinding for user1, user2, and group1 using the " +"cluster-admin ClusterRole\n" +"\t\t kubectl create clusterrolebinding cluster-admin --clusterrole=cluster-" +"admin --user=user1 --user=user2 --group=group1" +msgstr "" +"\n" +"\t\t # Creare un ClusterRoleBinding per user1, user2 e group1 utilizzando il " +"cluster-admin ClusterRole\n" +"\t\t kubectl create clusterrolebinding cluster-admin --clusterrole=cluster-" +"admin --user=user1 --user=user2 --group=group1" + +#: pkg/kubectl/cmd/create_rolebinding.go:35 +msgid "" +"\n" +"\t\t # Create a RoleBinding for user1, user2, and group1 using the admin " +"ClusterRole\n" +"\t\t kubectl create rolebinding admin --clusterrole=admin --user=user1 --" +"user=user2 --group=group1" +msgstr "" +"\n" +"\t\t # Crea un RoleBinding per user1, user2, and group1 utilizzando l'admin " +"ClusterRole\n" +"\t\t kubectl create rolebinding admin --clusterrole=admin --user=user1 --" +"user=user2 --group=group1" + +#: pkg/kubectl/cmd/create_configmap.go:44 +msgid "" +"\n" +"\t\t # Create a new configmap named my-config based on folder bar\n" +"\t\t kubectl create configmap my-config --from-file=path/to/bar\n" +"\n" +"\t\t # Create a new configmap named my-config with specified keys instead of " +"file basenames on disk\n" +"\t\t kubectl create configmap my-config --from-file=key1=/path/to/bar/file1." +"txt --from-file=key2=/path/to/bar/file2.txt\n" +"\n" +"\t\t # Create a new configmap named my-config with key1=config1 and " +"key2=config2\n" +"\t\t kubectl create configmap my-config --from-literal=key1=config1 --from-" +"literal=key2=config2" +msgstr "" +"\n" +"\t\t # Crea un nuovo configmap denominato my-config in base alla cartella " +"bar\n" +"\t\t kubectl create configmap my-config --from-file=path/to/bar\n" +"\n" +"\t\t # Crea un nuovo configmap denominato my-config con le chiavi " +"specificate anziché i nomi dei file su disco\n" +"\t\t kubectl create configmap my-config --from-file=key1=/path/to/bar/file1." +"txt --from-file=key2=/path/to/bar/file2.txt\n" +"\n" +"\t\t # Crea un nuovo configmap denominato my-config con key1 = config1 e " +"key2 = config2\n" +"\t\t kubectl create configmap my-config --from-literal=key1=config1 --from-" +"literal=key2=config2" + +#: pkg/kubectl/cmd/create_secret.go:135 +msgid "" +"\n" +"\t\t # If you don't already have a .dockercfg file, you can create a " +"dockercfg secret directly by using:\n" +"\t\t kubectl create secret docker-registry my-secret --docker-" +"server=DOCKER_REGISTRY_SERVER --docker-username=DOCKER_USER --docker-" +"password=DOCKER_PASSWORD --docker-email=DOCKER_EMAIL" +msgstr "" +"\n" +"\t\t # Se non si dispone ancora di un file .dockercfg, è possibile creare un " +"secret dockercfg direttamente utilizzando:\n" +"\t\t kubectl create secret docker-registry my-secret --docker-" +"server=DOCKER_REGISTRY_SERVER --docker-username=DOCKER_USER --docker-" +"password=DOCKER_PASSWORD --docker-email=DOCKER_EMAIL" + +#: pkg/kubectl/cmd/top_node.go:65 +msgid "" +"\n" +"\t\t # Show metrics for all nodes\n" +"\t\t kubectl top node\n" +"\n" +"\t\t # Show metrics for a given node\n" +"\t\t kubectl top node NODE_NAME" +msgstr "" +"\n" +"\t\t # Mostra metriche per tutti i nodi\n" +"\t\t kubectl top node\n" +"\n" +"\t\t # Mostra metriche per un determinato nodo\n" +"\t\t kubectl top node NODE_NAME" + +#: pkg/kubectl/cmd/apply.go:84 +msgid "" +"\n" +"\t\t# Apply the configuration in pod.json to a pod.\n" +"\t\tkubectl apply -f ./pod.json\n" +"\n" +"\t\t# Apply the JSON passed into stdin to a pod.\n" +"\t\tcat pod.json | kubectl apply -f -\n" +"\n" +"\t\t# Note: --prune is still in Alpha\n" +"\t\t# Apply the configuration in manifest.yaml that matches label app=nginx " +"and delete all the other resources that are not in the file and match label " +"app=nginx.\n" +"\t\tkubectl apply --prune -f manifest.yaml -l app=nginx\n" +"\n" +"\t\t# Apply the configuration in manifest.yaml and delete all the other " +"configmaps that are not in the file.\n" +"\t\tkubectl apply --prune -f manifest.yaml --all --prune-whitelist=core/v1/" +"ConfigMap" +msgstr "" +"\n" +"\t\t# Applica la configurazione pod.json a un pod.\n" +"\t\tkubectl apply -f ./pod.json\n" +"\n" +"\t\t# Applicare il JSON passato in stdin a un pod.\n" +"\t\tcat pod.json | kubectl apply -f -\n" +"\n" +"\t\t# Nota: --prune è ancora in in Alpha\n" +"\t\t# Applica la configurazione manifest.yaml che corrisponde alla label app " +"= nginx ed elimina tutte le altre risorse che non sono nel file e nella label " +"corrispondente app = nginx.\n" +"\t\tkubectl apply --prune -f manifest.yaml -l app=nginx\n" +"\n" +"\t\t# Applica la configurazione manifest.yaml ed elimina tutti gli altri " +"configmaps non presenti nel file.\n" +"\t\tkubectl apply --prune -f manifest.yaml --all --prune-whitelist=core/v1/" +"ConfigMap" + +#: pkg/kubectl/cmd/autoscale.go:40 +#, c-format +msgid "" +"\n" +"\t\t# Auto scale a deployment \"foo\", with the number of pods between 2 and " +"10, target CPU utilization specified so a default autoscaling policy will be " +"used:\n" +"\t\tkubectl autoscale deployment foo --min=2 --max=10\n" +"\n" +"\t\t# Auto scale a replication controller \"foo\", with the number of pods " +"between 1 and 5, target CPU utilization at 80%:\n" +"\t\tkubectl autoscale rc foo --max=5 --cpu-percent=80" +msgstr "" +"\n" +"\t\t# Auto scale un deployment \"foo\", con il numero di pod compresi tra 2 " +"e 10, utilizzo della CPU target specificato in modo da utilizzare una " +"politica di autoscaling predefinita:\n" +"\t\tkubectl autoscale deployment foo --min=2 --max=10\n" +"\n" +"\t\t# Auto scale un controller di replica \"foo\", con il numero di pod " +"compresi tra 1 e 5, utilizzo dell'utilizzo della CPU a 80%:\n" +"\t\tkubectl autoscale rc foo --max=5 --cpu-percent=80" + +#: pkg/kubectl/cmd/convert.go:49 +msgid "" +"\n" +"\t\t# Convert 'pod.yaml' to latest version and print to stdout.\n" +"\t\tkubectl convert -f pod.yaml\n" +"\n" +"\t\t# Convert the live state of the resource specified by 'pod.yaml' to the " +"latest version\n" +"\t\t# and print to stdout in json format.\n" +"\t\tkubectl convert -f pod.yaml --local -o json\n" +"\n" +"\t\t# Convert all files under current directory to latest version and create " +"them all.\n" +"\t\tkubectl convert -f . | kubectl create -f -" +msgstr "" +"\n" +"\t\t# Converte 'pod.yaml' alla versione più recente e stampa in stdout.\n" +"\t\tkubectl convert -f pod.yaml\n" +"\n" +"\t\t# Converte lo stato live della risorsa specificata da 'pod.yaml' nella " +"versione più recente.\n" +"\t\t# e stampa in stdout nel formato json.\n" +"\t\tkubectl convert -f pod.yaml --local -o json\n" +"\n" +"\t\t# Converte tutti i file nella directory corrente alla versione più " +"recente e li crea tutti.\n" +"\t\tkubectl convert -f . | kubectl create -f -" + +#: pkg/kubectl/cmd/create_clusterrole.go:34 +msgid "" +"\n" +"\t\t# Create a ClusterRole named \"pod-reader\" that allows user to perform " +"\"get\", \"watch\" and \"list\" on pods\n" +"\t\tkubectl create clusterrole pod-reader --verb=get,list,watch --" +"resource=pods\n" +"\n" +"\t\t# Create a ClusterRole named \"pod-reader\" with ResourceName specified\n" +"\t\tkubectl create clusterrole pod-reader --verb=get,list,watch --" +"resource=pods --resource-name=readablepod" +msgstr "" +"\n" +"\t\t# Crea un ClusterRole denominato \"pod-reader\" che consente all'utente " +"di eseguire \"get\", \"watch\" e \"list\" sui pod\n" +"\t\tkubectl create clusterrole pod-reader --verb=get,list,watch --" +"resource=pods\n" +"\n" +"\t\t# Crea un ClusterRole denominato \"pod-reader\" con ResourceName " +"specificato\n" +"\t\tkubectl create clusterrole pod-reader --verb=get,list,watch --" +"resource=pods --resource-name=readablepod" + +#: pkg/kubectl/cmd/create_role.go:41 +msgid "" +"\n" +"\t\t# Create a Role named \"pod-reader\" that allows user to perform \"get\", " +"\"watch\" and \"list\" on pods\n" +"\t\tkubectl create role pod-reader --verb=get --verb=list --verb=watch --" +"resource=pods\n" +"\n" +"\t\t# Create a Role named \"pod-reader\" with ResourceName specified\n" +"\t\tkubectl create role pod-reader --verb=get --verg=list --verb=watch --" +"resource=pods --resource-name=readablepod" +msgstr "" +"\n" +"\t\t# Crea un ruolo denominato \"pod-reader\" che consente all'utente di " +"eseguire \"get\", \"watch\" e \"list\" sui pod\n" +"\t\tkubectl create role pod-reader --verb=get --verb=list --verb=watch --" +"resource=pods\n" +"\n" +"\t\t# Crea un ruolo denominato \"pod-reader\" con ResourceName specificato\n" +"\t\tkubectl create role pod-reader --verb=get --verg=list --verb=watch --" +"resource=pods --resource-name=readablepod" + +#: pkg/kubectl/cmd/create_quota.go:35 +msgid "" +"\n" +"\t\t# Create a new resourcequota named my-quota\n" +"\t\tkubectl create quota my-quota --hard=cpu=1,memory=1G,pods=2,services=3," +"replicationcontrollers=2,resourcequotas=1,secrets=5," +"persistentvolumeclaims=10\n" +"\n" +"\t\t# Create a new resourcequota named best-effort\n" +"\t\tkubectl create quota best-effort --hard=pods=100 --scopes=BestEffort" +msgstr "" +"\n" +"\t\t# Crea una nuova resourcequota chiamata my-quota\n" +"\t\tkubectl create quota my-quota --hard=cpu=1,memory=1G,pods=2,services=3," +"replicationcontrollers=2,resourcequotas=1,secrets=5," +"persistentvolumeclaims=10\n" +"\n" +"\t\t# Creare una nuova resourcequota denominata best-effort\n" +"\t\tkubectl create quota best-effort --hard=pods=100 --scopes=BestEffort" + +#: pkg/kubectl/cmd/create_pdb.go:35 +#, c-format +msgid "" +"\n" +"\t\t# Create a pod disruption budget named my-pdb that will select all pods " +"with the app=rails label\n" +"\t\t# and require at least one of them being available at any point in time.\n" +"\t\tkubectl create poddisruptionbudget my-pdb --selector=app=rails --min-" +"available=1\n" +"\n" +"\t\t# Create a pod disruption budget named my-pdb that will select all pods " +"with the app=nginx label\n" +"\t\t# and require at least half of the pods selected to be available at any " +"point in time.\n" +"\t\tkubectl create pdb my-pdb --selector=app=nginx --min-available=50%" +msgstr "" +"\n" +"\t\t# Crea un pod disruption budget chiamato my-pdb che seleziona tutti i pod " +"con label app = rail\n" +"\t\t# e richiede che almeno uno di essi sia disponibile in qualsiasi " +"momento.\n" +"\t\tkubectl create poddisruptionbudget my-pdb --selector=app=rails --min-" +"available=1\n" +"\n" +"\t\t# Crea un pod disruption budget con nome my-pdb che seleziona tutti i pod " +"con label app = nginx \n" +"\t\t# e richiede che almeno la metà dei pod selezionati sia disponibile in " +"qualsiasi momento.\n" +"\t\tkubectl create pdb my-pdb --selector=app=nginx --min-available=50%" + +#: pkg/kubectl/cmd/create.go:47 +msgid "" +"\n" +"\t\t# Create a pod using the data in pod.json.\n" +"\t\tkubectl create -f ./pod.json\n" +"\n" +"\t\t# Create a pod based on the JSON passed into stdin.\n" +"\t\tcat pod.json | kubectl create -f -\n" +"\n" +"\t\t# Edit the data in docker-registry.yaml in JSON using the v1 API format " +"then create the resource using the edited data.\n" +"\t\tkubectl create -f docker-registry.yaml --edit --output-version=v1 -o json" +msgstr "" +"\n" +"\t\t# Crea un pod utilizzando i dati in pod.json.\n" +"\t\tkubectl create -f ./pod.json\n" +"\n" +"\t\t# Crea un pod basato sul JSON passato in stdin.\n" +"\t\tcat pod.json | kubectl create -f -\n" +"\n" +"\t\t# Modifica i dati in docker-registry.yaml in JSON utilizzando il formato " +"API v1 quindi creare la risorsa utilizzando i dati modificati.\n" +"\t\tkubectl create -f docker-registry.yaml --edit --output-version=v1 -o json" + +#: pkg/kubectl/cmd/expose.go:53 +msgid "" +"\n" +"\t\t# Create a service for a replicated nginx, which serves on port 80 and " +"connects to the containers on port 8000.\n" +"\t\tkubectl expose rc nginx --port=80 --target-port=8000\n" +"\n" +"\t\t# Create a service for a replication controller identified by type and " +"name specified in \"nginx-controller.yaml\", which serves on port 80 and " +"connects to the containers on port 8000.\n" +"\t\tkubectl expose -f nginx-controller.yaml --port=80 --target-port=8000\n" +"\n" +"\t\t# Create a service for a pod valid-pod, which serves on port 444 with the " +"name \"frontend\"\n" +"\t\tkubectl expose pod valid-pod --port=444 --name=frontend\n" +"\n" +"\t\t# Create a second service based on the above service, exposing the " +"container port 8443 as port 443 with the name \"nginx-https\"\n" +"\t\tkubectl expose service nginx --port=443 --target-port=8443 --name=nginx-" +"https\n" +"\n" +"\t\t# Create a service for a replicated streaming application on port 4100 " +"balancing UDP traffic and named 'video-stream'.\n" +"\t\tkubectl expose rc streamer --port=4100 --protocol=udp --name=video-" +"stream\n" +"\n" +"\t\t# Create a service for a replicated nginx using replica set, which serves " +"on port 80 and connects to the containers on port 8000.\n" +"\t\tkubectl expose rs nginx --port=80 --target-port=8000\n" +"\n" +"\t\t# Create a service for an nginx deployment, which serves on port 80 and " +"connects to the containers on port 8000.\n" +"\t\tkubectl expose deployment nginx --port=80 --target-port=8000" +msgstr "" +"\n" +"\t\t# Crea un servizio per un nginx replicato, che serve nella porta 80 e si " +"collega ai container sulla porta 8000.\n" +"\t\tkubectl expose rc nginx --port=80 --target-port=8000\n" +"\n" +"\t\t# Crea un servizio per un controller di replica identificato per tipo e " +"nome specificato in \"nginx-controller.yaml\", che serve nella porta 80 e si " +"collega ai container sulla porta 8000.\n" +"\t\tkubectl expose -f nginx-controller.yaml --port=80 --target-port=8000\n" +"\n" +"\t\t# Crea un servizio per un pod valid-pod, che serve nella porta 444 con il " +"nome \"frontend\"\n" +"\t\tkubectl expose pod valid-pod --port=444 --name=frontend\n" +"\n" +"\t\t# Crea un secondo servizio basato sul servizio sopra, esponendo la porta " +"container 8443 come porta 443 con il nome \"nginx-https\"\n" +"\t\tkubectl expose service nginx --port=443 --target-port=8443 --name=nginx-" +"https\n" +"\n" +"\t\t# Crea un servizio per un'applicazione di replica in porta 4100 che " +"bilanci il traffico UDP e denominato \"video stream\".\n" +"\t\tkubectl expose rc streamer --port=4100 --protocol=udp --name=video-" +"stream\n" +"\n" +"\t\t# Crea un servizio per un nginx replicato utilizzando l'insieme di " +"replica, che serve nella porta 80 e si collega ai contenitori sulla porta " +"8000.\n" +"\t\tkubectl expose rs nginx --port=80 --target-port=8000\n" +"\n" +"\t\t# Crea un servizio per una distribuzione di nginx, che serve nella porta " +"80 e si collega ai contenitori della porta 8000.\n" +"\t\tkubectl expose deployment nginx --port=80 --target-port=8000" + +#: pkg/kubectl/cmd/delete.go:68 +msgid "" +"\n" +"\t\t# Delete a pod using the type and name specified in pod.json.\n" +"\t\tkubectl delete -f ./pod.json\n" +"\n" +"\t\t# Delete a pod based on the type and name in the JSON passed into stdin.\n" +"\t\tcat pod.json | kubectl delete -f -\n" +"\n" +"\t\t# Delete pods and services with same names \"baz\" and \"foo\"\n" +"\t\tkubectl delete pod,service baz foo\n" +"\n" +"\t\t# Delete pods and services with label name=myLabel.\n" +"\t\tkubectl delete pods,services -l name=myLabel\n" +"\n" +"\t\t# Delete a pod with minimal delay\n" +"\t\tkubectl delete pod foo --now\n" +"\n" +"\t\t# Force delete a pod on a dead node\n" +"\t\tkubectl delete pod foo --grace-period=0 --force\n" +"\n" +"\t\t# Delete all pods\n" +"\t\tkubectl delete pods --all" +msgstr "" +"\n" +"\t\t# Elimina un pod utilizzando il tipo e il nome specificati in pod.json.\n" +"\t\tkubectl delete -f ./pod.json\n" +"\n" +"\t\t# Elimina un pod in base al tipo e al nome del JSON passato in stdin.\n" +"\t\tcat pod.json | kubectl delete -f -\n" +"\n" +"\t\t# Elimina i baccelli ei servizi con gli stessi nomi \"baz\" e \"foo\"\n" +"\t\tkubectl delete pod,service baz foo\n" +"\n" +"\t\t# Elimina i baccelli ei servizi con il nome dell'etichetta = myLabel.\n" +"\t\tkubectl delete pods,services -l name=myLabel\n" +"\n" +"\t\t# Eliminare un pod con un ritardo minimo\n" +"\t\tkubectl delete pod foo --now\n" +"\n" +"\t\t# Forza elimina un pod in un nodo morto\n" +"\t\tkubectl delete pod foo --grace-period=0 --force\n" +"\n" +"\t\t# Elimina tutti i pod\n" +"\t\tkubectl delete pods --all" + +#: pkg/kubectl/cmd/describe.go:54 +msgid "" +"\n" +"\t\t# Describe a node\n" +"\t\tkubectl describe nodes kubernetes-node-emt8.c.myproject.internal\n" +"\n" +"\t\t# Describe a pod\n" +"\t\tkubectl describe pods/nginx\n" +"\n" +"\t\t# Describe a pod identified by type and name in \"pod.json\"\n" +"\t\tkubectl describe -f pod.json\n" +"\n" +"\t\t# Describe all pods\n" +"\t\tkubectl describe pods\n" +"\n" +"\t\t# Describe pods by label name=myLabel\n" +"\t\tkubectl describe po -l name=myLabel\n" +"\n" +"\t\t# Describe all pods managed by the 'frontend' replication controller (rc-" +"created pods\n" +"\t\t# get the name of the rc as a prefix in the pod the name).\n" +"\t\tkubectl describe pods frontend" +msgstr "" +"\n" +"\t\t# Descrive un nodo\n" +"\t\tkubectl describe nodes kubernetes-node-emt8.c.myproject.internal\n" +"\n" +"\t\t# Descrive un pod\n" +"\t\tkubectl describe pods/nginx\n" +"\n" +"\t\t# Descrive un pod identificato da tipo e nome in \"pod.json\"\n" +"\t\tkubectl describe -f pod.json\n" +"\n" +"\t\t# Descrive tutti i pod\n" +"\t\tkubectl describe pods\n" +"\n" +"\t\t# Descrive i pod con label name=myLabel\n" +"\t\tkubectl describe po -l name=myLabel\n" +"\n" +"\t\t# Descrivere tutti i pod gestiti dal controller di replica \"frontend" +"\" (rc-created pods\n" +"\t\t# ottiene il nome del rc come un prefisso del nome pod).\n" +"\t\tkubectl describe pods frontend" + +#: pkg/kubectl/cmd/drain.go:165 +msgid "" +"\n" +"\t\t# Drain node \"foo\", even if there are pods not managed by a " +"ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet on it.\n" +"\t\t$ kubectl drain foo --force\n" +"\n" +"\t\t# As above, but abort if there are pods not managed by a " +"ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet, and use a " +"grace period of 15 minutes.\n" +"\t\t$ kubectl drain foo --grace-period=900" +msgstr "" +"\n" +"\t\t# Drain node \"foo\", anche se ci sono i baccelli non gestiti da " +"ReplicationController, ReplicaSet, Job, DaemonSet o StatefulSet su di esso.\n" +"\t\t$ kubectl drain foo --force\n" +"\n" +"\t\t# Come sopra, ma interrompere se ci sono i baccelli non gestiti da " +"ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet, e " +"utilizzare un periodo di grazia di 15 minuti.\n" +"\t\t$ kubectl drain foo --grace-period=900" + +#: pkg/kubectl/cmd/edit.go:80 +msgid "" +"\n" +"\t\t# Edit the service named 'docker-registry':\n" +"\t\tkubectl edit svc/docker-registry\n" +"\n" +"\t\t# Use an alternative editor\n" +"\t\tKUBE_EDITOR=\"nano\" kubectl edit svc/docker-registry\n" +"\n" +"\t\t# Edit the job 'myjob' in JSON using the v1 API format:\n" +"\t\tkubectl edit job.v1.batch/myjob -o json\n" +"\n" +"\t\t# Edit the deployment 'mydeployment' in YAML and save the modified config " +"in its annotation:\n" +"\t\tkubectl edit deployment/mydeployment -o yaml --save-config" +msgstr "" +"\n" +"\t\t# Modifica il servizio denominato 'docker-registry':\n" +"\t\tkubectl edit svc/docker-registry\n" +"\n" +"\t\t# Usa un editor alternativo\n" +"\t\tKUBE_EDITOR=\"nano\" kubectl edit svc/docker-registry\n" +"\n" +"\t\t# Modifica il lavoro 'myjob' in JSON utilizzando il formato API v1:\n" +"\t\tkubectl edit job.v1.batch/myjob -o json\n" +"\n" +"\t\t# Modifica la distribuzione 'mydeployment' in YAML e salvare la " +"configurazione modificata nella sua annotazione:\n" +"\t\tkubectl edit deployment/mydeployment -o yaml --save-config" + +#: pkg/kubectl/cmd/exec.go:41 +msgid "" +"\n" +"\t\t# Get output from running 'date' from pod 123456-7890, using the first " +"container by default\n" +"\t\tkubectl exec 123456-7890 date\n" +"\n" +"\t\t# Get output from running 'date' in ruby-container from pod 123456-7890\n" +"\t\tkubectl exec 123456-7890 -c ruby-container date\n" +"\n" +"\t\t# Switch to raw terminal mode, sends stdin to 'bash' in ruby-container " +"from pod 123456-7890\n" +"\t\t# and sends stdout/stderr from 'bash' back to the client\n" +"\t\tkubectl exec 123456-7890 -c ruby-container -i -t -- bash -il" +msgstr "" +"\n" +"\t\t# Ottieni l'output dalla 'data' di esecuzione del pod 123456-7890, " +"utilizzando il primo contenitore per impostazione predefinita\n" +"\t\tkubectl exec 123456-7890 date\n" +"\n" +"\t\t# Ottieni l'output dalla data di esecuzione in ruby-container del pod " +"123456-7890\n" +"\t\tkubectl exec 123456-7890 -c ruby-container date\n" +"\n" +"\t\t# Passare alla modalità raw terminal, invia stdin a 'bash' in ruby-" +"container del pod 123456-7890\n" +"\t\t# and sends stdout/stderr from 'bash' back to the client\n" +"\t\tkubectl exec 123456-7890 -c ruby-container -i -t -- bash -il" + +#: pkg/kubectl/cmd/attach.go:42 +msgid "" +"\n" +"\t\t# Get output from running pod 123456-7890, using the first container by " +"default\n" +"\t\tkubectl attach 123456-7890\n" +"\n" +"\t\t# Get output from ruby-container from pod 123456-7890\n" +"\t\tkubectl attach 123456-7890 -c ruby-container\n" +"\n" +"\t\t# Switch to raw terminal mode, sends stdin to 'bash' in ruby-container " +"from pod 123456-7890\n" +"\t\t# and sends stdout/stderr from 'bash' back to the client\n" +"\t\tkubectl attach 123456-7890 -c ruby-container -i -t\n" +"\n" +"\t\t# Get output from the first pod of a ReplicaSet named nginx\n" +"\t\tkubectl attach rs/nginx\n" +"\t\t" +msgstr "" +"\n" +"\t\t# Ottieni l'output dal pod 123456-7890 in esecuzione, utilizzando il " +"primo contenitore per impostazione predefinita\n" +"\t\tkubectl attach 123456-7890\n" +"\n" +"\t\t# Ottieni l'output dal ruby-container del pod 123456-7890\n" +"\t\tkubectl attach 123456-7890 -c ruby-container\n" +"\n" +"\t\t# Passa alla modalità raw terminal, invia stdin a 'bash' in ruby-" +"container del pod 123456-7890\n" +"\t\t# e invia stdout/stderr da 'bash' al client\n" +"\t\tkubectl attach 123456-7890 -c ruby-container -i -t\n" +"\n" +"\t\t# Ottieni l'output dal primo pod di una ReplicaSet denominata nginx\n" +"\t\tkubectl attach rs/nginx\n" +"\t\t" + +#: pkg/kubectl/cmd/explain.go:39 +msgid "" +"\n" +"\t\t# Get the documentation of the resource and its fields\n" +"\t\tkubectl explain pods\n" +"\n" +"\t\t# Get the documentation of a specific field of a resource\n" +"\t\tkubectl explain pods.spec.containers" +msgstr "" +"\n" +"\t\t# Ottieni la documentazione della risorsa e i relativi campi\n" +"\t\tkubectl explain pods\n" +"\n" +"\t\t# Ottieni la documentazione di un campo specifico di una risorsa\n" +"\t\tkubectl explain pods.spec.containers" + +#: pkg/kubectl/cmd/completion.go:65 +msgid "" +"\n" +"\t\t# Install bash completion on a Mac using homebrew\n" +"\t\tbrew install bash-completion\n" +"\t\tprintf \"\n" +"# Bash completion support\n" +"source $(brew --prefix)/etc/bash_completion\n" +"\" >> $HOME/.bash_profile\n" +"\t\tsource $HOME/.bash_profile\n" +"\n" +"\t\t# Load the kubectl completion code for bash into the current shell\n" +"\t\tsource <(kubectl completion bash)\n" +"\n" +"\t\t# Write bash completion code to a file and source if from .bash_profile\n" +"\t\tkubectl completion bash > ~/.kube/completion.bash.inc\n" +"\t\tprintf \"\n" +"# Kubectl shell completion\n" +"source '$HOME/.kube/completion.bash.inc'\n" +"\" >> $HOME/.bash_profile\n" +"\t\tsource $HOME/.bash_profile\n" +"\n" +"\t\t# Load the kubectl completion code for zsh[1] into the current shell\n" +"\t\tsource <(kubectl completion zsh)" +msgstr "" +"\n" +"\t\t# Installa il completamento di bash su un Mac utilizzando homebrew\n" +"\t\tbrew install bash-completion\n" +"\t\tprintf \"\n" +"# Bash completion support\n" +"source $(brew --prefix)/etc/bash_completion\n" +"\" >> $HOME/.bash_profile\n" +"\t\tsource $HOME/.bash_profile\n" +"\n" +"\t\t# Carica il codice di completamento kubectl per bash nella shell " +"corrente\n" +"\t\tsource <(kubectl completion bash)\n" +"\n" +"\t\t# Scrive il codice di completamento bash in un file e lo carica da ." +"bash_profile\n" +"\t\tkubectl completion bash > ~/.kube/completion.bash.inc\n" +"\t\tprintf \"\n" +"# Kubectl shell completion\n" +"source '$HOME/.kube/completion.bash.inc'\n" +"\" >> $HOME/.bash_profile\n" +"\t\tsource $HOME/.bash_profile\n" +"\n" +"\t\t# Carica il codice di completamento kubectl per zsh [1] nella shell " +"corrente\n" +"\t\tsource <(kubectl completion zsh)" + +#: pkg/kubectl/cmd/get.go:64 +msgid "" +"\n" +"\t\t# List all pods in ps output format.\n" +"\t\tkubectl get pods\n" +"\n" +"\t\t# List all pods in ps output format with more information (such as node " +"name).\n" +"\t\tkubectl get pods -o wide\n" +"\n" +"\t\t# List a single replication controller with specified NAME in ps output " +"format.\n" +"\t\tkubectl get replicationcontroller web\n" +"\n" +"\t\t# List a single pod in JSON output format.\n" +"\t\tkubectl get -o json pod web-pod-13je7\n" +"\n" +"\t\t# List a pod identified by type and name specified in \"pod.yaml\" in " +"JSON output format.\n" +"\t\tkubectl get -f pod.yaml -o json\n" +"\n" +"\t\t# Return only the phase value of the specified pod.\n" +"\t\tkubectl get -o template pod/web-pod-13je7 --template={{.status.phase}}\n" +"\n" +"\t\t# List all replication controllers and services together in ps output " +"format.\n" +"\t\tkubectl get rc,services\n" +"\n" +"\t\t# List one or more resources by their type and names.\n" +"\t\tkubectl get rc/web service/frontend pods/web-pod-13je7\n" +"\n" +"\t\t# List all resources with different types.\n" +"\t\tkubectl get all" +msgstr "" +"\n" +"\t\t# Elenca tutti i pod in formato output ps.\n" +"\t\tkubectl get pods\n" +"\n" +"\t\t# Elenca tutti i pod in formato output ps con maggiori informazioni (ad " +"esempio il nome del nodo).\n" +"\t\tkubectl get pods -o wide\n" +"\n" +"\t\t# Elenca un controller di replica singolo con NAME specificato nel " +"formato di output ps.\n" +"\t\tkubectl get replicationcontroller web\n" +"\n" +"\t\t# Elenca un singolo pod nel formato di uscita JSON.\n" +"\t\tkubectl get -o json pod web-pod-13je7\n" +"\n" +"\t\t# Elenca un pod identificato per tipo e nome specificato in \"pod.yaml\" " +"nel formato di uscita JSON.\n" +"\t\tkubectl get -f pod.yaml -o json\n" +"\n" +"\t\t# Restituisce solo il valore di fase del pod specificato.\n" +"\t\tkubectl get -o template pod/web-pod-13je7 --template={{.status.phase}}\n" +"\n" +"\t\t# Elenca tutti i controller e servizi di replica insieme in formato " +"output ps.\n" +"\t\tkubectl get rc,services\n" +"\n" +"\t\t# Elenca una o più risorse per il tipo e per i nomi.\n" +"\t\tkubectl get rc/web service/frontend pods/web-pod-13je7\n" +"\n" +"\t\t# Elenca tutte le risorse con tipi diversi.\n" +"\t\tkubectl get all" + +#: pkg/kubectl/cmd/portforward.go:53 +msgid "" +"\n" +"\t\t# Listen on ports 5000 and 6000 locally, forwarding data to/from ports " +"5000 and 6000 in the pod\n" +"\t\tkubectl port-forward mypod 5000 6000\n" +"\n" +"\t\t# Listen on port 8888 locally, forwarding to 5000 in the pod\n" +"\t\tkubectl port-forward mypod 8888:5000\n" +"\n" +"\t\t# Listen on a random port locally, forwarding to 5000 in the pod\n" +"\t\tkubectl port-forward mypod :5000\n" +"\n" +"\t\t# Listen on a random port locally, forwarding to 5000 in the pod\n" +"\t\tkubectl port-forward mypod 0:5000" +msgstr "" +"\n" +"\t\t# Ascolta localmente le porte 5000 e 6000, inoltrando i dati da/verso le " +"porte 5000 e 6000 nel pod\n" +"\t\tkubectl port-forward mypod 5000 6000\n" +"\n" +"\t\t# Ascolta localmente la porta 8888, inoltra a 5000 nel pod\n" +"\t\tkubectl port-forward mypod 8888:5000\n" +"\n" +"\t\t# Ascolta localmente una porta casuale, inoltra a 5000 nel pod\n" +"\t\tkubectl port-forward mypod :5000\n" +"\n" +"\t\t# Ascolta localmente una porta casuale, inoltra a 5000 nel pod\n" +"\t\tkubectl port-forward mypod 0:5000" + +#: pkg/kubectl/cmd/drain.go:118 +msgid "" +"\n" +"\t\t# Mark node \"foo\" as schedulable.\n" +"\t\t$ kubectl uncordon foo" +msgstr "" +"\n" +"\t\t# Segna il nodo \"foo\" come programmabile.\n" +"\t\t$ Kubectl uncordon foo" + +#: pkg/kubectl/cmd/drain.go:93 +msgid "" +"\n" +"\t\t# Mark node \"foo\" as unschedulable.\n" +"\t\tkubectl cordon foo" +msgstr "" +"\n" +"\t\t# Segna il nodo \"foo\" come non programmabile.\n" +"\t\tkubectl cordon foo" + +#: pkg/kubectl/cmd/patch.go:66 +msgid "" +"\n" +"\t\t# Partially update a node using strategic merge patch\n" +"\t\tkubectl patch node k8s-node-1 -p '{\"spec\":{\"unschedulable\":true}}'\n" +"\n" +"\t\t# Partially update a node identified by the type and name specified in " +"\"node.json\" using strategic merge patch\n" +"\t\tkubectl patch -f node.json -p '{\"spec\":{\"unschedulable\":true}}'\n" +"\n" +"\t\t# Update a container's image; spec.containers[*].name is required because " +"it's a merge key\n" +"\t\tkubectl patch pod valid-pod -p '{\"spec\":{\"containers\":[{\"name\":" +"\"kubernetes-serve-hostname\",\"image\":\"new image\"}]}}'\n" +"\n" +"\t\t# Update a container's image using a json patch with positional arrays\n" +"\t\tkubectl patch pod valid-pod --type='json' -p='[{\"op\": \"replace\", " +"\"path\": \"/spec/containers/0/image\", \"value\":\"new image\"}]'" +msgstr "" +"\n" +"\t\t# Aggiorna parzialmente un nodo utilizzando merge patch strategica\n" +"\t\tkubectl patch node k8s-node-1 -p '{\"spec\":{\"unschedulable\":true}}'\n" +"\n" +"\t\t# Aggiorna parzialmente un nodo identificato dal tipo e dal nome " +"specificato in \"node.json\" utilizzando merge patch strategica\n" +"\t\tkubectl patch -f node.json -p '{\"spec\":{\"unschedulable\":true}}'\n" +"\n" +"\t\t# Aggiorna l'immagine di un contenitore; spec.containers [*]. name è " +"richiesto perché è una chiave di fusione\n" +"\t\tkubectl patch pod valid-pod -p '{\"spec\":{\"containers\":[{\"name\":" +"\"kubernetes-serve-hostname\",\"image\":\"new image\"}]}}'\n" +"\n" +"\t\t# Aggiorna l'immagine di un contenitore utilizzando una patch json con " +"array posizionali\n" +"\t\tkubectl patch pod valid-pod --type='json' -p='[{\"op\": \"replace\", " +"\"path\": \"/spec/containers/0/image\", \"value\":\"new image\"}]'" + +#: pkg/kubectl/cmd/options.go:29 +msgid "" +"\n" +"\t\t# Print flags inherited by all commands\n" +"\t\tkubectl options" +msgstr "" +"\n" +"\t\t# Stampa i flag ereditati da tutti i comandi\n" +"\t\tkubectl options" + +#: pkg/kubectl/cmd/clusterinfo.go:41 +msgid "" +"\n" +"\t\t# Print the address of the master and cluster services\n" +"\t\tkubectl cluster-info" +msgstr "" +"\n" +"\t\t# Stampa l'indirizzo dei servizi master e cluster\n" +"\t\tkubectl cluster-info" + +#: pkg/kubectl/cmd/version.go:32 +msgid "" +"\n" +"\t\t# Print the client and server versions for the current context\n" +"\t\tkubectl version" +msgstr "" +"\n" +"\t\t# Stampa le versioni client e server per il current context\n" +"\t\tkubectl version" + +#: pkg/kubectl/cmd/apiversions.go:34 +msgid "" +"\n" +"\t\t# Print the supported API versions\n" +"\t\tkubectl api-versions" +msgstr "" +"\n" +"\t\t# Stampa le versioni API supportate\n" +"\t\tkubectl api-versions" + +#: pkg/kubectl/cmd/replace.go:50 +msgid "" +"\n" +"\t\t# Replace a pod using the data in pod.json.\n" +"\t\tkubectl replace -f ./pod.json\n" +"\n" +"\t\t# Replace a pod based on the JSON passed into stdin.\n" +"\t\tcat pod.json | kubectl replace -f -\n" +"\n" +"\t\t# Update a single-container pod's image version (tag) to v4\n" +"\t\tkubectl get pod mypod -o yaml | sed 's/\\(image: myimage\\):.*$/:v4/' | " +"kubectl replace -f -\n" +"\n" +"\t\t# Force replace, delete and then re-create the resource\n" +"\t\tkubectl replace --force -f ./pod.json" +msgstr "" + +#: pkg/kubectl/cmd/logs.go:40 +msgid "" +"\n" +"\t\t# Return snapshot logs from pod nginx with only one container\n" +"\t\tkubectl logs nginx\n" +"\n" +"\t\t# Return snapshot logs for the pods defined by label app=nginx\n" +"\t\tkubectl logs -lapp=nginx\n" +"\n" +"\t\t# Return snapshot of previous terminated ruby container logs from pod " +"web-1\n" +"\t\tkubectl logs -p -c ruby web-1\n" +"\n" +"\t\t# Begin streaming the logs of the ruby container in pod web-1\n" +"\t\tkubectl logs -f -c ruby web-1\n" +"\n" +"\t\t# Display only the most recent 20 lines of output in pod nginx\n" +"\t\tkubectl logs --tail=20 nginx\n" +"\n" +"\t\t# Show all logs from pod nginx written in the last hour\n" +"\t\tkubectl logs --since=1h nginx\n" +"\n" +"\t\t# Return snapshot logs from first container of a job named hello\n" +"\t\tkubectl logs job/hello\n" +"\n" +"\t\t# Return snapshot logs from container nginx-1 of a deployment named " +"nginx\n" +"\t\tkubectl logs deployment/nginx -c nginx-1" +msgstr "" + +#: pkg/kubectl/cmd/proxy.go:53 +msgid "" +"\n" +"\t\t# Run a proxy to kubernetes apiserver on port 8011, serving static " +"content from ./local/www/\n" +"\t\tkubectl proxy --port=8011 --www=./local/www/\n" +"\n" +"\t\t# Run a proxy to kubernetes apiserver on an arbitrary local port.\n" +"\t\t# The chosen port for the server will be output to stdout.\n" +"\t\tkubectl proxy --port=0\n" +"\n" +"\t\t# Run a proxy to kubernetes apiserver, changing the api prefix to k8s-" +"api\n" +"\t\t# This makes e.g. the pods api available at localhost:8001/k8s-api/v1/" +"pods/\n" +"\t\tkubectl proxy --api-prefix=/k8s-api" +msgstr "" + +#: pkg/kubectl/cmd/scale.go:43 +msgid "" +"\n" +"\t\t# Scale a replicaset named 'foo' to 3.\n" +"\t\tkubectl scale --replicas=3 rs/foo\n" +"\n" +"\t\t# Scale a resource identified by type and name specified in \"foo.yaml\" " +"to 3.\n" +"\t\tkubectl scale --replicas=3 -f foo.yaml\n" +"\n" +"\t\t# If the deployment named mysql's current size is 2, scale mysql to 3.\n" +"\t\tkubectl scale --current-replicas=2 --replicas=3 deployment/mysql\n" +"\n" +"\t\t# Scale multiple replication controllers.\n" +"\t\tkubectl scale --replicas=5 rc/foo rc/bar rc/baz\n" +"\n" +"\t\t# Scale job named 'cron' to 3.\n" +"\t\tkubectl scale --replicas=3 job/cron" +msgstr "" +"\n" +"\t\t# Scala un replicaset denominato 'foo' a 3.\n" +"\t\tkubectl scale --replicas=3 rs/foo\n" +"\n" +"\t\t# Scala una risorsa identificata per tipo e nome specificato in \"foo.yaml" +"\" a 3.\n" +"\t\tkubectl scale --replicas=3 -f foo.yaml\n" +"\n" +"\t\t# Se la distribuzione corrente di mysql è 2, scala mysql a 3.\n" +"\t\tkubectl scale --current-replicas=2 --replicas=3 deployment/mysql\n" +"\n" +"\t\t# Scalare più controllori di replica.\n" +"\t\tkubectl scale --replicas=5 rc/foo rc/bar rc/baz\n" +"\n" +"\t\t# Scala il lavoro denominato 'cron' a 3.\n" +"\t\tkubectl scale --replicas=3 job/cron" + +#: pkg/kubectl/cmd/apply_set_last_applied.go:67 +msgid "" +"\n" +"\t\t# Set the last-applied-configuration of a resource to match the contents " +"of a file.\n" +"\t\tkubectl apply set-last-applied -f deploy.yaml\n" +"\n" +"\t\t# Execute set-last-applied against each configuration file in a " +"directory.\n" +"\t\tkubectl apply set-last-applied -f path/\n" +"\n" +"\t\t# Set the last-applied-configuration of a resource to match the contents " +"of a file, will create the annotation if it does not already exist.\n" +"\t\tkubectl apply set-last-applied -f deploy.yaml --create-annotation=true\n" +"\t\t" +msgstr "" +"\n" +"\t\t# Imposta l'ultima-configurazione-applicata di una risorsa che " +"corrisponda al contenuto di un file.\n" +"\t\tkubectl apply set-last-applied -f deploy.yaml\n" +"\n" +"\t\t# Esegue set-last-applied per ogni file di configurazione in una " +"directory.\n" +"\t\tkubectl apply set-last-applied -f path/\n" +"\n" +"\t\t# Imposta la configurazione dell'ultima applicazione di una risorsa che " +"corrisponda al contenuto di un file, creerà l'annotazione se non esiste già.\n" +"\t\tkubectl apply set-last-applied -f deploy.yaml --create-annotation=true\n" +"\t\t" + +#: pkg/kubectl/cmd/top_pod.go:61 +msgid "" +"\n" +"\t\t# Show metrics for all pods in the default namespace\n" +"\t\tkubectl top pod\n" +"\n" +"\t\t# Show metrics for all pods in the given namespace\n" +"\t\tkubectl top pod --namespace=NAMESPACE\n" +"\n" +"\t\t# Show metrics for a given pod and its containers\n" +"\t\tkubectl top pod POD_NAME --containers\n" +"\n" +"\t\t# Show metrics for the pods defined by label name=myLabel\n" +"\t\tkubectl top pod -l name=myLabel" +msgstr "" + +#: pkg/kubectl/cmd/stop.go:40 +msgid "" +"\n" +"\t\t# Shut down foo.\n" +"\t\tkubectl stop replicationcontroller foo\n" +"\n" +"\t\t# Stop pods and services with label name=myLabel.\n" +"\t\tkubectl stop pods,services -l name=myLabel\n" +"\n" +"\t\t# Shut down the service defined in service.json\n" +"\t\tkubectl stop -f service.json\n" +"\n" +"\t\t# Shut down all resources in the path/to/resources directory\n" +"\t\tkubectl stop -f path/to/resources" +msgstr "" +"\n" +"\t\t# Spegni foo.\n" +"\t\tkubectl stop replicationcontroller foo\n" +"\n" +"\t\t# Stop di tutti i pod e servizi con label name=myLabel.\n" +"\t\tkubectl stop pods,services -l name=myLabel\n" +"\n" +"\t\t# Spegnere il servizio definito in service.json\n" +"\t\tkubectl stop -f service.json\n" +"\n" +"\t\t# Spegnere tutte le resources in path/to/resources directory\n" +"\t\tkubectl stop -f path/to/resources" + +#: pkg/kubectl/cmd/run.go:57 +msgid "" +"\n" +"\t\t# Start a single instance of nginx.\n" +"\t\tkubectl run nginx --image=nginx\n" +"\n" +"\t\t# Start a single instance of hazelcast and let the container expose port " +"5701 .\n" +"\t\tkubectl run hazelcast --image=hazelcast --port=5701\n" +"\n" +"\t\t# Start a single instance of hazelcast and set environment variables " +"\"DNS_DOMAIN=cluster\" and \"POD_NAMESPACE=default\" in the container.\n" +"\t\tkubectl run hazelcast --image=hazelcast --env=\"DNS_DOMAIN=cluster\" --" +"env=\"POD_NAMESPACE=default\"\n" +"\n" +"\t\t# Start a replicated instance of nginx.\n" +"\t\tkubectl run nginx --image=nginx --replicas=5\n" +"\n" +"\t\t# Dry run. Print the corresponding API objects without creating them.\n" +"\t\tkubectl run nginx --image=nginx --dry-run\n" +"\n" +"\t\t# Start a single instance of nginx, but overload the spec of the " +"deployment with a partial set of values parsed from JSON.\n" +"\t\tkubectl run nginx --image=nginx --overrides='{ \"apiVersion\": \"v1\", " +"\"spec\": { ... } }'\n" +"\n" +"\t\t# Start a pod of busybox and keep it in the foreground, don't restart it " +"if it exits.\n" +"\t\tkubectl run -i -t busybox --image=busybox --restart=Never\n" +"\n" +"\t\t# Start the nginx container using the default command, but use custom " +"arguments (arg1 .. argN) for that command.\n" +"\t\tkubectl run nginx --image=nginx -- ... \n" +"\n" +"\t\t# Start the nginx container using a different command and custom " +"arguments.\n" +"\t\tkubectl run nginx --image=nginx --command -- ... \n" +"\n" +"\t\t# Start the perl container to compute π to 2000 places and print it out.\n" +"\t\tkubectl run pi --image=perl --restart=OnFailure -- perl -Mbignum=bpi -wle " +"'print bpi(2000)'\n" +"\n" +"\t\t# Start the cron job to compute π to 2000 places and print it out every 5 " +"minutes.\n" +"\t\tkubectl run pi --schedule=\"0/5 * * * ?\" --image=perl --" +"restart=OnFailure -- perl -Mbignum=bpi -wle 'print bpi(2000)'" +msgstr "" + +#: pkg/kubectl/cmd/taint.go:67 +msgid "" +"\n" +"\t\t# Update node 'foo' with a taint with key 'dedicated' and value 'special-" +"user' and effect 'NoSchedule'.\n" +"\t\t# If a taint with that key and effect already exists, its value is " +"replaced as specified.\n" +"\t\tkubectl taint nodes foo dedicated=special-user:NoSchedule\n" +"\n" +"\t\t# Remove from node 'foo' the taint with key 'dedicated' and effect " +"'NoSchedule' if one exists.\n" +"\t\tkubectl taint nodes foo dedicated:NoSchedule-\n" +"\n" +"\t\t# Remove from node 'foo' all the taints with key 'dedicated'\n" +"\t\tkubectl taint nodes foo dedicated-" +msgstr "" + +#: pkg/kubectl/cmd/label.go:77 +msgid "" +"\n" +"\t\t# Update pod 'foo' with the label 'unhealthy' and the value 'true'.\n" +"\t\tkubectl label pods foo unhealthy=true\n" +"\n" +"\t\t# Update pod 'foo' with the label 'status' and the value 'unhealthy', " +"overwriting any existing value.\n" +"\t\tkubectl label --overwrite pods foo status=unhealthy\n" +"\n" +"\t\t# Update all pods in the namespace\n" +"\t\tkubectl label pods --all status=unhealthy\n" +"\n" +"\t\t# Update a pod identified by the type and name in \"pod.json\"\n" +"\t\tkubectl label -f pod.json status=unhealthy\n" +"\n" +"\t\t# Update pod 'foo' only if the resource is unchanged from version 1.\n" +"\t\tkubectl label pods foo status=unhealthy --resource-version=1\n" +"\n" +"\t\t# Update pod 'foo' by removing a label named 'bar' if it exists.\n" +"\t\t# Does not require the --overwrite flag.\n" +"\t\tkubectl label pods foo bar-" +msgstr "" + +#: pkg/kubectl/cmd/rollingupdate.go:54 +msgid "" +"\n" +"\t\t# Update pods of frontend-v1 using new replication controller data in " +"frontend-v2.json.\n" +"\t\tkubectl rolling-update frontend-v1 -f frontend-v2.json\n" +"\n" +"\t\t# Update pods of frontend-v1 using JSON data passed into stdin.\n" +"\t\tcat frontend-v2.json | kubectl rolling-update frontend-v1 -f -\n" +"\n" +"\t\t# Update the pods of frontend-v1 to frontend-v2 by just changing the " +"image, and switching the\n" +"\t\t# name of the replication controller.\n" +"\t\tkubectl rolling-update frontend-v1 frontend-v2 --image=image:v2\n" +"\n" +"\t\t# Update the pods of frontend by just changing the image, and keeping the " +"old name.\n" +"\t\tkubectl rolling-update frontend --image=image:v2\n" +"\n" +"\t\t# Abort and reverse an existing rollout in progress (from frontend-v1 to " +"frontend-v2).\n" +"\t\tkubectl rolling-update frontend-v1 frontend-v2 --rollback" +msgstr "" + +#: pkg/kubectl/cmd/apply_view_last_applied.go:52 +msgid "" +"\n" +"\t\t# View the last-applied-configuration annotations by type/name in YAML.\n" +"\t\tkubectl apply view-last-applied deployment/nginx\n" +"\n" +"\t\t# View the last-applied-configuration annotations by file in JSON\n" +"\t\tkubectl apply view-last-applied -f deploy.yaml -o json" +msgstr "" +"\n" +"\t\t# Visualizza le annotazioni dell'ultima-configurazione-applicata per tipo/" +"nome in YAML.\n" +"\t\tkubectl apply view-last-applied deployment/nginx\n" +"\n" +"\t\t# # Visualizza le annotazioni dell'ultima-configurazione-applicata per " +"file in JSON.\n" +"\t\tkubectl apply view-last-applied -f deploy.yaml -o json" + +#: pkg/kubectl/cmd/apply.go:75 +msgid "" +"\n" +"\t\tApply a configuration to a resource by filename or stdin.\n" +"\t\tThis resource will be created if it doesn't exist yet.\n" +"\t\tTo use 'apply', always create the resource initially with either 'apply' " +"or 'create --save-config'.\n" +"\n" +"\t\tJSON and YAML formats are accepted.\n" +"\n" +"\t\tAlpha Disclaimer: the --prune functionality is not yet complete. Do not " +"use unless you are aware of what the current state is. See https://issues.k8s." +"io/34274." +msgstr "" +"\n" +"\t\tApplicare una configurazione a una risorsa per nomefile o stdin.\n" +"\t\tQuesta risorsa verrà creata se non esiste ancora.\n" +"\t\tPer utilizzare 'apply', creare sempre la risorsa inizialmente con 'apply' " +"o 'create --save-config'.\n" +"\n" +"\t\tSono accettati i formati JSON e YAML.\n" +"\n" +"\t\tDisclaimer Alpha: la funzionalità --prune non è ancora completa. Non " +"utilizzare a meno che non si sia a conoscenza di quale sia lo stato attuale. " +"Vedi https://issues.k8s.io/34274." + +#: pkg/kubectl/cmd/convert.go:38 +msgid "" +"\n" +"\t\tConvert config files between different API versions. Both YAML\n" +"\t\tand JSON formats are accepted.\n" +"\n" +"\t\tThe command takes filename, directory, or URL as input, and convert it " +"into format\n" +"\t\tof version specified by --output-version flag. If target version is not " +"specified or\n" +"\t\tnot supported, convert to latest version.\n" +"\n" +"\t\tThe default output will be printed to stdout in YAML format. One can use -" +"o option\n" +"\t\tto change to output destination." +msgstr "" + +#: pkg/kubectl/cmd/create_clusterrole.go:31 +msgid "" +"\n" +"\t\tCreate a ClusterRole." +msgstr "" +"\n" +"\t\n" +"Crea un ClusterRole." + +#: pkg/kubectl/cmd/create_clusterrolebinding.go:32 +msgid "" +"\n" +"\t\tCreate a ClusterRoleBinding for a particular ClusterRole." +msgstr "" +"\n" +"\t\tCrea un ClusterRoleBinding per un ClusterRole particolare." + +#: pkg/kubectl/cmd/create_rolebinding.go:32 +msgid "" +"\n" +"\t\tCreate a RoleBinding for a particular Role or ClusterRole." +msgstr "" +"\n" +"\t\tCrea un RoleBinding per un particolare Ruolo o ClusterRole." + +#: pkg/kubectl/cmd/create_secret.go:200 +msgid "" +"\n" +"\t\tCreate a TLS secret from the given public/private key pair.\n" +"\n" +"\t\tThe public/private key pair must exist before hand. The public key " +"certificate must be .PEM encoded and match the given private key." +msgstr "" +"\n" +"\t\tCrea un TLS secret dalla coppia di chiavi pubblica/privata.\n" +"\n" +"\t\tLa coppia di chiavi pubblica/privata deve esistere prima. Il certificato " +"chiave pubblica deve essere .PEM codificato e corrispondere alla chiave " +"privata data." + +#: pkg/kubectl/cmd/create_configmap.go:32 +msgid "" +"\n" +"\t\tCreate a configmap based on a file, directory, or specified literal " +"value.\n" +"\n" +"\t\tA single configmap may package one or more key/value pairs.\n" +"\n" +"\t\tWhen creating a configmap based on a file, the key will default to the " +"basename of the file, and the value will\n" +"\t\tdefault to the file content. If the basename is an invalid key, you may " +"specify an alternate key.\n" +"\n" +"\t\tWhen creating a configmap based on a directory, each file whose basename " +"is a valid key in the directory will be\n" +"\t\tpackaged into the configmap. Any directory entries except regular files " +"are ignored (e.g. subdirectories,\n" +"\t\tsymlinks, devices, pipes, etc)." +msgstr "" + +#: pkg/kubectl/cmd/create_namespace.go:32 +msgid "" +"\n" +"\t\tCreate a namespace with the specified name." +msgstr "" +"\n" +"\t\tCreare un namespace con il nome specificato." + +#: pkg/kubectl/cmd/create_secret.go:119 +msgid "" +"\n" +"\t\tCreate a new secret for use with Docker registries.\n" +"\n" +"\t\tDockercfg secrets are used to authenticate against Docker registries.\n" +"\n" +"\t\tWhen using the Docker command line to push images, you can authenticate " +"to a given registry by running\n" +"\n" +"\t\t $ docker login DOCKER_REGISTRY_SERVER --username=DOCKER_USER --" +"password=DOCKER_PASSWORD --email=DOCKER_EMAIL'.\n" +"\n" +" That produces a ~/.dockercfg file that is used by subsequent 'docker " +"push' and 'docker pull' commands to\n" +"\t\tauthenticate to the registry. The email address is optional.\n" +"\n" +"\t\tWhen creating applications, you may have a Docker registry that requires " +"authentication. In order for the\n" +"\t\tnodes to pull images on your behalf, they have to have the credentials. " +"You can provide this information\n" +"\t\tby creating a dockercfg secret and attaching it to your service account." +msgstr "" + +#: pkg/kubectl/cmd/create_pdb.go:32 +msgid "" +"\n" +"\t\tCreate a pod disruption budget with the specified name, selector, and " +"desired minimum available pods" +msgstr "" +"\n" +"\t\tCrea un pod disruption budget con il nome specificato, selector e il " +"numero minimo di pod disponibili" + +#: pkg/kubectl/cmd/create.go:42 +msgid "" +"\n" +"\t\tCreate a resource by filename or stdin.\n" +"\n" +"\t\tJSON and YAML formats are accepted." +msgstr "" +"\n" +"\t\tCrea una risorsa per nome file o stdin.\n" +"\n" +"\t\tSono accettati i formati JSON e YAML." + +#: pkg/kubectl/cmd/create_quota.go:32 +msgid "" +"\n" +"\t\tCreate a resourcequota with the specified name, hard limits and optional " +"scopes" +msgstr "" +"\n" +"\t\tCrea una resourcequota con il nome specificato, hard limits e gli scope " +"opzionali" + +#: pkg/kubectl/cmd/create_role.go:38 +msgid "" +"\n" +"\t\tCreate a role with single rule." +msgstr "" +"\n" +"\t\tCrea un ruolo con una singola regola." + +#: pkg/kubectl/cmd/create_secret.go:47 +msgid "" +"\n" +"\t\tCreate a secret based on a file, directory, or specified literal value.\n" +"\n" +"\t\tA single secret may package one or more key/value pairs.\n" +"\n" +"\t\tWhen creating a secret based on a file, the key will default to the " +"basename of the file, and the value will\n" +"\t\tdefault to the file content. If the basename is an invalid key, you may " +"specify an alternate key.\n" +"\n" +"\t\tWhen creating a secret based on a directory, each file whose basename is " +"a valid key in the directory will be\n" +"\t\tpackaged into the secret. Any directory entries except regular files are " +"ignored (e.g. subdirectories,\n" +"\t\tsymlinks, devices, pipes, etc)." +msgstr "" + +#: pkg/kubectl/cmd/create_serviceaccount.go:32 +msgid "" +"\n" +"\t\tCreate a service account with the specified name." +msgstr "" +"\n" +"\t\tCreare un service account con il nome specificato." + +#: pkg/kubectl/cmd/run.go:52 +msgid "" +"\n" +"\t\tCreate and run a particular image, possibly replicated.\n" +"\n" +"\t\tCreates a deployment or job to manage the created container(s)." +msgstr "" +"\n" +"\t\tCrea ed esegue un'immagine particolare, eventualmente replicata.\n" +"\n" +"\t\tCrea un deployment o un job per gestire i container creati." + +#: pkg/kubectl/cmd/autoscale.go:34 +msgid "" +"\n" +"\t\tCreates an autoscaler that automatically chooses and sets the number of " +"pods that run in a kubernetes cluster.\n" +"\n" +"\t\tLooks up a Deployment, ReplicaSet, or ReplicationController by name and " +"creates an autoscaler that uses the given resource as a reference.\n" +"\t\tAn autoscaler can automatically increase or decrease number of pods " +"deployed within the system as needed." +msgstr "" +"\n" +"\t\tCrea un autoscaler che automaticamente sceglie e imposta il numero di pod " +"che vengono eseguiti in un cluster di kubernets.\n" +"\n" +"\t\tEsegue una ricerca di un Deployment, ReplicaSet o ReplicationController " +"per nome e crea un autoscaler che utilizza la risorsa indicata come " +"riferimento.\n" +"\t\tUn autoscaler può aumentare o diminuire automaticamente il numero di pod " +"distribuiti all'interno del sistema se necessario." + +#: pkg/kubectl/cmd/delete.go:40 +msgid "" +"\n" +"\t\tDelete resources by filenames, stdin, resources and names, or by " +"resources and label selector.\n" +"\n" +"\t\tJSON and YAML formats are accepted. Only one type of the arguments may be " +"specified: filenames,\n" +"\t\tresources and names, or resources and label selector.\n" +"\n" +"\t\tSome resources, such as pods, support graceful deletion. These resources " +"define a default period\n" +"\t\tbefore they are forcibly terminated (the grace period) but you may " +"override that value with\n" +"\t\tthe --grace-period flag, or pass --now to set a grace-period of 1. " +"Because these resources often\n" +"\t\trepresent entities in the cluster, deletion may not be acknowledged " +"immediately. If the node\n" +"\t\thosting a pod is down or cannot reach the API server, termination may " +"take significantly longer\n" +"\t\tthan the grace period. To force delete a resource,\tyou must pass a grace" +"\tperiod of 0 and specify\n" +"\t\tthe --force flag.\n" +"\n" +"\t\tIMPORTANT: Force deleting pods does not wait for confirmation that the " +"pod's processes have been\n" +"\t\tterminated, which can leave those processes running until the node " +"detects the deletion and\n" +"\t\tcompletes graceful deletion. If your processes use shared storage or talk " +"to a remote API and\n" +"\t\tdepend on the name of the pod to identify themselves, force deleting " +"those pods may result in\n" +"\t\tmultiple processes running on different machines using the same " +"identification which may lead\n" +"\t\tto data corruption or inconsistency. Only force delete pods when you are " +"sure the pod is\n" +"\t\tterminated, or if your application can tolerate multiple copies of the " +"same pod running at once.\n" +"\t\tAlso, if you force delete pods the scheduler may place new pods on those " +"nodes before the node\n" +"\t\thas released those resources and causing those pods to be evicted " +"immediately.\n" +"\n" +"\t\tNote that the delete command does NOT do resource version checks, so if " +"someone\n" +"\t\tsubmits an update to a resource right when you submit a delete, their " +"update\n" +"\t\twill be lost along with the rest of the resource." +msgstr "" + +#: pkg/kubectl/cmd/stop.go:31 +msgid "" +"\n" +"\t\tDeprecated: Gracefully shut down a resource by name or filename.\n" +"\n" +"\t\tThe stop command is deprecated, all its functionalities are covered by " +"delete command.\n" +"\t\tSee 'kubectl delete --help' for more details.\n" +"\n" +"\t\tAttempts to shut down and delete a resource that supports graceful " +"termination.\n" +"\t\tIf the resource is scalable it will be scaled to 0 before deletion." +msgstr "" + +#: pkg/kubectl/cmd/top_node.go:60 +msgid "" +"\n" +"\t\tDisplay Resource (CPU/Memory/Storage) usage of nodes.\n" +"\n" +"\t\tThe top-node command allows you to see the resource consumption of nodes." +msgstr "" +"\n" +"\t\tVisualizza l'utilizzo di risorse (CPU/Memoria/Storage) dei nodi.\n" +"\n" +"\t\tIl comando top-node consente di visualizzare il consumo di risorse dei " +"nodi." + +#: pkg/kubectl/cmd/top_pod.go:53 +msgid "" +"\n" +"\t\tDisplay Resource (CPU/Memory/Storage) usage of pods.\n" +"\n" +"\t\tThe 'top pod' command allows you to see the resource consumption of " +"pods.\n" +"\n" +"\t\tDue to the metrics pipeline delay, they may be unavailable for a few " +"minutes\n" +"\t\tsince pod creation." +msgstr "" +"\n" +"\t\tVisualizza l'utilizzo di risorse (CPU/Memoria/Storage) dei pod.\n" +"\n" +"\t\tIl comando \"top pod\" consente di visualizzare il consumo delle risorse " +"dei pod.\n" +"\n" +"\t\tA causa del ritardo della pipeline metrica, potrebbero non essere " +"disponibili per alcuni minuti\n" +"\t\teal momento della creazione dei pod." + +#: pkg/kubectl/cmd/top.go:33 +msgid "" +"\n" +"\t\tDisplay Resource (CPU/Memory/Storage) usage.\n" +"\n" +"\t\tThe top command allows you to see the resource consumption for nodes or " +"pods.\n" +"\n" +"\t\tThis command requires Heapster to be correctly configured and working on " +"the server. " +msgstr "" +"\n" +"\t\tVisualizza l'utilizzo di risorse (CPU/Memoria/Storage).\n" +"\n" +"\t\tIl comando top consente di visualizzare il consumo di risorse per nodi o " +"pod.\n" +"\n" +"\t\tQuesto comando richiede che Heapster sia configurato correttamente e che " +"funzioni sul server." + +#: pkg/kubectl/cmd/drain.go:140 +msgid "" +"\n" +"\t\tDrain node in preparation for maintenance.\n" +"\n" +"\t\tThe given node will be marked unschedulable to prevent new pods from " +"arriving.\n" +"\t\t'drain' evicts the pods if the APIServer supports eviction\n" +"\t\t(http://kubernetes.io/docs/admin/disruptions/). Otherwise, it will use " +"normal DELETE\n" +"\t\tto delete the pods.\n" +"\t\tThe 'drain' evicts or deletes all pods except mirror pods (which cannot " +"be deleted through\n" +"\t\tthe API server). If there are DaemonSet-managed pods, drain will not " +"proceed\n" +"\t\twithout --ignore-daemonsets, and regardless it will not delete any\n" +"\t\tDaemonSet-managed pods, because those pods would be immediately replaced " +"by the\n" +"\t\tDaemonSet controller, which ignores unschedulable markings. If there are " +"any\n" +"\t\tpods that are neither mirror pods nor managed by ReplicationController,\n" +"\t\tReplicaSet, DaemonSet, StatefulSet or Job, then drain will not delete any " +"pods unless you\n" +"\t\tuse --force. --force will also allow deletion to proceed if the managing " +"resource of one\n" +"\t\tor more pods is missing.\n" +"\n" +"\t\t'drain' waits for graceful termination. You should not operate on the " +"machine until\n" +"\t\tthe command completes.\n" +"\n" +"\t\tWhen you are ready to put the node back into service, use kubectl " +"uncordon, which\n" +"\t\twill make the node schedulable again.\n" +"\n" +"\t\t![Workflow](http://kubernetes.io/images/docs/kubectl_drain.svg)" +msgstr "" + +#: pkg/kubectl/cmd/edit.go:56 +msgid "" +"\n" +"\t\tEdit a resource from the default editor.\n" +"\n" +"\t\tThe edit command allows you to directly edit any API resource you can " +"retrieve via the\n" +"\t\tcommand line tools. It will open the editor defined by your KUBE_EDITOR, " +"or EDITOR\n" +"\t\tenvironment variables, or fall back to 'vi' for Linux or 'notepad' for " +"Windows.\n" +"\t\tYou can edit multiple objects, although changes are applied one at a " +"time. The command\n" +"\t\taccepts filenames as well as command line arguments, although the files " +"you point to must\n" +"\t\tbe previously saved versions of resources.\n" +"\n" +"\t\tEditing is done with the API version used to fetch the resource.\n" +"\t\tTo edit using a specific API version, fully-qualify the resource, " +"version, and group.\n" +"\n" +"\t\tThe default format is YAML. To edit in JSON, specify \"-o json\".\n" +"\n" +"\t\tThe flag --windows-line-endings can be used to force Windows line " +"endings,\n" +"\t\totherwise the default for your operating system will be used.\n" +"\n" +"\t\tIn the event an error occurs while updating, a temporary file will be " +"created on disk\n" +"\t\tthat contains your unapplied changes. The most common error when updating " +"a resource\n" +"\t\tis another editor changing the resource on the server. When this occurs, " +"you will have\n" +"\t\tto apply your changes to the newer version of the resource, or update " +"your temporary\n" +"\t\tsaved copy to include the latest resource version." +msgstr "" + +#: pkg/kubectl/cmd/drain.go:115 +msgid "" +"\n" +"\t\tMark node as schedulable." +msgstr "" +"\n" +"\t\tContrassegna il nodo come programmabile." + +#: pkg/kubectl/cmd/drain.go:90 +msgid "" +"\n" +"\t\tMark node as unschedulable." +msgstr "" +"\n" +"\t\tContrassegnare il nodo come non programmabile." + +#: pkg/kubectl/cmd/completion.go:47 +msgid "" +"\n" +"\t\tOutput shell completion code for the specified shell (bash or zsh).\n" +"\t\tThe shell code must be evalutated to provide interactive\n" +"\t\tcompletion of kubectl commands. This can be done by sourcing it from\n" +"\t\tthe .bash_profile.\n" +"\n" +"\t\tNote: this requires the bash-completion framework, which is not " +"installed\n" +"\t\tby default on Mac. This can be installed by using homebrew:\n" +"\n" +"\t\t $ brew install bash-completion\n" +"\n" +"\t\tOnce installed, bash_completion must be evaluated. This can be done by " +"adding the\n" +"\t\tfollowing line to the .bash_profile\n" +"\n" +"\t\t $ source $(brew --prefix)/etc/bash_completion\n" +"\n" +"\t\tNote for zsh users: [1] zsh completions are only supported in versions of " +"zsh >= 5.2" +msgstr "" + +#: pkg/kubectl/cmd/rollingupdate.go:45 +msgid "" +"\n" +"\t\tPerform a rolling update of the given ReplicationController.\n" +"\n" +"\t\tReplaces the specified replication controller with a new replication " +"controller by updating one pod at a time to use the\n" +"\t\tnew PodTemplate. The new-controller.json must specify the same namespace " +"as the\n" +"\t\texisting replication controller and overwrite at least one (common) label " +"in its replicaSelector.\n" +"\n" +"\t\t![Workflow](http://kubernetes.io/images/docs/kubectl_rollingupdate.svg)" +msgstr "" + +#: pkg/kubectl/cmd/replace.go:40 +msgid "" +"\n" +"\t\tReplace a resource by filename or stdin.\n" +"\n" +"\t\tJSON and YAML formats are accepted. If replacing an existing resource, " +"the\n" +"\t\tcomplete resource spec must be provided. This can be obtained by\n" +"\n" +"\t\t $ kubectl get TYPE NAME -o yaml\n" +"\n" +"\t\tPlease refer to the models in https://htmlpreview.github.io/?https://" +"github.com/kubernetes/kubernetes/blob/HEAD/docs/api-reference/v1/definitions." +"html to find if a field is mutable." +msgstr "" + +#: pkg/kubectl/cmd/scale.go:34 +msgid "" +"\n" +"\t\tSet a new size for a Deployment, ReplicaSet, Replication Controller, or " +"Job.\n" +"\n" +"\t\tScale also allows users to specify one or more preconditions for the " +"scale action.\n" +"\n" +"\t\tIf --current-replicas or --resource-version is specified, it is validated " +"before the\n" +"\t\tscale is attempted, and it is guaranteed that the precondition holds true " +"when the\n" +"\t\tscale is sent to the server." +msgstr "" +"\n" +"\t\tImposta una nuova dimensione per Deployment, ReplicaSet, Replication " +"Controller, o Job.\n" +"\n" +"\t\tScala consente anche agli utenti di specificare una o più condizioni " +"preliminari per l'azione della scala.\n" +"\n" +"\t\tSe --current-replicas o --resource-version sono specificate, viene " +"convalidata prima di\n" +"\t\ttentare scale, ed è garantito che la precondizione vale quando\n" +"\t\tscale viene inviata al server.." + +#: pkg/kubectl/cmd/apply_set_last_applied.go:62 +msgid "" +"\n" +"\t\tSet the latest last-applied-configuration annotations by setting it to " +"match the contents of a file.\n" +"\t\tThis results in the last-applied-configuration being updated as though " +"'kubectl apply -f ' was run,\n" +"\t\twithout updating any other parts of the object." +msgstr "" +"\n" +"\t\tImposta le annotazioni dell'ultima-configurazione-applicata impostandola " +"in modo che corrisponda al contenuto di un file.\n" +"\t\tCiò determina l'aggiornamento dell'ultima-configurazione-applicata come " +"se 'kubectl apply -f ' fosse stato eseguito,\n" +"\t\tsenza aggiornare altre parti dell'oggetto." + +#: pkg/kubectl/cmd/proxy.go:36 +msgid "" +"\n" +"\t\tTo proxy all of the kubernetes api and nothing else, use:\n" +"\n" +"\t\t $ kubectl proxy --api-prefix=/\n" +"\n" +"\t\tTo proxy only part of the kubernetes api and also some static files:\n" +"\n" +"\t\t $ kubectl proxy --www=/my/files --www-prefix=/static/ --api-prefix=/" +"api/\n" +"\n" +"\t\tThe above lets you 'curl localhost:8001/api/v1/pods'.\n" +"\n" +"\t\tTo proxy the entire kubernetes api at a different root, use:\n" +"\n" +"\t\t $ kubectl proxy --api-prefix=/custom/\n" +"\n" +"\t\tThe above lets you 'curl localhost:8001/custom/api/v1/pods'" +msgstr "" +"\n" +"\t\tPer proxy tutti i kubernetes api e nient'altro, utilizzare:\n" +"\n" +"\t\t $ kubectl proxy --api-prefix=/\n" +"\n" +"\t\tPer proxy solo una parte dei kubernetes api e anche alcuni file static\n" +"\n" +"\t\t $ kubectl proxy --www=/my/files --www-prefix=/static/ --api-prefix=/" +"api/\n" +"\n" +"\t\tQuanto sopra consente 'curl localhost:8001/api/v1/pods'.\n" +"\n" +"\t\tPer eseguire il proxy tutti i kubernetes api in una radice diversa, " +"utilizzare:\n" +"\n" +"\t\t $ kubectl proxy --api-prefix=/custom/\n" +"\n" +"\t\tQuanto sopra ti permette 'curl localhost:8001/custom/api/v1/pods'" + +#: pkg/kubectl/cmd/patch.go:59 +msgid "" +"\n" +"\t\tUpdate field(s) of a resource using strategic merge patch\n" +"\n" +"\t\tJSON and YAML formats are accepted.\n" +"\n" +"\t\tPlease refer to the models in https://htmlpreview.github.io/?https://" +"github.com/kubernetes/kubernetes/blob/HEAD/docs/api-reference/v1/definitions." +"html to find if a field is mutable." +msgstr "" +"\n" +"\t\tAggiorna i campi di una risorsa utilizzando la merge patch strategica\n" +"\n" +"\t\tSono accettati i formati JSON e YAML.\n" +"\n" +"\t\tSi prega di fare riferimento ai modelli in https://htmlpreview.github.io/?" +"https://github.com/kubernetes/kubernetes/blob/HEAD/docs/api-reference/v1/" +"definitions.html per trovare se un campo è mutevole." + +#: pkg/kubectl/cmd/label.go:70 +#, c-format +msgid "" +"\n" +"\t\tUpdate the labels on a resource.\n" +"\n" +"\t\t* A label must begin with a letter or number, and may contain letters, " +"numbers, hyphens, dots, and underscores, up to %[1]d characters.\n" +"\t\t* If --overwrite is true, then existing labels can be overwritten, " +"otherwise attempting to overwrite a label will result in an error.\n" +"\t\t* If --resource-version is specified, then updates will use this resource " +"version, otherwise the existing resource-version will be used." +msgstr "" +"\n" +"\t\tAggiorna le label di una risorsa.\n" +"\n" +"\t\t* A label must begin with a letter or number, and may contain letters, " +"numbers, hyphens, dots, and underscores, up to %[1]d characters.\n" +"\t\t* If --overwrite is true, then existing labels can be overwritten, " +"otherwise attempting to overwrite a label will result in an error.\n" +"\t\t* If --resource-version is specified, then updates will use this resource " +"version, otherwise the existing resource-version will be used." + +#: pkg/kubectl/cmd/taint.go:58 +#, c-format +msgid "" +"\n" +"\t\tUpdate the taints on one or more nodes.\n" +"\n" +"\t\t* A taint consists of a key, value, and effect. As an argument here, it " +"is expressed as key=value:effect.\n" +"\t\t* The key must begin with a letter or number, and may contain letters, " +"numbers, hyphens, dots, and underscores, up to %[1]d characters.\n" +"\t\t* The value must begin with a letter or number, and may contain letters, " +"numbers, hyphens, dots, and underscores, up to %[2]d characters.\n" +"\t\t* The effect must be NoSchedule, PreferNoSchedule or NoExecute.\n" +"\t\t* Currently taint can only apply to node." +msgstr "" + +#: pkg/kubectl/cmd/apply_view_last_applied.go:46 +msgid "" +"\n" +"\t\tView the latest last-applied-configuration annotations by type/name or " +"file.\n" +"\n" +"\t\tThe default output will be printed to stdout in YAML format. One can use -" +"o option\n" +"\t\tto change output format." +msgstr "" +"\n" +"\t\tVisualizza le annotazioni dell'ultima-configurazione-applicata per tipo/" +"nome o file.\n" +"\n" +"\t\tL'output predefinito verrà stampato su stdout nel formato YAML. Si può " +"usare l'opzione -o\n" +"\t\tPer cambiare il formato di output." + +#: pkg/kubectl/cmd/cp.go:37 +msgid "" +"\n" +"\t # !!!Important Note!!!\n" +"\t # Requires that the 'tar' binary is present in your container\n" +"\t # image. If 'tar' is not present, 'kubectl cp' will fail.\n" +"\n" +"\t # Copy /tmp/foo_dir local directory to /tmp/bar_dir in a remote pod in " +"the default namespace\n" +"\t\tkubectl cp /tmp/foo_dir :/tmp/bar_dir\n" +"\n" +" # Copy /tmp/foo local file to /tmp/bar in a remote pod in a specific " +"container\n" +"\t\tkubectl cp /tmp/foo :/tmp/bar -c \n" +"\n" +"\t\t# Copy /tmp/foo local file to /tmp/bar in a remote pod in namespace \n" +"\t\tkubectl cp /tmp/foo /:/tmp/bar\n" +"\n" +"\t\t# Copy /tmp/foo from a remote pod to /tmp/bar locally\n" +"\t\tkubectl cp /:/tmp/foo /tmp/bar" +msgstr "" + +#: pkg/kubectl/cmd/create_secret.go:205 +msgid "" +"\n" +"\t # Create a new TLS secret named tls-secret with the given key pair:\n" +"\t kubectl create secret tls tls-secret --cert=path/to/tls.cert --key=path/" +"to/tls.key" +msgstr "" +"\n" +"\t # Crea un nuovo secret TLS denominato tls-secret con la coppia di dati " +"fornita:\n" +"\t kubectl create secret tls tls-secret --cert=path/to/tls.cert --key=path/" +"to/tls.key" + +#: pkg/kubectl/cmd/create_namespace.go:35 +msgid "" +"\n" +"\t # Create a new namespace named my-namespace\n" +"\t kubectl create namespace my-namespace" +msgstr "" +"\n" +"\t # Crea un nuovo namespace denominato my-namespace\n" +"\t kubectl create namespace my-namespace" + +#: pkg/kubectl/cmd/create_secret.go:59 +msgid "" +"\n" +"\t # Create a new secret named my-secret with keys for each file in folder " +"bar\n" +"\t kubectl create secret generic my-secret --from-file=path/to/bar\n" +"\n" +"\t # Create a new secret named my-secret with specified keys instead of " +"names on disk\n" +"\t kubectl create secret generic my-secret --from-file=ssh-privatekey=~/.ssh/" +"id_rsa --from-file=ssh-publickey=~/.ssh/id_rsa.pub\n" +"\n" +"\t # Create a new secret named my-secret with key1=supersecret and " +"key2=topsecret\n" +"\t kubectl create secret generic my-secret --from-literal=key1=supersecret --" +"from-literal=key2=topsecret" +msgstr "" +"\n" +"\t # Crea un nuovo secret denominato my-secret con i tasti per ogni file " +"nella barra delle cartelle\n" +"\t kubectl create secret generic my-secret --from-file=path/to/bar\n" +"\n" +"\t # Crea un nuovo secret denominato my-secret con le chiavi specificate " +"anziché i nomi sul disco\n" +"\t kubectl create secret generic my-secret --from-file=ssh-privatekey=~/.ssh/" +"id_rsa --from-file=ssh-publickey=~/.ssh/id_rsa.pub\n" +"\n" +"\t # Crea un nuovo secret denominato my-secret con key1 = supersecret e key2 " +"= topsecret\n" +"\t kubectl create secret generic my-secret --from-literal=key1=supersecret --" +"from-literal=key2=topsecret" + +#: pkg/kubectl/cmd/create_serviceaccount.go:35 +msgid "" +"\n" +"\t # Create a new service account named my-service-account\n" +"\t kubectl create serviceaccount my-service-account" +msgstr "" +"\n" +"\t # Crea un nuovo service account denominato my-service-account\n" +"\t kubectl create serviceaccount my-service-account" + +#: pkg/kubectl/cmd/create_service.go:232 +msgid "" +"\n" +"\t# Create a new ExternalName service named my-ns \n" +"\tkubectl create service externalname my-ns --external-name bar.com" +msgstr "" +"\n" +"\t# Crea un nuovo servizio ExternalName denominato my-ns \n" +"\tkubectl create service externalname my-ns --external-name bar.com" + +#: pkg/kubectl/cmd/create_service.go:225 +msgid "" +"\n" +"\tCreate an ExternalName service with the specified name.\n" +"\n" +"\tExternalName service references to an external DNS address instead of\n" +"\tonly pods, which will allow application authors to reference services\n" +"\tthat exist off platform, on other clusters, or locally." +msgstr "" +"\n" +"\tCrea un servizio ExternalName con il nome specificato.\n" +"\n" +"\tIl servizio ExternalName fa riferimento a un indirizzo DNS esterno \n" +"\tsolo pod, che permetteranno agli autori delle applicazioni di utilizzare i " +"servizi di riferimento\n" +"\tche esistono fuori dalla piattaforma, su altri cluster, o localmente.." + +#: pkg/kubectl/cmd/help.go:30 +msgid "" +"\n" +"\tHelp provides help for any command in the application.\n" +"\tSimply type kubectl help [path to command] for full details." +msgstr "" +"\n" +"\tHelp fornisce assistenza per qualsiasi comando nell'applicazione.\n" +"\tBasta digitare kubectl help [path to command] per i dettagli completi." + +#: pkg/kubectl/cmd/create_service.go:173 +msgid "" +"\n" +" # Create a new LoadBalancer service named my-lbs\n" +" kubectl create service loadbalancer my-lbs --tcp=5678:8080" +msgstr "" +"\n" +" # Creare un nuovo servizio LoadBalancer denominato my-lbs\n" +" kubectl create service loadbalancer my-lbs --tcp=5678:8080" + +#: pkg/kubectl/cmd/create_service.go:53 +msgid "" +"\n" +" # Create a new clusterIP service named my-cs\n" +" kubectl create service clusterip my-cs --tcp=5678:8080\n" +"\n" +" # Create a new clusterIP service named my-cs (in headless mode)\n" +" kubectl create service clusterip my-cs --clusterip=\"None\"" +msgstr "" +"\n" +" # Creare un nuovo servizio clusterIP denominato my-cs\n" +" kubectl create service clusterip my-cs --tcp=5678:8080\n" +"\n" +" # Creare un nuovo servizio clusterIP denominato my-cs (in modalità " +"headless)\n" +" kubectl create service clusterip my-cs --clusterip=\"None\"" + +#: pkg/kubectl/cmd/create_deployment.go:36 +msgid "" +"\n" +" # Create a new deployment named my-dep that runs the busybox image.\n" +" kubectl create deployment my-dep --image=busybox" +msgstr "" +"\n" +" # Crea una nuovo deployment chiamato my-dep che esegue l'immagine " +"busybox.\n" +" kubectl create deployment my-dep --image=busybox" + +#: pkg/kubectl/cmd/create_service.go:116 +msgid "" +"\n" +" # Create a new nodeport service named my-ns\n" +" kubectl create service nodeport my-ns --tcp=5678:8080" +msgstr "" +"\n" +" # Creare un nuovo servizio nodeport denominato my-ns\n" +" kubectl create service nodeport my-ns --tcp=5678:8080" + +#: pkg/kubectl/cmd/clusterinfo_dump.go:62 +msgid "" +"\n" +" # Dump current cluster state to stdout\n" +" kubectl cluster-info dump\n" +"\n" +" # Dump current cluster state to /path/to/cluster-state\n" +" kubectl cluster-info dump --output-directory=/path/to/cluster-state\n" +"\n" +" # Dump all namespaces to stdout\n" +" kubectl cluster-info dump --all-namespaces\n" +"\n" +" # Dump a set of namespaces to /path/to/cluster-state\n" +" kubectl cluster-info dump --namespaces default,kube-system --output-" +"directory=/path/to/cluster-state" +msgstr "" +"\n" +" # Dump dello stato corrente del cluster verso stdout\n" +" kubectl cluster-info dump\n" +"\n" +" # Dump dello stato corrente del cluster verso /path/to/cluster-state\n" +" kubectl cluster-info dump --output-directory=/path/to/cluster-state\n" +"\n" +" # Dump di tutti i namespaces verso stdout\n" +" kubectl cluster-info dump --all-namespaces\n" +"\n" +" # Dump di un set di namespace verso /path/to/cluster-state\n" +" kubectl cluster-info dump --namespaces default,kube-system --output-" +"directory=/path/to/cluster-state" + +#: pkg/kubectl/cmd/annotate.go:78 +msgid "" +"\n" +" # Update pod 'foo' with the annotation 'description' and the value 'my " +"frontend'.\n" +" # If the same annotation is set multiple times, only the last value will " +"be applied\n" +" kubectl annotate pods foo description='my frontend'\n" +"\n" +" # Update a pod identified by type and name in \"pod.json\"\n" +" kubectl annotate -f pod.json description='my frontend'\n" +"\n" +" # Update pod 'foo' with the annotation 'description' and the value 'my " +"frontend running nginx', overwriting any existing value.\n" +" kubectl annotate --overwrite pods foo description='my frontend running " +"nginx'\n" +"\n" +" # Update all pods in the namespace\n" +" kubectl annotate pods --all description='my frontend running nginx'\n" +"\n" +" # Update pod 'foo' only if the resource is unchanged from version 1.\n" +" kubectl annotate pods foo description='my frontend running nginx' --" +"resource-version=1\n" +"\n" +" # Update pod 'foo' by removing an annotation named 'description' if it " +"exists.\n" +" # Does not require the --overwrite flag.\n" +" kubectl annotate pods foo description-" +msgstr "" +"\n" +" # Aggiorna il pod 'foo' con annotazione 'description'e il valore 'my " +"frontend'.\n" +" # Se la stessa annotazione è impostata più volte, verrà applicato solo " +"l'ultimo valore\n" +" kubectl annotate pods foo description='my frontend'\n" +"\n" +" # Aggiorna un pod identificato per tipo e nome in \"pod.json\"\n" +" kubectl annotate -f pod.json description='my frontend'\n" +"\n" +" # Aggiorna pod 'foo' con la annotazione 'description' e il valore 'my " +"frontend running nginx', sovrascrivendo qualsiasi valore esistente.\n" +" kubectl annotate --overwrite pods foo description='my frontend running " +"nginx'\n" +"\n" +" # Aggiorna tutti i baccelli nel namespace\n" +" kubectl annotate pods --all description='my frontend running nginx'\n" +"\n" +" # Aggiorna il pod 'foo' solo se la risorsa è invariata dalla versione 1.\n" +" kubectl annotate pods foo description='my frontend running nginx' --" +"resource-version=1\n" +"\n" +" # Aggiorna il pod 'foo' rimuovendo un'annotazione denominata " +"'descrizione' se esiste.\n" +" # Non richiede flag -overwrite.\n" +" kubectl annotate pods foo description-" + +#: pkg/kubectl/cmd/create_service.go:170 +msgid "" +"\n" +" Create a LoadBalancer service with the specified name." +msgstr "" +"\n" +" Crea un servizio LoadBalancer con il nome specificato." + +#: pkg/kubectl/cmd/create_service.go:50 +msgid "" +"\n" +" Create a clusterIP service with the specified name." +msgstr "" +"\n" +" Crea un servizio clusterIP con il nome specificato." + +#: pkg/kubectl/cmd/create_deployment.go:33 +msgid "" +"\n" +" Create a deployment with the specified name." +msgstr "" +"\n" +" Creare un deployment con il nome specificato." + +#: pkg/kubectl/cmd/create_service.go:113 +msgid "" +"\n" +" Create a nodeport service with the specified name." +msgstr "" +"\n" +" Creare un servizio nodeport con il nome specificato." + +#: pkg/kubectl/cmd/clusterinfo_dump.go:53 +msgid "" +"\n" +" Dumps cluster info out suitable for debugging and diagnosing cluster " +"problems. By default, dumps everything to\n" +" stdout. You can optionally specify a directory with --output-directory. " +"If you specify a directory, kubernetes will\n" +" build a set of files in that directory. By default only dumps things in " +"the 'kube-system' namespace, but you can\n" +" switch to a different namespace with the --namespaces flag, or specify --" +"all-namespaces to dump all namespaces.\n" +"\n" +" The command also dumps the logs of all of the pods in the cluster, these " +"logs are dumped into different directories\n" +" based on namespace and pod name." +msgstr "" +"\n" +" Dump delle informazioni di cluster idonee per il debug e la diagnostica " +"di problemi di cluster. Per impostazione predefinita, tutto\n" +"    verso stdout. È possibile specificare opzionalmente una directory con --" +"output-directory. Se si specifica una directory, kubernetes \n" +" creearà un insieme di file in quella directory. Per impostazione " +"predefinita, dumps solo i dati del namespace \"kube-system\", ma è\n" +" possibile passare ad namespace diverso con il flag --namespaces o " +"specificare --all-namespaces per il dump di tutti i namespace.\n" +"\n" +"     Il comando esegue dump anche dei log di tutti i pod del cluster, questi " +"log vengono scaricati in directory differenti\n" +"     basati sul namespace e sul nome del pod." + +#: pkg/kubectl/cmd/clusterinfo.go:37 +msgid "" +"\n" +" Display addresses of the master and services with label kubernetes.io/" +"cluster-service=true\n" +" To further debug and diagnose cluster problems, use 'kubectl cluster-info " +"dump'." +msgstr "" +"\n" +" Visualizza gli indirizzi del master e dei servizi con label kubernetes.io/" +"cluster-service=true\n" +"  Per ulteriore debug e diagnosticare i problemi di cluster, utilizzare " +"'kubectl cluster-info dump'." + +#: pkg/kubectl/cmd/create_quota.go:62 +msgid "" +"A comma-delimited set of quota scopes that must all match each object tracked " +"by the quota." +msgstr "" +"Un insieme delimitato-da-virgole di quota scopes che devono corrispondere a " +"ciascun oggetto gestito dalla quota." + +#: pkg/kubectl/cmd/create_quota.go:61 +msgid "" +"A comma-delimited set of resource=quantity pairs that define a hard limit." +msgstr "" +"Un insieme delimitato-da-virgola di coppie risorsa = quantità che definiscono " +"un hard limit." + +#: pkg/kubectl/cmd/create_pdb.go:64 +msgid "" +"A label selector to use for this budget. Only equality-based selector " +"requirements are supported." +msgstr "" +"Un label selector da utilizzare per questo budget. Sono supportati solo i " +"selettori equality-based selector." + +#: pkg/kubectl/cmd/expose.go:104 +msgid "" +"A label selector to use for this service. Only equality-based selector " +"requirements are supported. If empty (the default) infer the selector from " +"the replication controller or replica set.)" +msgstr "" +"Un selettore di label da utilizzare per questo servizio. Sono supportati solo " +"equality-based selector. Se vuota (default) dedurre il selettore dal " +"replication controller o replica set.)" + +#: pkg/kubectl/cmd/run.go:139 +msgid "A schedule in the Cron format the job should be run with." +msgstr "Un calendario in formato Cron del lavoro che deve essere eseguito." + +#: pkg/kubectl/cmd/expose.go:109 +msgid "" +"Additional external IP address (not managed by Kubernetes) to accept for the " +"service. If this IP is routed to a node, the service can be accessed by this " +"IP in addition to its generated service IP." +msgstr "" +"Indirizzo IP esterno aggiuntivo (non gestito da Kubernetes) da accettare per " +"il servizio. Se questo IP viene indirizzato a un nodo, è possibile accedere " +"da questo IP in aggiunta al service IP generato." + +#: pkg/kubectl/cmd/expose.go:110 pkg/kubectl/cmd/run.go:122 +msgid "" +"An inline JSON override for the generated object. If this is non-empty, it is " +"used to override the generated object. Requires that the object supply a " +"valid apiVersion field." +msgstr "" +"Un override JSON inline per l'oggetto generato. Se questo non è vuoto, viene " +"utilizzato per ignorare l'oggetto generato. Richiede che l'oggetto fornisca " +"un campo valido apiVersion." + +#: pkg/kubectl/cmd/run.go:137 +msgid "" +"An inline JSON override for the generated service object. If this is non-" +"empty, it is used to override the generated object. Requires that the object " +"supply a valid apiVersion field. Only used if --expose is true." +msgstr "" +"Un override JSON inline per l'oggetto di servizio generato. Se questo non è " +"vuoto, viene utilizzato per ignorare l'oggetto generato. Richiede che " +"l'oggetto fornisca un campo valido apiVersion. Utilizzato solo se --expose è " +"true." + +#: pkg/kubectl/cmd/apply.go:104 +msgid "Apply a configuration to a resource by filename or stdin" +msgstr "Applica una configurazione risorsa per nomefile o stdin" + +#: pkg/kubectl/cmd/certificates.go:72 +msgid "Approve a certificate signing request" +msgstr "Approva una richiesta di firma del certificato" + +#: pkg/kubectl/cmd/create_service.go:82 +msgid "" +"Assign your own ClusterIP or set to 'None' for a 'headless' service (no " +"loadbalancing)." +msgstr "" +"Assegnare il proprio ClusterIP o impostare su 'None' per un servizio " +"'headless' (nessun bilanciamento del carico)." + +#: pkg/kubectl/cmd/attach.go:70 +msgid "Attach to a running container" +msgstr "Collega a un container in esecuzione" + +#: pkg/kubectl/cmd/autoscale.go:56 +msgid "Auto-scale a Deployment, ReplicaSet, or ReplicationController" +msgstr "Auto-scale a Deployment, ReplicaSet, o ReplicationController" + +#: pkg/kubectl/cmd/expose.go:113 +msgid "" +"ClusterIP to be assigned to the service. Leave empty to auto-allocate, or set " +"to 'None' to create a headless service." +msgstr "" +"ClusterIP da assegnare al servizio. Lasciare vuoto per allocare " +"automaticamente o impostare su 'None' per creare un servizio headless." + +#: pkg/kubectl/cmd/create_clusterrolebinding.go:56 +msgid "ClusterRole this ClusterRoleBinding should reference" +msgstr "ClusterRole a cui questo ClusterRoleBinding fa riferimento" + +#: pkg/kubectl/cmd/create_rolebinding.go:56 +msgid "ClusterRole this RoleBinding should reference" +msgstr "ClusterRole a cui questo RoleBinding fa riferimento" + +#: pkg/kubectl/cmd/rollingupdate.go:102 +msgid "" +"Container name which will have its image upgraded. Only relevant when --image " +"is specified, ignored otherwise. Required when using --image on a multi-" +"container pod" +msgstr "" +"Nome container che avrà la sua immagine aggiornata. Soltanto rilevante quando " +"--image è specificato, altrimenti ignorato. Necessario quando si utilizza --" +"image su un contenitore a più contenitori" + +#: pkg/kubectl/cmd/convert.go:68 +msgid "Convert config files between different API versions" +msgstr "Convertire i file di configurazione tra diverse versioni APIs" + +#: pkg/kubectl/cmd/cp.go:65 +msgid "Copy files and directories to and from containers." +msgstr "Copiare file e directory da e verso i container." + +#: pkg/kubectl/cmd/create_clusterrolebinding.go:44 +msgid "Create a ClusterRoleBinding for a particular ClusterRole" +msgstr "Crea un ClusterRoleBinding per un ClusterRole particolare" + +#: pkg/kubectl/cmd/create_service.go:182 +msgid "Create a LoadBalancer service." +msgstr "Creare un servizio LoadBalancer." + +#: pkg/kubectl/cmd/create_service.go:125 +msgid "Create a NodePort service." +msgstr "Crea un servizio NodePort." + +#: pkg/kubectl/cmd/create_rolebinding.go:44 +msgid "Create a RoleBinding for a particular Role or ClusterRole" +msgstr "Crea un RoleBinding per un particolare Role o ClusterRole" + +#: pkg/kubectl/cmd/create_secret.go:214 +msgid "Create a TLS secret" +msgstr "Crea un secret TLS" + +#: pkg/kubectl/cmd/create_service.go:69 +msgid "Create a clusterIP service." +msgstr "Crea un servizio clusterIP." + +#: pkg/kubectl/cmd/create_configmap.go:60 +msgid "Create a configmap from a local file, directory or literal value" +msgstr "" +"Crea un configmap da un file locale, una directory o un valore letterale" + +#: pkg/kubectl/cmd/create_deployment.go:46 +msgid "Create a deployment with the specified name." +msgstr "Creare un deployment con il nome specificato." + +#: pkg/kubectl/cmd/create_namespace.go:45 +msgid "Create a namespace with the specified name" +msgstr "Crea un namespace con il nome specificato" + +#: pkg/kubectl/cmd/create_pdb.go:50 +msgid "Create a pod disruption budget with the specified name." +msgstr "Crea un pod disruption budget con il nome specificato." + +#: pkg/kubectl/cmd/create_quota.go:48 +msgid "Create a quota with the specified name." +msgstr "Crea una quota con il nome specificato." + +#: pkg/kubectl/cmd/create.go:63 +msgid "Create a resource by filename or stdin" +msgstr "Crea una risorsa per nome file o stdin" + +#: pkg/kubectl/cmd/create_secret.go:144 +msgid "Create a secret for use with a Docker registry" +msgstr "Crea un secret da utilizzare con un registro Docker" + +#: pkg/kubectl/cmd/create_secret.go:74 +msgid "Create a secret from a local file, directory or literal value" +msgstr "Crea un secret da un file locale, una directory o un valore letterale" + +#: pkg/kubectl/cmd/create_secret.go:35 +msgid "Create a secret using specified subcommand" +msgstr "Crea un secret utilizzando un subcommand specificato" + +#: pkg/kubectl/cmd/create_serviceaccount.go:45 +msgid "Create a service account with the specified name" +msgstr "Creare un account di servizio con il nome specificato" + +#: pkg/kubectl/cmd/create_service.go:37 +msgid "Create a service using specified subcommand." +msgstr "Crea un servizio utilizzando il subcommand specificato." + +#: pkg/kubectl/cmd/create_service.go:241 +msgid "Create an ExternalName service." +msgstr "Crea un servizio ExternalName." + +#: pkg/kubectl/cmd/delete.go:132 +msgid "" +"Delete resources by filenames, stdin, resources and names, or by resources " +"and label selector" +msgstr "" +"Elimina risorse selezionate per nomi di file, stdin, risorse e nomi, o per " +"risorsa e selettore di label" + +#: pkg/kubectl/cmd/config/delete_cluster.go:39 +msgid "Delete the specified cluster from the kubeconfig" +msgstr "Elimina il cluster specificato dal kubeconfig" + +#: pkg/kubectl/cmd/config/delete_context.go:39 +msgid "Delete the specified context from the kubeconfig" +msgstr "Elimina il context specificato dal kubeconfig" + +#: pkg/kubectl/cmd/certificates.go:122 +msgid "Deny a certificate signing request" +msgstr "Nega una richiesta di firma del certificato" + +#: pkg/kubectl/cmd/stop.go:59 +msgid "Deprecated: Gracefully shut down a resource by name or filename" +msgstr "Deprecated: spegne correttamente una risorsa per nome o nome file" + +#: pkg/kubectl/cmd/config/get_contexts.go:64 +msgid "Describe one or many contexts" +msgstr "Descrive uno o più context" + +#: pkg/kubectl/cmd/top_node.go:78 +msgid "Display Resource (CPU/Memory) usage of nodes" +msgstr "Visualizza l'utilizzo di risorse (CPU/Memoria) per nodo" + +#: pkg/kubectl/cmd/top_pod.go:80 +msgid "Display Resource (CPU/Memory) usage of pods" +msgstr "Visualizza l'utilizzo di risorse (CPU/Memoria) per pod." + +#: pkg/kubectl/cmd/top.go:44 +msgid "Display Resource (CPU/Memory) usage." +msgstr "Visualizza l'utilizzo di risorse (CPU/Memoria)." + +#: pkg/kubectl/cmd/clusterinfo.go:51 +msgid "Display cluster info" +msgstr "Visualizza informazioni sul cluster" + +#: pkg/kubectl/cmd/config/get_clusters.go:41 +msgid "Display clusters defined in the kubeconfig" +msgstr "Mostra i cluster definiti nel kubeconfig" + +#: pkg/kubectl/cmd/config/view.go:67 +msgid "Display merged kubeconfig settings or a specified kubeconfig file" +msgstr "" +"Visualizza le impostazioni merged di kubeconfig o un file kubeconfig " +"specificato" + +#: pkg/kubectl/cmd/get.go:111 +msgid "Display one or many resources" +msgstr "Visualizza una o più risorse" + +#: pkg/kubectl/cmd/config/current_context.go:49 +msgid "Displays the current-context" +msgstr "Visualizza il current-context" + +#: pkg/kubectl/cmd/explain.go:51 +msgid "Documentation of resources" +msgstr "Documentazione delle risorse" + +#: pkg/kubectl/cmd/drain.go:178 +msgid "Drain node in preparation for maintenance" +msgstr "Drain node in preparazione alla manutenzione" + +#: pkg/kubectl/cmd/clusterinfo_dump.go:39 +msgid "Dump lots of relevant info for debugging and diagnosis" +msgstr "Dump di un sacco di informazioni pertinenti per il debug e la diagnosi" + +#: pkg/kubectl/cmd/edit.go:110 +msgid "Edit a resource on the server" +msgstr "Modificare una risorsa sul server" + +#: pkg/kubectl/cmd/create_secret.go:160 +msgid "Email for Docker registry" +msgstr "Email per il registro Docker" + +#: pkg/kubectl/cmd/exec.go:69 +msgid "Execute a command in a container" +msgstr "Esegui un comando in un contenitore" + +#: pkg/kubectl/cmd/rollingupdate.go:103 +msgid "" +"Explicit policy for when to pull container images. Required when --image is " +"same as existing image, ignored otherwise." +msgstr "" +"Politica esplicita per il pull delle immagini container. Richiesto quando --" +"image è uguale all'immagine esistente, altrimenti ignorata." + +#: pkg/kubectl/cmd/portforward.go:76 +msgid "Forward one or more local ports to a pod" +msgstr "Inoltra una o più porte locali a un pod" + +#: pkg/kubectl/cmd/help.go:37 +msgid "Help about any command" +msgstr "Aiuto per qualsiasi comando" + +#: pkg/kubectl/cmd/expose.go:103 +msgid "" +"IP to assign to the Load Balancer. If empty, an ephemeral IP will be created " +"and used (cloud-provider specific)." +msgstr "" +"IP da assegnare al Load Balancer. Se vuota, un IP effimero verrà creato e " +"utilizzato (specifico per provider cloud)." + +#: pkg/kubectl/cmd/expose.go:112 +msgid "" +"If non-empty, set the session affinity for the service to this; legal values: " +"'None', 'ClientIP'" +msgstr "" +"Se non è vuoto, impostare l'affinità di sessione per il servizio; Valori " +"validi: 'None', 'ClientIP'" + +#: pkg/kubectl/cmd/annotate.go:136 +msgid "" +"If non-empty, the annotation update will only succeed if this is the current " +"resource-version for the object. Only valid when specifying a single resource." +msgstr "" +"Se non è vuoto, l'aggiornamento delle annotazioni avrà successo solo se " +"questa è la resource-version corrente per l'oggetto. Valido solo quando si " +"specifica una singola risorsa." + +#: pkg/kubectl/cmd/label.go:134 +msgid "" +"If non-empty, the labels update will only succeed if this is the current " +"resource-version for the object. Only valid when specifying a single resource." +msgstr "" +"Se non vuoto, l'aggiornamento delle label avrà successo solo se questa è la " +"resource-version corrente per l'oggetto. Valido solo quando si specifica una " +"singola risorsa." + +#: pkg/kubectl/cmd/rollingupdate.go:99 +msgid "" +"Image to use for upgrading the replication controller. Must be distinct from " +"the existing image (either new image or new image tag). Can not be used with " +"--filename/-f" +msgstr "" +"Immagine da utilizzare per aggiornare il replication controller. Deve essere " +"diversa dall'immagine esistente (nuova immagine o nuovo tag immagine). Non " +"può essere utilizzata con --filename/-f" + +#: pkg/kubectl/cmd/rollout/rollout.go:47 +msgid "Manage a deployment rollout" +msgstr "Gestisci un deployment rollout" + +#: pkg/kubectl/cmd/drain.go:128 +msgid "Mark node as schedulable" +msgstr "Contrassegnare il nodo come programmabile" + +#: pkg/kubectl/cmd/drain.go:103 +msgid "Mark node as unschedulable" +msgstr "Contrassegnare il nodo come non programmabile" + +#: pkg/kubectl/cmd/rollout/rollout_pause.go:74 +msgid "Mark the provided resource as paused" +msgstr "Imposta la risorsa indicata in pausa" + +#: pkg/kubectl/cmd/certificates.go:36 +msgid "Modify certificate resources." +msgstr "Modificare le risorse del certificato." + +#: pkg/kubectl/cmd/config/config.go:40 +msgid "Modify kubeconfig files" +msgstr "Modifica i file kubeconfig" + +#: pkg/kubectl/cmd/expose.go:108 +msgid "" +"Name or number for the port on the container that the service should direct " +"traffic to. Optional." +msgstr "" +"Nome o numero di porta nel container verso il quale il servizio deve dirigere " +"il traffico. Opzionale." + +#: pkg/kubectl/cmd/logs.go:113 +msgid "" +"Only return logs after a specific date (RFC3339). Defaults to all logs. Only " +"one of since-time / since may be used." +msgstr "" +"Restituisce solo i log dopo una data specificata (RFC3339). Predefinito tutti " +"i log. È possibile utilizzare solo uno tra data-inizio/a-partire-da." + +#: pkg/kubectl/cmd/completion.go:104 +msgid "Output shell completion code for the specified shell (bash or zsh)" +msgstr "" +"Codice di completamento shell di output per la shell specificata (bash o zsh)" + +#: pkg/kubectl/cmd/convert.go:85 +msgid "" +"Output the formatted object with the given group version (for ex: 'extensions/" +"v1beta1').)" +msgstr "" +"Output dell'oggetto formattato con la versione del gruppo fornito (per " +"esempio: 'extensions/v1beta1').)" + +#: pkg/kubectl/cmd/create_secret.go:158 +msgid "Password for Docker registry authentication" +msgstr "Password per l'autenticazione al registro di Docker" + +#: pkg/kubectl/cmd/create_secret.go:226 +msgid "Path to PEM encoded public key certificate." +msgstr "Percorso certificato di chiave pubblica codificato PEM." + +#: pkg/kubectl/cmd/create_secret.go:227 +msgid "Path to private key associated with given certificate." +msgstr "Percorso alla chiave privata associata a un certificato specificato." + +#: pkg/kubectl/cmd/rollingupdate.go:85 +msgid "Perform a rolling update of the given ReplicationController" +msgstr "Eseguire un rolling update del ReplicationController specificato" + +#: pkg/kubectl/cmd/scale.go:83 +msgid "" +"Precondition for resource version. Requires that the current resource version " +"match this value in order to scale." +msgstr "" +"Prerequisito per la versione delle risorse. Richiede che la versione corrente " +"delle risorse corrisponda a questo valore per scalare." + +#: pkg/kubectl/cmd/version.go:40 +msgid "Print the client and server version information" +msgstr "Stampa per client e server le informazioni sulla versione" + +#: pkg/kubectl/cmd/options.go:38 +msgid "Print the list of flags inherited by all commands" +msgstr "Stampa l'elenco flag ereditati da tutti i comandi" + +#: pkg/kubectl/cmd/logs.go:93 +msgid "Print the logs for a container in a pod" +msgstr "Stampa i log per container in un pod" + +#: pkg/kubectl/cmd/replace.go:71 +msgid "Replace a resource by filename or stdin" +msgstr "Sostituire una risorsa per nomefile o stdin" + +#: pkg/kubectl/cmd/rollout/rollout_resume.go:72 +msgid "Resume a paused resource" +msgstr "Riprendere una risorsa in pausa" + +#: pkg/kubectl/cmd/create_rolebinding.go:57 +msgid "Role this RoleBinding should reference" +msgstr "Ruolo di riferimento per RoleBinding" + +#: pkg/kubectl/cmd/run.go:97 +msgid "Run a particular image on the cluster" +msgstr "Esegui una particolare immagine nel cluster" + +#: pkg/kubectl/cmd/proxy.go:69 +msgid "Run a proxy to the Kubernetes API server" +msgstr "Eseguire un proxy al server Kubernetes API" + +#: pkg/kubectl/cmd/create_secret.go:161 +msgid "Server location for Docker registry" +msgstr "Posizione del server per il Registro Docker" + +#: pkg/kubectl/cmd/scale.go:71 +msgid "" +"Set a new size for a Deployment, ReplicaSet, Replication Controller, or Job" +msgstr "" +"Imposta una nuova dimensione per Deployment, ReplicaSet, Replication " +"Controller, o Job" + +#: pkg/kubectl/cmd/set/set.go:38 +msgid "Set specific features on objects" +msgstr "Imposta caratteristiche specifiche sugli oggetti" + +#: pkg/kubectl/cmd/apply_set_last_applied.go:83 +msgid "" +"Set the last-applied-configuration annotation on a live object to match the " +"contents of a file." +msgstr "" +"Imposta l'annotazione dell'ultima-configurazione-applicata ad un oggetto live " +"per abbinare il contenuto di un file." + +#: pkg/kubectl/cmd/set/set_selector.go:82 +msgid "Set the selector on a resource" +msgstr "Impostare il selettore di una risorsa" + +#: pkg/kubectl/cmd/config/create_cluster.go:68 +msgid "Sets a cluster entry in kubeconfig" +msgstr "Imposta una voce cluster in kubeconfig" + +#: pkg/kubectl/cmd/config/create_context.go:58 +msgid "Sets a context entry in kubeconfig" +msgstr "Imposta una voce context in kubeconfig" + +#: pkg/kubectl/cmd/config/create_authinfo.go:104 +msgid "Sets a user entry in kubeconfig" +msgstr "Imposta una voce utente in kubeconfig" + +#: pkg/kubectl/cmd/config/set.go:60 +msgid "Sets an individual value in a kubeconfig file" +msgstr "Imposta un singolo valore in un file kubeconfig" + +#: pkg/kubectl/cmd/config/use_context.go:49 +msgid "Sets the current-context in a kubeconfig file" +msgstr "Imposta il current-context in un file kubeconfig" + +#: pkg/kubectl/cmd/describe.go:86 +msgid "Show details of a specific resource or group of resources" +msgstr "Mostra i dettagli di una specifiche risorsa o un gruppo di risorse" + +#: pkg/kubectl/cmd/rollout/rollout_status.go:58 +msgid "Show the status of the rollout" +msgstr "Mostra lo stato del rollout" + +#: pkg/kubectl/cmd/expose.go:106 +msgid "Synonym for --target-port" +msgstr "Sinonimo di --target-port" + +#: pkg/kubectl/cmd/expose.go:88 +msgid "" +"Take a replication controller, service, deployment or pod and expose it as a " +"new Kubernetes Service" +msgstr "" +"Prende un replication controller, service, deployment o un pod e lo espone " +"come nuovo servizio Kubernetes" + +#: pkg/kubectl/cmd/run.go:117 +msgid "The image for the container to run." +msgstr "L'immagine per il container da eseguire." + +#: pkg/kubectl/cmd/run.go:119 +msgid "" +"The image pull policy for the container. If left empty, this value will not " +"be specified by the client and defaulted by the server" +msgstr "" +"La politica di pull dell'immagine per il container. Se lasciato vuoto, questo " +"valore non verrà specificato dal client e predefinito dal server" + +#: pkg/kubectl/cmd/rollingupdate.go:101 +msgid "" +"The key to use to differentiate between two different controllers, default " +"'deployment'. Only relevant when --image is specified, ignored otherwise" +msgstr "" +"La chiave da utilizzare per distinguere tra due controller diversi, " +"predefinito \"deployment\". Rilevante soltanto quando --image è specificato, " +"altrimenti ignorato" + +#: pkg/kubectl/cmd/create_pdb.go:63 +msgid "The minimum number or percentage of available pods this budget requires." +msgstr "" +"Il numero minimo o la percentuale di pod disponibili che questo budget " +"richiede." + +#: pkg/kubectl/cmd/expose.go:111 +msgid "The name for the newly created object." +msgstr "Il nome dell'oggetto appena creato." + +#: pkg/kubectl/cmd/autoscale.go:72 +msgid "" +"The name for the newly created object. If not specified, the name of the " +"input resource will be used." +msgstr "" +"Il nome dell'oggetto appena creato. Se non specificato, verrà utilizzato il " +"nome della risorsa di input." + +#: pkg/kubectl/cmd/run.go:116 +msgid "" +"The name of the API generator to use, see http://kubernetes.io/docs/user-" +"guide/kubectl-conventions/#generators for a list." +msgstr "" +"Il nome del generatore API da utilizzare, si veda http://kubernetes.io/docs/" +"user-guide/kubectl-conventions/#generators per un elenco." + +#: pkg/kubectl/cmd/autoscale.go:67 +msgid "" +"The name of the API generator to use. Currently there is only 1 generator." +msgstr "" +"Il nome del generatore API da utilizzare. Attualmente c'è solo 1 generatore." + +#: pkg/kubectl/cmd/expose.go:99 +msgid "" +"The name of the API generator to use. There are 2 generators: 'service/v1' " +"and 'service/v2'. The only difference between them is that service port in v1 " +"is named 'default', while it is left unnamed in v2. Default is 'service/v2'." +msgstr "" +"Il nome del generatore API da utilizzare. Ci sono 2 generatori: 'service/v1' " +"e 'service/v2'. L'unica differenza tra loro è che la porta di servizio in v1 " +"è denominata \"predefinita\", mentre viene lasciata unnamed in v2. Il valore " +"predefinito è 'service/v2'." + +#: pkg/kubectl/cmd/run.go:136 +msgid "" +"The name of the generator to use for creating a service. Only used if --" +"expose is true" +msgstr "" +"Il nome del generatore da utilizzare per la creazione di un servizio. " +"Utilizzato solo se --expose è true" + +#: pkg/kubectl/cmd/expose.go:100 +msgid "The network protocol for the service to be created. Default is 'TCP'." +msgstr "" +"Il protocollo di rete per il servizio da creare. Il valore predefinito è " +"'TCP'." + +#: pkg/kubectl/cmd/expose.go:101 +msgid "" +"The port that the service should serve on. Copied from the resource being " +"exposed, if unspecified" +msgstr "" +"La porta che il servizio deve servire. Copiato dalla risorsa esposta, se non " +"specificata" + +#: pkg/kubectl/cmd/run.go:124 +msgid "" +"The port that this container exposes. If --expose is true, this is also the " +"port used by the service that is created." +msgstr "" +"La porta che questo contenitore espone. Se --expose è true, questa è anche la " +"porta utilizzata dal servizio creato." + +#: pkg/kubectl/cmd/run.go:134 +msgid "" +"The resource requirement limits for this container. For example, 'cpu=200m," +"memory=512Mi'. Note that server side components may assign limits depending " +"on the server configuration, such as limit ranges." +msgstr "" +"I limiti delle richieste di risorse per questo contenitore. Ad esempio, " +"'cpu=200m,memory=512Mi'. Si noti che i componenti lato server possono " +"assegnare i limiti a seconda della configurazione del server, ad esempio " +"intervalli di limiti." + +#: pkg/kubectl/cmd/run.go:133 +msgid "" +"The resource requirement requests for this container. For example, 'cpu=100m," +"memory=256Mi'. Note that server side components may assign requests " +"depending on the server configuration, such as limit ranges." +msgstr "" +"La risorsa necessita di richieste di requisiti per questo pod. Ad esempio, " +"'cpu = 100m, memoria = 256Mi'. Si noti che i componenti lato server possono " +"assegnare i requisiti a seconda della configurazione del server, ad esempio " +"intervalli di limiti." + +#: pkg/kubectl/cmd/run.go:131 +msgid "" +"The restart policy for this Pod. Legal values [Always, OnFailure, Never]. " +"If set to 'Always' a deployment is created, if set to 'OnFailure' a job is " +"created, if set to 'Never', a regular pod is created. For the latter two --" +"replicas must be 1. Default 'Always', for CronJobs ` + "`" + `Never` + "`" + `." +msgstr "" +"La politica di riavvio per questo Pod. Valori accettati [Always, OnFailure, " +"Never]. Se impostato su 'Always' viene creato un deployment, se impostato su " +"'OnFailure' viene creato un job, se impostato su 'Never', viene creato un " +"pod. Per questi ultimi due le - repliche devono essere 1. Predefinito " +"'Always', per CronJobs ` + "`" + `Never` + "`" + `." + +#: pkg/kubectl/cmd/create_secret.go:88 +msgid "The type of secret to create" +msgstr "Tipo di segreto da creare" + +#: pkg/kubectl/cmd/expose.go:102 +msgid "" +"Type for this service: ClusterIP, NodePort, or LoadBalancer. Default is " +"'ClusterIP'." +msgstr "" +"Digitare per questo servizio: ClusterIP, NodePort o LoadBalancer. " +"Ppredefinito è 'ClusterIP'." + +#: pkg/kubectl/cmd/rollout/rollout_undo.go:72 +msgid "Undo a previous rollout" +msgstr "Annulla un precedente rollout" + +#: pkg/kubectl/cmd/config/unset.go:48 +msgid "Unsets an individual value in a kubeconfig file" +msgstr "Annulla singolo valore in un file kubeconfig" + +#: pkg/kubectl/cmd/patch.go:96 +msgid "Update field(s) of a resource using strategic merge patch" +msgstr "Aggiornare campo/i risorsa utilizzando merge patch strategici" + +#: pkg/kubectl/cmd/set/set_image.go:95 +msgid "Update image of a pod template" +msgstr "Aggiorna immagine di un pod template" + +#: pkg/kubectl/cmd/set/set_resources.go:102 +msgid "Update resource requests/limits on objects with pod templates" +msgstr "Aggiorna richieste di risorse/limiti sugli oggetti con pod template" + +#: pkg/kubectl/cmd/annotate.go:116 +msgid "Update the annotations on a resource" +msgstr "Aggiorna annotazioni di risorsa" + +#: pkg/kubectl/cmd/label.go:114 +msgid "Update the labels on a resource" +msgstr "Aggiorna label di una risorsa" + +#: pkg/kubectl/cmd/taint.go:87 +msgid "Update the taints on one or more nodes" +msgstr "Aggiorna i taints su uno o più nodi" + +#: pkg/kubectl/cmd/create_secret.go:156 +msgid "Username for Docker registry authentication" +msgstr "Nome utente per l'autenticazione nel registro Docker" + +#: pkg/kubectl/cmd/apply_view_last_applied.go:64 +msgid "View latest last-applied-configuration annotations of a resource/object" +msgstr "" +"Visualizza ultime annotazioni dell'ultima configurazione applicata per " +"risorsa/oggetto" + +#: pkg/kubectl/cmd/rollout/rollout_history.go:52 +msgid "View rollout history" +msgstr "Visualizza la storia del rollout" + +#: pkg/kubectl/cmd/clusterinfo_dump.go:46 +msgid "" +"Where to output the files. If empty or '-' uses stdout, otherwise creates a " +"directory hierarchy in that directory" +msgstr "" +"Dove eseguire l'output dei file. Se vuota o '-' utilizza lo stdout, " +"altrimenti crea una gerarchia di directory in quella directory" + +#: pkg/kubectl/cmd/run_test.go:85 +msgid "dummy restart flag)" +msgstr "flag di riavvio finto)" + +#: pkg/kubectl/cmd/create_service.go:254 +msgid "external name of service" +msgstr "nome esterno del servizio" + +#: pkg/kubectl/cmd/cmd.go:227 +msgid "kubectl controls the Kubernetes cluster manager" +msgstr "Kubectl controlla il gestore cluster di Kubernetes" +`) + +func translationsKubectlIt_itLc_messagesK8sPoBytes() ([]byte, error) { + return _translationsKubectlIt_itLc_messagesK8sPo, nil +} + +func translationsKubectlIt_itLc_messagesK8sPo() (*asset, error) { + bytes, err := translationsKubectlIt_itLc_messagesK8sPoBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "translations/kubectl/it_IT/LC_MESSAGES/k8s.po", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _translationsKubectlJa_jpLc_messagesK8sMo = []byte("\xde\x12\x04\x95\x00\x00\x00\x00\x11\x00\x00\x00\x1c\x00\x00\x00\xa4\x00\x00\x00\x17\x00\x00\x00,\x01\x00\x00\x00\x00\x00\x00\x88\x01\x00\x008\x00\x00\x00\x89\x01\x00\x000\x00\x00\x00\xc2\x01\x00\x000\x00\x00\x00\xf3\x01\x00\x00\x1d\x00\x00\x00$\x02\x00\x00*\x00\x00\x00B\x02\x00\x00A\x00\x00\x00m\x02\x00\x00\x1c\x00\x00\x00\xaf\x02\x00\x00\x17\x00\x00\x00\xcc\x02\x00\x00\"\x00\x00\x00\xe4\x02\x00\x00\"\x00\x00\x00\a\x03\x00\x00\x1f\x00\x00\x00*\x03\x00\x00-\x00\x00\x00J\x03\x00\x00-\x00\x00\x00x\x03\x00\x00/\x00\x00\x00\xa6\x03\x00\x00$\x00\x00\x00\xd6\x03\x00\x00\xc5\x00\x00\x00\xfb\x03\x00\x00\xa5\x01\x00\x00\xc1\x04\x00\x00c\x00\x00\x00g\x06\x00\x00:\x00\x00\x00\xcb\x06\x00\x00=\x00\x00\x00\x06\a\x00\x007\x00\x00\x00D\a\x00\x00:\x00\x00\x00|\a\x00\x00b\x00\x00\x00\xb7\a\x00\x00-\x00\x00\x00\x1a\b\x00\x00%\x00\x00\x00H\b\x00\x007\x00\x00\x00n\b\x00\x00:\x00\x00\x00\xa6\b\x00\x004\x00\x00\x00\xe1\b\x00\x00:\x00\x00\x00\x16\t\x00\x00:\x00\x00\x00Q\t\x00\x00:\x00\x00\x00\x8c\t\x00\x003\x00\x00\x00\xc7\t\x00\x00\x1d\x01\x00\x00\xfb\t\x00\x00\x01\x00\x00\x00\n\x00\x00\x00\v\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11\x00\x00\x00\t\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\a\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\f\x00\x00\x00\x05\x00\x00\x00\r\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00Apply a configuration to a resource by filename or stdin\x00Delete the specified cluster from the kubeconfig\x00Delete the specified context from the kubeconfig\x00Describe one or many contexts\x00Display clusters defined in the kubeconfig\x00Display merged kubeconfig settings or a specified kubeconfig file\x00Displays the current-context\x00Modify kubeconfig files\x00Sets a cluster entry in kubeconfig\x00Sets a context entry in kubeconfig\x00Sets a user entry in kubeconfig\x00Sets an individual value in a kubeconfig file\x00Sets the current-context in a kubeconfig file\x00Unsets an individual value in a kubeconfig file\x00Update the annotations on a resource\x00watch is only supported on individual resources and resource collections - %d resources were found\x00watch is only supported on individual resources and resource collections - %d resources were found\x00Project-Id-Version: gettext-go-examples-hello\nReport-Msgid-Bugs-To: \nPOT-Creation-Date: 2013-12-12 20:03+0000\nPO-Revision-Date: 2017-06-01 17:54-0700\nLast-Translator: Giri Kuncoro \nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\nX-Generator: Poedit 2.0.1\nX-Poedit-SourceCharset: UTF-8\nLanguage-Team: \nPlural-Forms: nplurals=2; plural=(n > 1);\nLanguage: ja\n\x00\u30d5\u30a1\u30a4\u30eb\u540d\u3092\u6307\u5b9a\u307e\u305f\u306f\u6a19\u6e96\u5165\u529b\u7d4c\u7531\u3067\u30ea\u30bd\u30fc\u30b9\u306b\u30b3\u30f3\u30d5\u30a3\u30b0\u3092\u9069\u7528\u3059\u308b\x00kubeconfig\u304b\u3089\u6307\u5b9a\u3057\u305f\u30af\u30e9\u30b9\u30bf\u30fc\u3092\u524a\u9664\u3059\u308b\x00kubeconfig\u304b\u3089\u6307\u5b9a\u3057\u305f\u30b3\u30f3\u30c6\u30ad\u30b9\u30c8\u3092\u524a\u9664\u3059\u308b\x001\u3064\u307e\u305f\u306f\u8907\u6570\u306e\u30b3\u30f3\u30c6\u30ad\u30b9\u30c8\u3092\u8a18\u8ff0\u3059\u308b\x00kubeconfig\u3067\u5b9a\u7fa9\u3055\u308c\u305f\u30af\u30e9\u30b9\u30bf\u30fc\u3092\u8868\u793a\u3059\u308b\x00\u30de\u30fc\u30b8\u3055\u308c\u305fkubeconfig\u306e\u8a2d\u5b9a\u307e\u305f\u306f\u6307\u5b9a\u3055\u308c\u305fkubeconfig\u30d5\u30a1\u30a4\u30eb\u3092\u8868\u793a\u3059\u308b\x00\u30ab\u30ec\u30f3\u30c8\u30b3\u30f3\u30c6\u30ad\u30b9\u30c8\u3092\u8868\u793a\u3059\u308b\x00kubeconfig\u30d5\u30a1\u30a4\u30eb\u3092\u5909\u66f4\u3059\u308b\x00kubeconfig\u306b\u30af\u30e9\u30b9\u30bf\u30fc\u30a8\u30f3\u30c8\u30ea\u3092\u8a2d\u5b9a\u3059\u308b\x00kubeconfig\u306b\u30b3\u30f3\u30c6\u30ad\u30b9\u30c8\u30a8\u30f3\u30c8\u30ea\u3092\u8a2d\u5b9a\u3059\u308b\x00kubeconfig\u306b\u30e6\u30fc\u30b6\u30fc\u30a8\u30f3\u30c8\u30ea\u3092\u8a2d\u5b9a\u3059\u308b\x00kubeconfig\u30d5\u30a1\u30a4\u30eb\u5185\u306e\u5909\u6570\u3092\u500b\u5225\u306b\u8a2d\u5b9a\u3059\u308b\x00kubeconfig\u306b\u30ab\u30ec\u30f3\u30c8\u30b3\u30f3\u30c6\u30ad\u30b9\u30c8\u3092\u8a2d\u5b9a\u3059\u308b\x00kubeconfig\u30d5\u30a1\u30a4\u30eb\u304b\u3089\u5909\u6570\u3092\u500b\u5225\u306b\u524a\u9664\u3059\u308b\x00\u30ea\u30bd\u30fc\u30b9\u306e\u30a2\u30ce\u30c6\u30fc\u30b7\u30e7\u30f3\u3092\u66f4\u65b0\u3059\u308b\x00watch\u306f\u5358\u4e00\u30ea\u30bd\u30fc\u30b9\u53ca\u3073\u30ea\u30bd\u30fc\u30b9\u30b3\u30ec\u30af\u30b7\u30e7\u30f3\u306e\u307f\u30b5\u30dd\u30fc\u30c8\u3057\u3066\u3044\u307e\u3059 - %d\u500b\u306e\u30ea\u30bd\u30fc\u30b9\u304c\u898b\u3064\u304b\u308a\u307e\u3057\u305f\x00watch\u306f\u5358\u4e00\u30ea\u30bd\u30fc\u30b9\u53ca\u3073\u30ea\u30bd\u30fc\u30b9\u30b3\u30ec\u30af\u30b7\u30e7\u30f3\u306e\u307f\u30b5\u30dd\u30fc\u30c8\u3057\u3066\u3044\u307e\u3059 - %d\u500b\u306e\u30ea\u30bd\u30fc\u30b9\u304c\u898b\u3064\u304b\u308a\u307e\u3057\u305f\x00") func translationsKubectlJa_jpLc_messagesK8sMoBytes() ([]byte, error) { @@ -9980,6 +12898,8 @@ var _bindata = map[string]func() (*asset, error){ "translations/kubectl/en_US/LC_MESSAGES/k8s.po": translationsKubectlEn_usLc_messagesK8sPo, "translations/kubectl/fr_FR/LC_MESSAGES/k8s.mo": translationsKubectlFr_frLc_messagesK8sMo, "translations/kubectl/fr_FR/LC_MESSAGES/k8s.po": translationsKubectlFr_frLc_messagesK8sPo, + "translations/kubectl/it_IT/LC_MESSAGES/k8s.mo": translationsKubectlIt_itLc_messagesK8sMo, + "translations/kubectl/it_IT/LC_MESSAGES/k8s.po": translationsKubectlIt_itLc_messagesK8sPo, "translations/kubectl/ja_JP/LC_MESSAGES/k8s.mo": translationsKubectlJa_jpLc_messagesK8sMo, "translations/kubectl/ja_JP/LC_MESSAGES/k8s.po": translationsKubectlJa_jpLc_messagesK8sPo, "translations/kubectl/template.pot": translationsKubectlTemplatePot, @@ -10056,6 +12976,12 @@ var _bintree = &bintree{nil, map[string]*bintree{ "k8s.po": {translationsKubectlFr_frLc_messagesK8sPo, map[string]*bintree{}}, }}, }}, + "it_IT": {nil, map[string]*bintree{ + "LC_MESSAGES": {nil, map[string]*bintree{ + "k8s.mo": {translationsKubectlIt_itLc_messagesK8sMo, map[string]*bintree{}}, + "k8s.po": {translationsKubectlIt_itLc_messagesK8sPo, map[string]*bintree{}}, + }}, + }}, "ja_JP": {nil, map[string]*bintree{ "LC_MESSAGES": {nil, map[string]*bintree{ "k8s.mo": {translationsKubectlJa_jpLc_messagesK8sMo, map[string]*bintree{}}, diff --git a/pkg/generated/openapi/BUILD b/pkg/generated/openapi/BUILD index bc59dcd9045a2..86cd0e7c36205 100644 --- a/pkg/generated/openapi/BUILD +++ b/pkg/generated/openapi/BUILD @@ -55,8 +55,8 @@ openapi_library( "k8s.io/apiserver/pkg/apis/audit/v1beta1", "k8s.io/apiserver/pkg/apis/example/v1", "k8s.io/client-go/pkg/version", + "k8s.io/code-generator/test/apis/testgroup/v1", "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1", - "k8s.io/kube-gen/test/apis/testgroup/v1", "k8s.io/metrics/pkg/apis/custom_metrics/v1alpha1", "k8s.io/metrics/pkg/apis/metrics/v1alpha1", ], diff --git a/pkg/generated/openapi/def.bzl b/pkg/generated/openapi/def.bzl index 3b1f1164bee1f..fa530f2b74549 100644 --- a/pkg/generated/openapi/def.bzl +++ b/pkg/generated/openapi/def.bzl @@ -17,7 +17,7 @@ def openapi_library(name, tags, srcs, openapi_targets=[], vendor_targets=[]): srcs = srcs + ["//hack/boilerplate:boilerplate.go.txt"], outs = ["zz_generated.openapi.go"], cmd = " ".join([ - "$(location //vendor/k8s.io/kube-gen/cmd/openapi-gen)", + "$(location //vendor/k8s.io/code-generator/cmd/openapi-gen)", "--v 1", "--logtostderr", "--go-header-file $(location //hack/boilerplate:boilerplate.go.txt)", @@ -27,5 +27,5 @@ def openapi_library(name, tags, srcs, openapi_targets=[], vendor_targets=[]): "&& cp pkg/generated/openapi/zz_generated.openapi.go $(location :zz_generated.openapi.go)", ]), go_deps = deps, - tools = ["//vendor/k8s.io/kube-gen/cmd/openapi-gen"], + tools = ["//vendor/k8s.io/code-generator/cmd/openapi-gen"], ) diff --git a/pkg/kubeapiserver/admission/BUILD b/pkg/kubeapiserver/admission/BUILD index c1a833dcd13ae..95a3eb4c30655 100644 --- a/pkg/kubeapiserver/admission/BUILD +++ b/pkg/kubeapiserver/admission/BUILD @@ -42,6 +42,7 @@ filegroup( srcs = [ ":package-srcs", "//pkg/kubeapiserver/admission/configuration:all-srcs", + "//pkg/kubeapiserver/admission/util:all-srcs", ], tags = ["automanaged"], ) diff --git a/pkg/kubeapiserver/admission/util/BUILD b/pkg/kubeapiserver/admission/util/BUILD new file mode 100644 index 0000000000000..401e3a39a58b5 --- /dev/null +++ b/pkg/kubeapiserver/admission/util/BUILD @@ -0,0 +1,31 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["initializer.go"], + tags = ["automanaged"], + deps = [ + "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/kubeapiserver/admission/util/initializer.go b/pkg/kubeapiserver/admission/util/initializer.go new file mode 100644 index 0000000000000..165c4ee9ba86d --- /dev/null +++ b/pkg/kubeapiserver/admission/util/initializer.go @@ -0,0 +1,56 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apiserver/pkg/admission" +) + +// IsUpdatingInitializedObject returns if the operation is trying to update an +// already initialized object. +func IsUpdatingInitializedObject(a admission.Attributes) (bool, error) { + if a.GetOperation() != admission.Update { + return false, nil + } + oldObj := a.GetOldObject() + accessor, err := meta.Accessor(oldObj) + if err != nil { + return false, err + } + if accessor.GetInitializers() == nil { + return true, nil + } + return false, nil +} + +// IsUpdatingUninitializedObject returns if the operation is trying to update an +// object that is not initialized yet. +func IsUpdatingUninitializedObject(a admission.Attributes) (bool, error) { + if a.GetOperation() != admission.Update { + return false, nil + } + oldObj := a.GetOldObject() + accessor, err := meta.Accessor(oldObj) + if err != nil { + return false, err + } + if accessor.GetInitializers() == nil { + return false, nil + } + return true, nil +} diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index cc21d4cacb244..6bdc013f45617 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -51,7 +51,7 @@ type AnonymousAuthenticationOptions struct { } type BootstrapTokenAuthenticationOptions struct { - Allow bool + Enable bool } type KeystoneAuthenticationOptions struct { @@ -94,7 +94,7 @@ func NewBuiltInAuthenticationOptions() *BuiltInAuthenticationOptions { func (s *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions { return s. - WithAnyonymous(). + WithAnonymous(). WithBootstrapToken(). WithClientCert(). WithKeystone(). @@ -106,7 +106,7 @@ func (s *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions { WithWebHook() } -func (s *BuiltInAuthenticationOptions) WithAnyonymous() *BuiltInAuthenticationOptions { +func (s *BuiltInAuthenticationOptions) WithAnonymous() *BuiltInAuthenticationOptions { s.Anonymous = &AnonymousAuthenticationOptions{Allow: true} return s } @@ -178,7 +178,11 @@ func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) { } if s.BootstrapToken != nil { - fs.BoolVar(&s.BootstrapToken.Allow, "experimental-bootstrap-token-auth", s.BootstrapToken.Allow, ""+ + fs.BoolVar(&s.BootstrapToken.Enable, "experimental-bootstrap-token-auth", s.BootstrapToken.Enable, ""+ + "Deprecated (use --enable-bootstrap-token-auth).") + fs.MarkDeprecated("experimental-bootstrap-token-auth", "use --enable-bootstrap-token-auth instead.") + + fs.BoolVar(&s.BootstrapToken.Enable, "enable-bootstrap-token-auth", s.BootstrapToken.Enable, ""+ "Enable to allow secrets of type 'bootstrap.kubernetes.io/token' in the 'kube-system' "+ "namespace to be used for TLS bootstrapping authentication.") } @@ -266,7 +270,7 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() authenticator.Au } if s.BootstrapToken != nil { - ret.BootstrapToken = s.BootstrapToken.Allow + ret.BootstrapToken = s.BootstrapToken.Enable } if s.ClientCert != nil { diff --git a/pkg/kubeapiserver/server/insecure_handler.go b/pkg/kubeapiserver/server/insecure_handler.go index 24127f546a896..8186a0ac7abc8 100644 --- a/pkg/kubeapiserver/server/insecure_handler.go +++ b/pkg/kubeapiserver/server/insecure_handler.go @@ -46,7 +46,7 @@ func BuildInsecureHandlerChain(apiHandler http.Handler, c *server.Config) http.H handler = genericapifilters.WithAuthentication(handler, c.RequestContextMapper, insecureSuperuser{}, nil) handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true") handler = genericfilters.WithPanicRecovery(handler) - handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.RequestContextMapper, c.LongRunningFunc) + handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.RequestContextMapper, c.LongRunningFunc, c.RequestTimeout) handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.RequestContextMapper, c.LongRunningFunc) handler = genericapifilters.WithRequestInfo(handler, server.NewRequestInfoResolver(c), c.RequestContextMapper) handler = apirequest.WithRequestContext(handler, c.RequestContextMapper) diff --git a/pkg/kubectl/BUILD b/pkg/kubectl/BUILD index 35f8eff04c854..48ab26973d6a6 100644 --- a/pkg/kubectl/BUILD +++ b/pkg/kubectl/BUILD @@ -129,6 +129,7 @@ go_library( "//pkg/client/unversioned:go_default_library", "//pkg/controller/daemon:go_default_library", "//pkg/controller/deployment/util:go_default_library", + "//pkg/controller/statefulset:go_default_library", "//pkg/credentialprovider:go_default_library", "//pkg/kubectl/resource:go_default_library", "//pkg/kubectl/util:go_default_library", @@ -184,6 +185,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//pkg/kubectl/apps:all-srcs", "//pkg/kubectl/cmd:all-srcs", "//pkg/kubectl/metricsutil:all-srcs", "//pkg/kubectl/plugins:all-srcs", diff --git a/pkg/kubectl/apps/BUILD b/pkg/kubectl/apps/BUILD new file mode 100644 index 0000000000000..0d9d988760ffd --- /dev/null +++ b/pkg/kubectl/apps/BUILD @@ -0,0 +1,42 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_library( + name = "go_default_library", + srcs = ["kind_visitor.go"], + tags = ["automanaged"], + deps = ["//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) + +go_test( + name = "go_default_xtest", + srcs = [ + "apps_suite_test.go", + "kind_visitor_test.go", + ], + deps = [ + ":go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", + ], +) diff --git a/pkg/kubectl/apps/apps_suite_test.go b/pkg/kubectl/apps/apps_suite_test.go new file mode 100644 index 0000000000000..b1bc1310ea036 --- /dev/null +++ b/pkg/kubectl/apps/apps_suite_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apps_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestApps(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Apps Suite") +} diff --git a/pkg/kubectl/apps/kind_visitor.go b/pkg/kubectl/apps/kind_visitor.go new file mode 100644 index 0000000000000..0170883e1e586 --- /dev/null +++ b/pkg/kubectl/apps/kind_visitor.go @@ -0,0 +1,95 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apps + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// KindVisitor is used with GroupKindElement to call a particular function depending on the +// Kind of a schema.GroupKind +type KindVisitor interface { + VisitDaemonSet(kind GroupKindElement) + VisitDeployment(kind GroupKindElement) + VisitJob(kind GroupKindElement) + VisitPod(kind GroupKindElement) + VisitReplicaSet(kind GroupKindElement) + VisitReplicationController(kind GroupKindElement) + VisitStatefulSet(kind GroupKindElement) +} + +// GroupKindElement defines a Kubernetes API group elem +type GroupKindElement schema.GroupKind + +// Accept calls the Visit method on visitor that corresponds to elem's Kind +func (elem GroupKindElement) Accept(visitor KindVisitor) error { + if elem.GroupMatch("apps", "extensions") && elem.Kind == "DaemonSet" { + visitor.VisitDaemonSet(elem) + return nil + } + if elem.GroupMatch("apps", "extensions") && elem.Kind == "Deployment" { + visitor.VisitDeployment(elem) + return nil + } + if elem.GroupMatch("batch") && elem.Kind == "Job" { + visitor.VisitJob(elem) + return nil + } + if elem.GroupMatch("", "core") && elem.Kind == "Pod" { + visitor.VisitPod(elem) + return nil + } + if elem.GroupMatch("extensions") && elem.Kind == "ReplicaSet" { + visitor.VisitReplicaSet(elem) + return nil + } + if elem.GroupMatch("", "core") && elem.Kind == "ReplicationController" { + visitor.VisitReplicationController(elem) + return nil + } + if elem.GroupMatch("apps") && elem.Kind == "StatefulSet" { + visitor.VisitStatefulSet(elem) + return nil + } + return fmt.Errorf("no visitor method exists for %v", elem) +} + +// GroupMatch returns true if and only if elem's group matches one +// of the group arguments +func (elem GroupKindElement) GroupMatch(groups ...string) bool { + for _, g := range groups { + if elem.Group == g { + return true + } + } + return false +} + +// NoOpKindVisitor implements KindVisitor with no-op functions. +type NoOpKindVisitor struct{} + +var _ KindVisitor = &NoOpKindVisitor{} + +func (*NoOpKindVisitor) VisitDaemonSet(kind GroupKindElement) {} +func (*NoOpKindVisitor) VisitDeployment(kind GroupKindElement) {} +func (*NoOpKindVisitor) VisitJob(kind GroupKindElement) {} +func (*NoOpKindVisitor) VisitPod(kind GroupKindElement) {} +func (*NoOpKindVisitor) VisitReplicaSet(kind GroupKindElement) {} +func (*NoOpKindVisitor) VisitReplicationController(kind GroupKindElement) {} +func (*NoOpKindVisitor) VisitStatefulSet(kind GroupKindElement) {} diff --git a/pkg/kubectl/apps/kind_visitor_test.go b/pkg/kubectl/apps/kind_visitor_test.go new file mode 100644 index 0000000000000..c5c2d1a631c50 --- /dev/null +++ b/pkg/kubectl/apps/kind_visitor_test.go @@ -0,0 +1,173 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apps_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/kubernetes/pkg/kubectl/apps" +) + +var _ = Describe("When KindVisitor accepts a GroupKind", func() { + + var visitor *TestKindVisitor + + BeforeEach(func() { + visitor = &TestKindVisitor{map[string]int{}} + }) + + It("should Visit DaemonSet iff the Kind is a DaemonSet", func() { + kind := apps.GroupKindElement{ + Kind: "DaemonSet", + Group: "apps", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "DaemonSet": 1, + })) + + kind = apps.GroupKindElement{ + Kind: "DaemonSet", + Group: "extensions", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "DaemonSet": 2, + })) + }) + + It("should Visit Deployment iff the Kind is a Deployment", func() { + kind := apps.GroupKindElement{ + Kind: "Deployment", + Group: "apps", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "Deployment": 1, + })) + + kind = apps.GroupKindElement{ + Kind: "Deployment", + Group: "extensions", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "Deployment": 2, + })) + }) + + It("should Visit Job iff the Kind is a Job", func() { + kind := apps.GroupKindElement{ + Kind: "Job", + Group: "batch", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "Job": 1, + })) + + }) + + It("should Visit Pod iff the Kind is a Pod", func() { + kind := apps.GroupKindElement{ + Kind: "Pod", + Group: "", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "Pod": 1, + })) + + kind = apps.GroupKindElement{ + Kind: "Pod", + Group: "core", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "Pod": 2, + })) + }) + + It("should Visit ReplicationController iff the Kind is a ReplicationController", func() { + kind := apps.GroupKindElement{ + Kind: "ReplicationController", + Group: "", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "ReplicationController": 1, + })) + + kind = apps.GroupKindElement{ + Kind: "ReplicationController", + Group: "core", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "ReplicationController": 2, + })) + }) + + It("should Visit ReplicaSet iff the Kind is a ReplicaSet", func() { + kind := apps.GroupKindElement{ + Kind: "ReplicaSet", + Group: "extensions", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "ReplicaSet": 1, + })) + }) + + It("should Visit StatefulSet iff the Kind is a StatefulSet", func() { + kind := apps.GroupKindElement{ + Kind: "StatefulSet", + Group: "apps", + } + Expect(kind.Accept(visitor)).ShouldNot(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{ + "StatefulSet": 1, + })) + }) + + It("should give an error if the Kind is unknown", func() { + kind := apps.GroupKindElement{ + Kind: "Unknown", + Group: "apps", + } + Expect(kind.Accept(visitor)).Should(HaveOccurred()) + Expect(visitor.visits).To(Equal(map[string]int{})) + }) +}) + +// TestKindVisitor increments a value each time a Visit method was called +type TestKindVisitor struct { + visits map[string]int +} + +var _ apps.KindVisitor = &TestKindVisitor{} + +func (t *TestKindVisitor) Visit(kind apps.GroupKindElement) { t.visits[kind.Kind] += 1 } + +func (t *TestKindVisitor) VisitDaemonSet(kind apps.GroupKindElement) { t.Visit(kind) } +func (t *TestKindVisitor) VisitDeployment(kind apps.GroupKindElement) { t.Visit(kind) } +func (t *TestKindVisitor) VisitJob(kind apps.GroupKindElement) { t.Visit(kind) } +func (t *TestKindVisitor) VisitPod(kind apps.GroupKindElement) { t.Visit(kind) } +func (t *TestKindVisitor) VisitReplicaSet(kind apps.GroupKindElement) { t.Visit(kind) } +func (t *TestKindVisitor) VisitReplicationController(kind apps.GroupKindElement) { t.Visit(kind) } +func (t *TestKindVisitor) VisitStatefulSet(kind apps.GroupKindElement) { t.Visit(kind) } diff --git a/pkg/kubectl/cmd/apply.go b/pkg/kubectl/cmd/apply.go index 9ab7e60e9f492..6b6172ccd72b7 100644 --- a/pkg/kubectl/cmd/apply.go +++ b/pkg/kubectl/cmd/apply.go @@ -187,7 +187,7 @@ func parsePruneResources(mapper meta.RESTMapper, gvks []string) ([]pruneResource } func RunApply(f cmdutil.Factory, cmd *cobra.Command, out, errOut io.Writer, options *ApplyOptions) error { - schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) + schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagBool(cmd, "openapi-validation"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) if err != nil { return err } diff --git a/pkg/kubectl/cmd/cmd.go b/pkg/kubectl/cmd/cmd.go index 68f16591a4537..c0e4f7f5efeba 100644 --- a/pkg/kubectl/cmd/cmd.go +++ b/pkg/kubectl/cmd/cmd.go @@ -285,7 +285,6 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob NewCmdCreate(f, out, err), NewCmdExposeService(f, out), NewCmdRun(f, in, out, err), - deprecatedAlias("run-container", NewCmdRun(f, in, out, err)), set.NewCmdSet(f, out, err), }, }, @@ -303,9 +302,7 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob Commands: []*cobra.Command{ rollout.NewCmdRollout(f, out, err), NewCmdRollingUpdate(f, out), - deprecatedAlias("rollingupdate", NewCmdRollingUpdate(f, out)), NewCmdScale(f, out), - deprecatedAlias("resize", NewCmdScale(f, out)), NewCmdAutoscale(f, out), }, }, @@ -314,7 +311,6 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob Commands: []*cobra.Command{ NewCmdCertificate(f, out), NewCmdClusterInfo(f, out), - deprecatedAlias("clusterinfo", NewCmdClusterInfo(f, out)), NewCmdTop(f, out, err), NewCmdCordon(f, out), NewCmdUncordon(f, out), @@ -341,7 +337,6 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob NewCmdApply("kubectl", f, out, err), NewCmdPatch(f, out), NewCmdReplace(f, out), - deprecatedAlias("update", NewCmdReplace(f, out)), NewCmdConvert(f, out), }, }, @@ -383,7 +378,6 @@ func NewKubectlCommand(f cmdutil.Factory, in io.Reader, out, err io.Writer) *cob cmds.AddCommand(NewCmdPlugin(f, in, out, err)) cmds.AddCommand(NewCmdVersion(f, out)) cmds.AddCommand(NewCmdApiVersions(f, out)) - cmds.AddCommand(deprecatedAlias("apiversions", NewCmdApiVersions(f, out))) cmds.AddCommand(NewCmdOptions(out)) return cmds diff --git a/pkg/kubectl/cmd/completion.go b/pkg/kubectl/cmd/completion.go index 20414218c7213..d14ca66329b49 100644 --- a/pkg/kubectl/cmd/completion.go +++ b/pkg/kubectl/cmd/completion.go @@ -50,44 +50,41 @@ var ( completion of kubectl commands. This can be done by sourcing it from the .bash_profile. - Note: this requires the bash-completion framework, which is not installed - by default on Mac. This can be installed by using homebrew: - - $ brew install bash-completion - - Once installed, bash_completion must be evaluated. This can be done by adding the - following line to the .bash_profile - - $ source $(brew --prefix)/etc/bash_completion + Detailed instructions on how to do this are available here: + https://kubernetes.io/docs/tasks/tools/install-kubectl/#enabling-shell-autocompletion Note for zsh users: [1] zsh completions are only supported in versions of zsh >= 5.2`)) completion_example = templates.Examples(i18n.T(` - # Install bash completion on a Mac using homebrew - brew install bash-completion - printf " -# Bash completion support -source $(brew --prefix)/etc/bash_completion -" >> $HOME/.bash_profile - source $HOME/.bash_profile - - # Load the kubectl completion code for bash into the current shell - source <(kubectl completion bash) - - # Write bash completion code to a file and source if from .bash_profile - kubectl completion bash > ~/.kube/completion.bash.inc - printf " -# Kubectl shell completion -source '$HOME/.kube/completion.bash.inc' -" >> $HOME/.bash_profile - source $HOME/.bash_profile + # Installing bash completion on macOS using homebrew + ## If running Bash 3.2 included with macOS + brew install bash-completion + ## or, if running Bash 4.1+ + brew install bash-completion@2 + ## If kubectl is installed via homebrew, this should start working immediately. + ## If you've installed via other means, you may need add the completion to your completion directory + kubectl completion bash > $(brew --prefix)/etc/bash_completion.d/kubectl + + + # Installing bash completion on Linux + ## Load the kubectl completion code for bash into the current shell + source <(kubectl completion bash) + ## Write bash completion code to a file and source if from .bash_profile + kubectl completion bash > ~/.kube/completion.bash.inc + printf " + # Kubectl shell completion + source '$HOME/.kube/completion.bash.inc' + " >> $HOME/.bash_profile + source $HOME/.bash_profile # Load the kubectl completion code for zsh[1] into the current shell - source <(kubectl completion zsh)`)) + source <(kubectl completion zsh) + # Set the kubectl completion code for zsh[1] to autoload on startup + kubectl completion zsh > "${fpath[1]}/_kubectl"`)) ) var ( - completion_shells = map[string]func(out io.Writer, cmd *cobra.Command) error{ + completion_shells = map[string]func(out io.Writer, boilerPlate string, cmd *cobra.Command) error{ "bash": runCompletionBash, "zsh": runCompletionZsh, } @@ -126,20 +123,32 @@ func RunCompletion(out io.Writer, boilerPlate string, cmd *cobra.Command, args [ return cmdutil.UsageErrorf(cmd, "Unsupported shell type %q.", args[0]) } + return run(out, boilerPlate, cmd.Parent()) +} + +func runCompletionBash(out io.Writer, boilerPlate string, kubectl *cobra.Command) error { if len(boilerPlate) == 0 { boilerPlate = defaultBoilerPlate } if _, err := out.Write([]byte(boilerPlate)); err != nil { return err } - return run(out, cmd.Parent()) -} -func runCompletionBash(out io.Writer, kubectl *cobra.Command) error { return kubectl.GenBashCompletion(out) } -func runCompletionZsh(out io.Writer, kubectl *cobra.Command) error { +func runCompletionZsh(out io.Writer, boilerPlate string, kubectl *cobra.Command) error { + zsh_head := "#compdef kubectl\n" + + out.Write([]byte(zsh_head)) + + if len(boilerPlate) == 0 { + boilerPlate = defaultBoilerPlate + } + if _, err := out.Write([]byte(boilerPlate)); err != nil { + return err + } + zsh_initialization := ` __kubectl_bash_source() { alias shopt=':' @@ -293,6 +302,7 @@ BASH_COMPLETION_EOF } __kubectl_bash_source <(__kubectl_convert_bash_to_zsh) +_complete kubectl 2>/dev/null ` out.Write([]byte(zsh_tail)) return nil diff --git a/pkg/kubectl/cmd/convert.go b/pkg/kubectl/cmd/convert.go index f616dfb4cb1d5..b450ad14279cf 100644 --- a/pkg/kubectl/cmd/convert.go +++ b/pkg/kubectl/cmd/convert.go @@ -128,7 +128,7 @@ func (o *ConvertOptions) Complete(f cmdutil.Factory, out io.Writer, cmd *cobra.C // build the builder o.builder = f.NewBuilder(!o.local) if !o.local { - schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) + schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagBool(cmd, "openapi-validation"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) if err != nil { return err } diff --git a/pkg/kubectl/cmd/create.go b/pkg/kubectl/cmd/create.go index 5cf6275fc0791..7447d50c3a9d3 100644 --- a/pkg/kubectl/cmd/create.go +++ b/pkg/kubectl/cmd/create.go @@ -115,7 +115,7 @@ func RunCreate(f cmdutil.Factory, cmd *cobra.Command, out, errOut io.Writer, opt if options.EditBeforeCreate { return RunEditOnCreate(f, out, errOut, cmd, &options.FilenameOptions) } - schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) + schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagBool(cmd, "openapi-validation"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) if err != nil { return err } diff --git a/pkg/kubectl/cmd/create_clusterrole.go b/pkg/kubectl/cmd/create_clusterrole.go index 1aa5891ff6fd5..fd77ef0bc4d55 100644 --- a/pkg/kubectl/cmd/create_clusterrole.go +++ b/pkg/kubectl/cmd/create_clusterrole.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "io" + "strings" "github.com/spf13/cobra" @@ -133,6 +134,20 @@ func (c *CreateClusterRoleOptions) Validate() error { return fmt.Errorf("invalid verb: '%s' for nonResourceURL", v) } } + + for _, nonResourceURL := range c.NonResourceURLs { + if nonResourceURL == "*" { + continue + } + + if nonResourceURL == "" || !strings.HasPrefix(nonResourceURL, "/") { + return fmt.Errorf("nonResourceURL should start with /") + } + + if strings.ContainsRune(nonResourceURL[:len(nonResourceURL)-1], '*') { + return fmt.Errorf("nonResourceURL only supports wildcard matches when '*' is at the end") + } + } } return nil diff --git a/pkg/kubectl/cmd/create_clusterrole_test.go b/pkg/kubectl/cmd/create_clusterrole_test.go index 4d877307c8b9f..e930c6eacf0e0 100644 --- a/pkg/kubectl/cmd/create_clusterrole_test.go +++ b/pkg/kubectl/cmd/create_clusterrole_test.go @@ -375,6 +375,46 @@ func TestClusterRoleValidate(t *testing.T) { }, expectErr: false, }, + "test-invalid-empty-non-resource-url": { + clusterRoleOptions: &CreateClusterRoleOptions{ + CreateRoleOptions: &CreateRoleOptions{ + Name: "my-clusterrole", + Verbs: []string{"create"}, + }, + NonResourceURLs: []string{""}, + }, + expectErr: true, + }, + "test-invalid-non-resource-url": { + clusterRoleOptions: &CreateClusterRoleOptions{ + CreateRoleOptions: &CreateRoleOptions{ + Name: "my-clusterrole", + Verbs: []string{"create"}, + }, + NonResourceURLs: []string{"logs"}, + }, + expectErr: true, + }, + "test-invalid-non-resource-url-with-*": { + clusterRoleOptions: &CreateClusterRoleOptions{ + CreateRoleOptions: &CreateRoleOptions{ + Name: "my-clusterrole", + Verbs: []string{"create"}, + }, + NonResourceURLs: []string{"/logs/*/"}, + }, + expectErr: true, + }, + "test-invalid-non-resource-url-with-multiple-*": { + clusterRoleOptions: &CreateClusterRoleOptions{ + CreateRoleOptions: &CreateRoleOptions{ + Name: "my-clusterrole", + Verbs: []string{"create"}, + }, + NonResourceURLs: []string{"/logs*/*"}, + }, + expectErr: true, + }, "test-invalid-verb-for-non-resource-url": { clusterRoleOptions: &CreateClusterRoleOptions{ CreateRoleOptions: &CreateRoleOptions{ @@ -397,7 +437,7 @@ func TestClusterRoleValidate(t *testing.T) { }, }, }, - NonResourceURLs: []string{"/logs/"}, + NonResourceURLs: []string{"/logs/", "/logs/*"}, }, expectErr: false, }, diff --git a/pkg/kubectl/cmd/explain.go b/pkg/kubectl/cmd/explain.go index 982b0e8d34b87..5f10c14a5b527 100644 --- a/pkg/kubectl/cmd/explain.go +++ b/pkg/kubectl/cmd/explain.go @@ -65,7 +65,7 @@ func NewCmdExplain(f cmdutil.Factory, out, cmdErr io.Writer) *cobra.Command { // RunExplain executes the appropriate steps to print a model's documentation func RunExplain(f cmdutil.Factory, out, cmdErr io.Writer, cmd *cobra.Command, args []string) error { if len(args) == 0 { - fmt.Fprint(cmdErr, "You must specify the type of resource to explain. ", validResources) + fmt.Fprintf(cmdErr, "You must specify the type of resource to explain. %s\n", validResources) return cmdutil.UsageErrorf(cmd, "Required resource not specified.") } if len(args) > 1 { diff --git a/pkg/kubectl/cmd/logs.go b/pkg/kubectl/cmd/logs.go index 8c1ee97750cd1..6881f1e65e8f1 100644 --- a/pkg/kubectl/cmd/logs.go +++ b/pkg/kubectl/cmd/logs.go @@ -165,7 +165,8 @@ func (o *LogsOptions) Complete(f cmdutil.Factory, out io.Writer, cmd *cobra.Comm if limit := cmdutil.GetFlagInt64(cmd, "limit-bytes"); limit != 0 { logOptions.LimitBytes = &limit } - if tail := cmdutil.GetFlagInt64(cmd, "tail"); tail != -1 { + tail := cmdutil.GetFlagInt64(cmd, "tail") + if tail != -1 { logOptions.TailLines = &tail } if sinceSeconds := cmdutil.GetFlagDuration(cmd, "since"); sinceSeconds != 0 { @@ -185,7 +186,7 @@ func (o *LogsOptions) Complete(f cmdutil.Factory, out io.Writer, cmd *cobra.Comm if logOptions.Follow { return cmdutil.UsageErrorf(cmd, "only one of follow (-f) or selector (-l) is allowed") } - if logOptions.TailLines == nil { + if logOptions.TailLines == nil && tail != -1 { logOptions.TailLines = &selectorTail } } diff --git a/pkg/kubectl/cmd/portforward.go b/pkg/kubectl/cmd/portforward.go index 00bdd9cdf0463..b9ef5073662d5 100644 --- a/pkg/kubectl/cmd/portforward.go +++ b/pkg/kubectl/cmd/portforward.go @@ -59,10 +59,7 @@ var ( kubectl port-forward mypod 8888:5000 # Listen on a random port locally, forwarding to 5000 in the pod - kubectl port-forward mypod :5000 - - # Listen on a random port locally, forwarding to 5000 in the pod - kubectl port-forward mypod 0:5000`)) + kubectl port-forward mypod :5000`)) ) func NewCmdPortForward(f cmdutil.Factory, cmdOut, cmdErr io.Writer) *cobra.Command { diff --git a/pkg/kubectl/cmd/replace.go b/pkg/kubectl/cmd/replace.go index ccdd88f796d27..bdb82b4612733 100644 --- a/pkg/kubectl/cmd/replace.go +++ b/pkg/kubectl/cmd/replace.go @@ -92,7 +92,7 @@ func NewCmdReplace(f cmdutil.Factory, out io.Writer) *cobra.Command { } func RunReplace(f cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string, options *resource.FilenameOptions) error { - schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) + schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagBool(cmd, "openapi-validation"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) if err != nil { return err } @@ -171,7 +171,7 @@ func RunReplace(f cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []str } func forceReplace(f cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string, shortOutput bool, options *resource.FilenameOptions) error { - schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) + schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagBool(cmd, "openapi-validation"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) if err != nil { return err } diff --git a/pkg/kubectl/cmd/rollingupdate.go b/pkg/kubectl/cmd/rollingupdate.go index 74e04ffc923b9..be6c4d6e03b95 100644 --- a/pkg/kubectl/cmd/rollingupdate.go +++ b/pkg/kubectl/cmd/rollingupdate.go @@ -194,7 +194,7 @@ func RunRollingUpdate(f cmdutil.Factory, out io.Writer, cmd *cobra.Command, args mapper, typer := f.Object() if len(filename) != 0 { - schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) + schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagBool(cmd, "openapi-validation"), cmdutil.GetFlagString(cmd, "schema-cache-dir")) if err != nil { return err } diff --git a/pkg/kubectl/cmd/rollout/rollout.go b/pkg/kubectl/cmd/rollout/rollout.go index fb816c643d84f..a65ed4d8df568 100644 --- a/pkg/kubectl/cmd/rollout/rollout.go +++ b/pkg/kubectl/cmd/rollout/rollout.go @@ -42,6 +42,7 @@ var ( * deployments * daemonsets + * statefulsets `) ) diff --git a/pkg/kubectl/cmd/rollout/rollout_history.go b/pkg/kubectl/cmd/rollout/rollout_history.go index 986a51c734bcd..22f31e8e7c21d 100644 --- a/pkg/kubectl/cmd/rollout/rollout_history.go +++ b/pkg/kubectl/cmd/rollout/rollout_history.go @@ -44,7 +44,7 @@ var ( func NewCmdRolloutHistory(f cmdutil.Factory, out io.Writer) *cobra.Command { options := &resource.FilenameOptions{} - validArgs := []string{"deployment", "daemonset"} + validArgs := []string{"deployment", "daemonset", "statefulset"} argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ diff --git a/pkg/kubectl/cmd/rollout/rollout_status.go b/pkg/kubectl/cmd/rollout/rollout_status.go index be9c7f0692427..e54f22030f8af 100644 --- a/pkg/kubectl/cmd/rollout/rollout_status.go +++ b/pkg/kubectl/cmd/rollout/rollout_status.go @@ -50,7 +50,7 @@ var ( func NewCmdRolloutStatus(f cmdutil.Factory, out io.Writer) *cobra.Command { options := &resource.FilenameOptions{} - validArgs := []string{"deployment", "daemonset"} + validArgs := []string{"deployment", "daemonset", "statefulset"} argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ diff --git a/pkg/kubectl/cmd/rollout/rollout_undo.go b/pkg/kubectl/cmd/rollout/rollout_undo.go index 2db0034231539..c703924c42f16 100644 --- a/pkg/kubectl/cmd/rollout/rollout_undo.go +++ b/pkg/kubectl/cmd/rollout/rollout_undo.go @@ -64,7 +64,7 @@ var ( func NewCmdRolloutUndo(f cmdutil.Factory, out io.Writer) *cobra.Command { options := &UndoOptions{} - validArgs := []string{"deployment", "daemonset"} + validArgs := []string{"deployment", "daemonset", "statefulset"} argAliases := kubectl.ResourceAliases(validArgs) cmd := &cobra.Command{ diff --git a/pkg/kubectl/cmd/run.go b/pkg/kubectl/cmd/run.go index 0f27d4273e37e..d9811bd8a7890 100644 --- a/pkg/kubectl/cmd/run.go +++ b/pkg/kubectl/cmd/run.go @@ -176,6 +176,9 @@ func RunRun(f cmdutil.Factory, cmdIn io.Reader, cmdOut, cmdErr io.Writer, cmd *c if interactive && replicas != 1 { return cmdutil.UsageErrorf(cmd, "-i/--stdin requires that replicas is 1, found %d", replicas) } + if cmdutil.GetFlagBool(cmd, "expose") && len(cmdutil.GetFlagString(cmd, "port")) == 0 { + return cmdutil.UsageErrorf(cmd, "--port must be set when exposing a service") + } namespace, _, err := f.DefaultNamespace() if err != nil { @@ -537,11 +540,6 @@ func generateService(f cmdutil.Factory, cmd *cobra.Command, args []string, servi } names := generator.ParamNames() - port := cmdutil.GetFlagString(cmd, "port") - if len(port) == 0 { - return nil, fmt.Errorf("--port must be set when exposing a service") - } - params := map[string]interface{}{} for key, value := range paramsIn { _, isString := value.(string) diff --git a/pkg/kubectl/cmd/set/set_image.go b/pkg/kubectl/cmd/set/set_image.go index c1bb539a9a106..a865bd80f1d5a 100644 --- a/pkg/kubectl/cmd/set/set_image.go +++ b/pkg/kubectl/cmd/set/set_image.go @@ -196,9 +196,11 @@ func (o *ImageOptions) Run() error { continue } } - spec.Containers[i].Image = resolved - // Perform updates - transformed = true + if spec.Containers[i].Image != resolved { + spec.Containers[i].Image = resolved + // Perform updates + transformed = true + } } } // Add a new container if not found @@ -227,7 +229,10 @@ func (o *ImageOptions) Run() error { } if o.PrintObject != nil && (o.Local || o.DryRun) { - return o.PrintObject(o.Cmd, o.Local, o.Mapper, info.Object, o.Out) + if err := o.PrintObject(o.Cmd, o.Local, o.Mapper, info.Object, o.Out); err != nil { + return err + } + continue } // patch the change @@ -250,7 +255,10 @@ func (o *ImageOptions) Run() error { info.Refresh(obj, true) if len(o.Output) > 0 { - return o.PrintObject(o.Cmd, o.Local, o.Mapper, obj, o.Out) + if err := o.PrintObject(o.Cmd, o.Local, o.Mapper, obj, o.Out); err != nil { + return err + } + continue } cmdutil.PrintSuccess(o.Mapper, o.ShortOutput, o.Out, info.Mapping.Resource, info.Name, o.DryRun, "image updated") } diff --git a/pkg/kubectl/cmd/testing/BUILD b/pkg/kubectl/cmd/testing/BUILD index aabfba746734d..722e7c4ca6f30 100644 --- a/pkg/kubectl/cmd/testing/BUILD +++ b/pkg/kubectl/cmd/testing/BUILD @@ -35,6 +35,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/k8s.io/client-go/discovery:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/rest/fake:go_default_library", ], diff --git a/pkg/kubectl/cmd/testing/fake.go b/pkg/kubectl/cmd/testing/fake.go index 5b8731b66d602..0c9f31aea2827 100644 --- a/pkg/kubectl/cmd/testing/fake.go +++ b/pkg/kubectl/cmd/testing/fake.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/rest/fake" fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" @@ -308,6 +309,10 @@ func (f *FakeFactory) RESTClient() (*restclient.RESTClient, error) { return nil, nil } +func (f *FakeFactory) KubernetesClientSet() (*kubernetes.Clientset, error) { + return nil, nil +} + func (f *FakeFactory) ClientSet() (internalclientset.Interface, error) { return nil, nil } @@ -411,7 +416,7 @@ func (f *FakeFactory) ResolveImage(name string) (string, error) { return name, nil } -func (f *FakeFactory) Validator(validate bool, cacheDir string) (validation.Schema, error) { +func (f *FakeFactory) Validator(validate bool, openapi bool, cacheDir string) (validation.Schema, error) { return f.tf.Validator, f.tf.Err } @@ -583,6 +588,33 @@ func (f *fakeAPIFactory) JSONEncoder() runtime.Encoder { return testapi.Default.Codec() } +func (f *fakeAPIFactory) KubernetesClientSet() (*kubernetes.Clientset, error) { + fakeClient := f.tf.Client.(*fake.RESTClient) + clientset := kubernetes.NewForConfigOrDie(f.tf.ClientConfig) + + clientset.CoreV1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.AuthorizationV1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.AuthorizationV1beta1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.AuthorizationV1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.AuthorizationV1beta1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.AutoscalingV1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.AutoscalingV2alpha1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.BatchV1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.BatchV2alpha1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.CertificatesV1beta1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.ExtensionsV1beta1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.RbacV1alpha1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.RbacV1beta1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.StorageV1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.StorageV1beta1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.AppsV1beta1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.AppsV1beta2().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.PolicyV1beta1().RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + clientset.DiscoveryClient.RESTClient().(*restclient.RESTClient).Client = fakeClient.Client + + return clientset, f.tf.Err +} + func (f *fakeAPIFactory) ClientSet() (internalclientset.Interface, error) { // Swap the HTTP client out of the REST client with the fake // version. @@ -699,7 +731,7 @@ func (f *fakeAPIFactory) AttachablePodForObject(object runtime.Object, timeout t } } -func (f *fakeAPIFactory) Validator(validate bool, cacheDir string) (validation.Schema, error) { +func (f *fakeAPIFactory) Validator(validate bool, openapi bool, cacheDir string) (validation.Schema, error) { return f.tf.Validator, f.tf.Err } diff --git a/pkg/kubectl/cmd/top.go b/pkg/kubectl/cmd/top.go index a134904447e44..47c1968517d9e 100644 --- a/pkg/kubectl/cmd/top.go +++ b/pkg/kubectl/cmd/top.go @@ -47,7 +47,7 @@ func NewCmdTop(f cmdutil.Factory, out, errOut io.Writer) *cobra.Command { } // create subcommands - cmd.AddCommand(NewCmdTopNode(f, out)) - cmd.AddCommand(NewCmdTopPod(f, out)) + cmd.AddCommand(NewCmdTopNode(f, nil, out)) + cmd.AddCommand(NewCmdTopPod(f, nil, out)) return cmd } diff --git a/pkg/kubectl/cmd/top_node.go b/pkg/kubectl/cmd/top_node.go index 55cf148f6d46c..f4609627fad82 100644 --- a/pkg/kubectl/cmd/top_node.go +++ b/pkg/kubectl/cmd/top_node.go @@ -50,10 +50,23 @@ type HeapsterTopOptions struct { } func (o *HeapsterTopOptions) Bind(flags *pflag.FlagSet) { - flags.StringVar(&o.Namespace, "heapster-namespace", metricsutil.DefaultHeapsterNamespace, "Namespace Heapster service is located in") - flags.StringVar(&o.Service, "heapster-service", metricsutil.DefaultHeapsterService, "Name of Heapster service") - flags.StringVar(&o.Scheme, "heapster-scheme", metricsutil.DefaultHeapsterScheme, "Scheme (http or https) to connect to Heapster as") - flags.StringVar(&o.Port, "heapster-port", metricsutil.DefaultHeapsterPort, "Port name in service to use") + if len(o.Namespace) == 0 { + o.Namespace = metricsutil.DefaultHeapsterNamespace + } + if len(o.Service) == 0 { + o.Service = metricsutil.DefaultHeapsterService + } + if len(o.Scheme) == 0 { + o.Scheme = metricsutil.DefaultHeapsterScheme + } + if len(o.Port) == 0 { + o.Port = metricsutil.DefaultHeapsterPort + } + + flags.StringVar(&o.Namespace, "heapster-namespace", o.Namespace, "Namespace Heapster service is located in") + flags.StringVar(&o.Service, "heapster-service", o.Service, "Name of Heapster service") + flags.StringVar(&o.Scheme, "heapster-scheme", o.Scheme, "Scheme (http or https) to connect to Heapster as") + flags.StringVar(&o.Port, "heapster-port", o.Port, "Port name in service to use") } var ( @@ -70,8 +83,10 @@ var ( kubectl top node NODE_NAME`)) ) -func NewCmdTopNode(f cmdutil.Factory, out io.Writer) *cobra.Command { - options := &TopNodeOptions{} +func NewCmdTopNode(f cmdutil.Factory, options *TopNodeOptions, out io.Writer) *cobra.Command { + if options == nil { + options = &TopNodeOptions{} + } cmd := &cobra.Command{ Use: "node [NAME | -l label]", @@ -97,7 +112,6 @@ func NewCmdTopNode(f cmdutil.Factory, out io.Writer) *cobra.Command { } func (o *TopNodeOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string, out io.Writer) error { - var err error if len(args) == 1 { o.ResourceName = args[0] } else if len(args) > 1 { diff --git a/pkg/kubectl/cmd/top_node_test.go b/pkg/kubectl/cmd/top_node_test.go index 354afde1055b0..ba292a63bb162 100644 --- a/pkg/kubectl/cmd/top_node_test.go +++ b/pkg/kubectl/cmd/top_node_test.go @@ -67,7 +67,60 @@ func TestTopNodeAllMetrics(t *testing.T) { tf.ClientConfig = defaultClientConfig() buf := bytes.NewBuffer([]byte{}) - cmd := NewCmdTopNode(f, buf) + cmd := NewCmdTopNode(f, nil, buf) + cmd.Run(cmd, []string{}) + + // Check the presence of node names in the output. + result := buf.String() + for _, m := range metrics.Items { + if !strings.Contains(result, m.Name) { + t.Errorf("missing metrics for %s: \n%s", m.Name, result) + } + } +} + +func TestTopNodeAllMetricsCustomDefaults(t *testing.T) { + customBaseHeapsterServiceAddress := "/api/v1/namespaces/custom-namespace/services/https:custom-heapster-service:/proxy" + customBaseMetricsAddress := customBaseHeapsterServiceAddress + "/apis/metrics" + + initTestErrorHandler(t) + metrics, nodes := testNodeMetricsData() + expectedMetricsPath := fmt.Sprintf("%s/%s/nodes", customBaseMetricsAddress, metricsApiVersion) + expectedNodePath := fmt.Sprintf("/%s/%s/nodes", apiPrefix, apiVersion) + + f, tf, codec, ns := cmdtesting.NewAPIFactory() + tf.Printer = &testPrinter{} + tf.Client = &fake.RESTClient{ + APIRegistry: api.Registry, + NegotiatedSerializer: ns, + Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { + switch p, m := req.URL.Path, req.Method; { + case p == expectedMetricsPath && m == "GET": + body, err := marshallBody(metrics) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: body}, nil + case p == expectedNodePath && m == "GET": + return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, nodes)}, nil + default: + t.Fatalf("unexpected request: %#v\nGot URL: %#v\nExpected path: %#v", req, req.URL, expectedMetricsPath) + return nil, nil + } + }), + } + tf.Namespace = "test" + tf.ClientConfig = defaultClientConfig() + buf := bytes.NewBuffer([]byte{}) + + opts := &TopNodeOptions{ + HeapsterOptions: HeapsterTopOptions{ + Namespace: "custom-namespace", + Scheme: "https", + Service: "custom-heapster-service", + }, + } + cmd := NewCmdTopNode(f, opts, buf) cmd.Run(cmd, []string{}) // Check the presence of node names in the output. @@ -116,7 +169,7 @@ func TestTopNodeWithNameMetrics(t *testing.T) { tf.ClientConfig = defaultClientConfig() buf := bytes.NewBuffer([]byte{}) - cmd := NewCmdTopNode(f, buf) + cmd := NewCmdTopNode(f, nil, buf) cmd.Run(cmd, []string{expectedMetrics.Name}) // Check the presence of node names in the output. @@ -176,7 +229,7 @@ func TestTopNodeWithLabelSelectorMetrics(t *testing.T) { tf.ClientConfig = defaultClientConfig() buf := bytes.NewBuffer([]byte{}) - cmd := NewCmdTopNode(f, buf) + cmd := NewCmdTopNode(f, nil, buf) cmd.Flags().Set("selector", label) cmd.Run(cmd, []string{}) diff --git a/pkg/kubectl/cmd/top_pod.go b/pkg/kubectl/cmd/top_pod.go index c43837bce04c8..6e975bd5fcc82 100644 --- a/pkg/kubectl/cmd/top_pod.go +++ b/pkg/kubectl/cmd/top_pod.go @@ -72,8 +72,10 @@ var ( kubectl top pod -l name=myLabel`)) ) -func NewCmdTopPod(f cmdutil.Factory, out io.Writer) *cobra.Command { - options := &TopPodOptions{} +func NewCmdTopPod(f cmdutil.Factory, options *TopPodOptions, out io.Writer) *cobra.Command { + if options == nil { + options = &TopPodOptions{} + } cmd := &cobra.Command{ Use: "pod [NAME | -l label]", diff --git a/pkg/kubectl/cmd/top_pod_test.go b/pkg/kubectl/cmd/top_pod_test.go index d7a43fc146051..3766123fb7999 100644 --- a/pkg/kubectl/cmd/top_pod_test.go +++ b/pkg/kubectl/cmd/top_pod_test.go @@ -139,7 +139,150 @@ func TestTopPod(t *testing.T) { tf.ClientConfig = defaultClientConfig() buf := bytes.NewBuffer([]byte{}) - cmd := NewCmdTopPod(f, buf) + cmd := NewCmdTopPod(f, nil, buf) + for name, value := range testCase.flags { + cmd.Flags().Set(name, value) + } + cmd.Run(cmd, testCase.args) + + // Check the presence of pod names&namespaces/container names in the output. + result := buf.String() + if testCase.containers { + for _, containerName := range expectedContainerNames { + if !strings.Contains(result, containerName) { + t.Errorf("%s: missing metrics for container %s: \n%s", testCase.name, containerName, result) + } + } + } + for _, m := range expectedMetrics { + if !strings.Contains(result, m.Name) { + t.Errorf("%s: missing metrics for %s: \n%s", testCase.name, m.Name, result) + } + if testCase.listsNamespaces && !strings.Contains(result, m.Namespace) { + t.Errorf("%s: missing metrics for %s/%s: \n%s", testCase.name, m.Namespace, m.Name, result) + } + } + for _, name := range nonExpectedMetricsNames { + if strings.Contains(result, name) { + t.Errorf("%s: unexpected metrics for %s: \n%s", testCase.name, name, result) + } + } + } +} + +func TestTopPodCustomDefaults(t *testing.T) { + customBaseHeapsterServiceAddress := "/api/v1/namespaces/custom-namespace/services/https:custom-heapster-service:/proxy" + customBaseMetricsAddress := customBaseHeapsterServiceAddress + "/apis/metrics" + customTopPathPrefix := customBaseMetricsAddress + "/" + metricsApiVersion + + testNS := "custom-namespace" + testCases := []struct { + name string + namespace string + flags map[string]string + args []string + expectedPath string + expectedQuery string + namespaces []string + containers bool + listsNamespaces bool + }{ + { + name: "all namespaces", + flags: map[string]string{"all-namespaces": "true"}, + expectedPath: customTopPathPrefix + "/pods", + namespaces: []string{testNS, "secondtestns", "thirdtestns"}, + listsNamespaces: true, + }, + { + name: "all in namespace", + expectedPath: customTopPathPrefix + "/namespaces/" + testNS + "/pods", + namespaces: []string{testNS, testNS}, + }, + { + name: "pod with name", + args: []string{"pod1"}, + expectedPath: customTopPathPrefix + "/namespaces/" + testNS + "/pods/pod1", + namespaces: []string{testNS}, + }, + { + name: "pod with label selector", + flags: map[string]string{"selector": "key=value"}, + expectedPath: customTopPathPrefix + "/namespaces/" + testNS + "/pods", + expectedQuery: "labelSelector=" + url.QueryEscape("key=value"), + namespaces: []string{testNS, testNS}, + }, + { + name: "pod with container metrics", + flags: map[string]string{"containers": "true"}, + args: []string{"pod1"}, + expectedPath: customTopPathPrefix + "/namespaces/" + testNS + "/pods/pod1", + namespaces: []string{testNS}, + containers: true, + }, + } + initTestErrorHandler(t) + for _, testCase := range testCases { + t.Logf("Running test case: %s", testCase.name) + metricsList := testPodMetricsData() + var expectedMetrics []metricsapi.PodMetrics + var expectedContainerNames, nonExpectedMetricsNames []string + for n, m := range metricsList { + if n < len(testCase.namespaces) { + m.Namespace = testCase.namespaces[n] + expectedMetrics = append(expectedMetrics, m) + for _, c := range m.Containers { + expectedContainerNames = append(expectedContainerNames, c.Name) + } + } else { + nonExpectedMetricsNames = append(nonExpectedMetricsNames, m.Name) + } + } + + var response interface{} + if len(expectedMetrics) == 1 { + response = expectedMetrics[0] + } else { + response = metricsapi.PodMetricsList{ + ListMeta: metav1.ListMeta{ + ResourceVersion: "2", + }, + Items: expectedMetrics, + } + } + + f, tf, _, ns := cmdtesting.NewAPIFactory() + tf.Printer = &testPrinter{} + tf.Client = &fake.RESTClient{ + APIRegistry: api.Registry, + NegotiatedSerializer: ns, + Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { + switch p, m, q := req.URL.Path, req.Method, req.URL.RawQuery; { + case p == testCase.expectedPath && m == "GET" && (testCase.expectedQuery == "" || q == testCase.expectedQuery): + body, err := marshallBody(response) + if err != nil { + t.Errorf("%s: unexpected error: %v", testCase.name, err) + } + return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: body}, nil + default: + t.Fatalf("%s: unexpected request: %#v\nGot URL: %#v\nExpected path: %#v\nExpected query: %#v", + testCase.name, req, req.URL, testCase.expectedPath, testCase.expectedQuery) + return nil, nil + } + }), + } + tf.Namespace = testNS + tf.ClientConfig = defaultClientConfig() + buf := bytes.NewBuffer([]byte{}) + + opts := &TopPodOptions{ + HeapsterOptions: HeapsterTopOptions{ + Namespace: "custom-namespace", + Scheme: "https", + Service: "custom-heapster-service", + }, + } + cmd := NewCmdTopPod(f, opts, buf) for name, value := range testCase.flags { cmd.Flags().Set(name, value) } diff --git a/pkg/kubectl/cmd/util/BUILD b/pkg/kubectl/cmd/util/BUILD index 1035d9aaf8388..fb0f4c184b96c 100644 --- a/pkg/kubectl/cmd/util/BUILD +++ b/pkg/kubectl/cmd/util/BUILD @@ -34,6 +34,7 @@ go_library( "//pkg/controller:go_default_library", "//pkg/kubectl:go_default_library", "//pkg/kubectl/cmd/util/openapi:go_default_library", + "//pkg/kubectl/cmd/util/openapi/validation:go_default_library", "//pkg/kubectl/plugins:go_default_library", "//pkg/kubectl/resource:go_default_library", "//pkg/kubectl/validation:go_default_library", @@ -66,6 +67,7 @@ go_library( "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/util/homedir:go_default_library", @@ -114,6 +116,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//vendor/k8s.io/apimachinery/pkg/version:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", diff --git a/pkg/kubectl/cmd/util/clientcache.go b/pkg/kubectl/cmd/util/clientcache.go index da7c45cc6e737..5312cb3095c7f 100644 --- a/pkg/kubectl/cmd/util/clientcache.go +++ b/pkg/kubectl/cmd/util/clientcache.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" @@ -58,6 +59,35 @@ type ClientCache struct { // argument evaluation discoveryClientFactory DiscoveryClientFactory discoveryClient discovery.DiscoveryInterface + + kubernetesClientCache kubernetesClientCache +} + +// kubernetesClientCache creates a new kubernetes.Clientset one time +// and then returns the result for all future requests +type kubernetesClientCache struct { + // once makes sure the client is only initialized once + once sync.Once + // client is the cached client value + client *kubernetes.Clientset + // err is the cached error value + err error +} + +// KubernetesClientSetForVersion returns a new kubernetes.Clientset. It will cache the value +// the first time it is called and return the cached value on subsequent calls. +// If an error is encountered the first time KubernetesClientSetForVersion is called, +// the error will be cached. +func (c *ClientCache) KubernetesClientSetForVersion(requiredVersion *schema.GroupVersion) (*kubernetes.Clientset, error) { + c.kubernetesClientCache.once.Do(func() { + config, err := c.ClientConfigForVersion(requiredVersion) + if err != nil { + c.kubernetesClientCache.err = err + return + } + c.kubernetesClientCache.client, c.kubernetesClientCache.err = kubernetes.NewForConfig(config) + }) + return c.kubernetesClientCache.client, c.kubernetesClientCache.err } // also looks up the discovery client. We can't do this during init because the flags won't have been set diff --git a/pkg/kubectl/cmd/util/editor/editoptions.go b/pkg/kubectl/cmd/util/editor/editoptions.go index 03e77da5e3f7b..cc15bd59a0ad3 100644 --- a/pkg/kubectl/cmd/util/editor/editoptions.go +++ b/pkg/kubectl/cmd/util/editor/editoptions.go @@ -229,7 +229,7 @@ func (o *EditOptions) Run() error { glog.V(4).Infof("User edited:\n%s", string(edited)) // Apply validation - schema, err := o.f.Validator(o.EnableValidation, o.SchemaCacheDir) + schema, err := o.f.Validator(o.EnableValidation, o.UseOpenAPI, o.SchemaCacheDir) if err != nil { return preservedFile(err, file, o.ErrOut) } diff --git a/pkg/kubectl/cmd/util/factory.go b/pkg/kubectl/cmd/util/factory.go index 1fdae62c9ab31..142ae744a09ee 100644 --- a/pkg/kubectl/cmd/util/factory.go +++ b/pkg/kubectl/cmd/util/factory.go @@ -43,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" @@ -91,6 +92,10 @@ type ClientAccessFactory interface { // ClientSet gives you back an internal, generated clientset ClientSet() (internalclientset.Interface, error) + + // KubernetesClientSet gives you back an external clientset + KubernetesClientSet() (*kubernetes.Clientset, error) + // Returns a RESTClient for accessing Kubernetes resources or an error. RESTClient() (*restclient.RESTClient, error) // Returns a client.Config for accessing the Kubernetes server. @@ -220,7 +225,7 @@ type ObjectMappingFactory interface { AttachablePodForObject(object runtime.Object, timeout time.Duration) (*api.Pod, error) // Returns a schema that can validate objects stored on disk. - Validator(validate bool, cacheDir string) (validation.Schema, error) + Validator(validate bool, openapi bool, cacheDir string) (validation.Schema, error) // SwaggerSchema returns the schema declaration for the provided group version kind. SwaggerSchema(schema.GroupVersionKind) (*swagger.ApiDeclaration, error) // OpenAPISchema returns the schema openapi schema definiton diff --git a/pkg/kubectl/cmd/util/factory_client_access.go b/pkg/kubectl/cmd/util/factory_client_access.go index 9aaa4680c8172..e71a5a2571b8e 100644 --- a/pkg/kubectl/cmd/util/factory_client_access.go +++ b/pkg/kubectl/cmd/util/factory_client_access.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" @@ -167,6 +168,10 @@ func (f *ring0Factory) DiscoveryClient() (discovery.CachedDiscoveryInterface, er return f.discoveryFactory.DiscoveryClient() } +func (f *ring0Factory) KubernetesClientSet() (*kubernetes.Clientset, error) { + return f.clientCache.KubernetesClientSetForVersion(nil) +} + func (f *ring0Factory) ClientSet() (internalclientset.Interface, error) { return f.clientCache.ClientSetForVersion(nil) } diff --git a/pkg/kubectl/cmd/util/factory_object_mapping.go b/pkg/kubectl/cmd/util/factory_object_mapping.go index 5976982da67ba..e51d17caec58a 100644 --- a/pkg/kubectl/cmd/util/factory_object_mapping.go +++ b/pkg/kubectl/cmd/util/factory_object_mapping.go @@ -28,6 +28,7 @@ import ( "time" swagger "github.com/emicklei/go-restful-swagger12" + "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -47,6 +48,7 @@ import ( "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" + openapivalidation "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi/validation" "k8s.io/kubernetes/pkg/kubectl/resource" "k8s.io/kubernetes/pkg/kubectl/validation" "k8s.io/kubernetes/pkg/printers" @@ -402,8 +404,20 @@ func (f *ring1Factory) AttachablePodForObject(object runtime.Object, timeout tim return pod, err } -func (f *ring1Factory) Validator(validate bool, cacheDir string) (validation.Schema, error) { +func (f *ring1Factory) Validator(validate, openapi bool, cacheDir string) (validation.Schema, error) { if validate { + if openapi { + resources, err := f.OpenAPISchema(cacheDir) + if err == nil { + return validation.ConjunctiveSchema{ + openapivalidation.NewSchemaValidation(resources), + validation.NoDoubleKeySchema{}, + }, nil + } + + glog.Warningf("Failed to download OpenAPI (%v), falling back to swagger", err) + } + discovery, err := f.clientAccessFactory.DiscoveryClient() if err != nil { return nil, err diff --git a/pkg/kubectl/cmd/util/factory_test.go b/pkg/kubectl/cmd/util/factory_test.go index 66304691467e9..fbd46b9756130 100644 --- a/pkg/kubectl/cmd/util/factory_test.go +++ b/pkg/kubectl/cmd/util/factory_test.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/util/flag" manualfake "k8s.io/client-go/rest/fake" @@ -88,22 +89,16 @@ func TestPortsForObject(t *testing.T) { }, } - expected := []string{"101"} - got, err := f.PortsForObject(pod) + expected := sets.NewString("101") + ports, err := f.PortsForObject(pod) if err != nil { t.Fatalf("Unexpected error: %v", err) } - if len(expected) != len(got) { - t.Fatalf("Ports size mismatch! Expected %d, got %d", len(expected), len(got)) - } - sort.Strings(expected) - sort.Strings(got) + got := sets.NewString(ports...) - for i, port := range got { - if port != expected[i] { - t.Fatalf("Port mismatch! Expected %s, got %s", expected[i], port) - } + if !expected.Equal(got) { + t.Fatalf("Ports mismatch! Expected %v, got %v", expected, got) } } @@ -130,22 +125,18 @@ func TestProtocolsForObject(t *testing.T) { }, } - expected := "101/TCP,102/UDP" + expected := sets.NewString("101/TCP", "102/UDP") protocolsMap, err := f.ProtocolsForObject(pod) if err != nil { t.Fatalf("Unexpected error: %v", err) } - got := kubectl.MakeProtocols(protocolsMap) - expectedSlice := strings.Split(expected, ",") - gotSlice := strings.Split(got, ",") - sort.Strings(expectedSlice) - sort.Strings(gotSlice) + protocolsString := kubectl.MakeProtocols(protocolsMap) + protocolsStrings := strings.Split(protocolsString, ",") + got := sets.NewString(protocolsStrings...) - for i, protocol := range gotSlice { - if protocol != expectedSlice[i] { - t.Fatalf("Protocols mismatch! Expected %s, got %s", expectedSlice[i], protocol) - } + if !expected.Equal(got) { + t.Fatalf("Protocols mismatch! Expected %v, got %v", expected, got) } } diff --git a/pkg/kubectl/cmd/util/helpers.go b/pkg/kubectl/cmd/util/helpers.go index 12cdcbf642c28..78590ab25c18b 100644 --- a/pkg/kubectl/cmd/util/helpers.go +++ b/pkg/kubectl/cmd/util/helpers.go @@ -395,16 +395,19 @@ func GetPodRunningTimeoutFlag(cmd *cobra.Command) (time.Duration, error) { func AddValidateFlags(cmd *cobra.Command) { cmd.Flags().Bool("validate", true, "If true, use a schema to validate the input before sending it") cmd.Flags().String("schema-cache-dir", fmt.Sprintf("~/%s/%s", clientcmd.RecommendedHomeDir, clientcmd.RecommendedSchemaName), fmt.Sprintf("If non-empty, load/store cached API schemas in this directory, default is '$HOME/%s/%s'", clientcmd.RecommendedHomeDir, clientcmd.RecommendedSchemaName)) + cmd.Flags().Bool("openapi-validation", false, "If true, use openapi rather than swagger for validation.") cmd.MarkFlagFilename("schema-cache-dir") } func AddValidateOptionFlags(cmd *cobra.Command, options *ValidateOptions) { cmd.Flags().BoolVar(&options.EnableValidation, "validate", true, "If true, use a schema to validate the input before sending it") cmd.Flags().StringVar(&options.SchemaCacheDir, "schema-cache-dir", fmt.Sprintf("~/%s/%s", clientcmd.RecommendedHomeDir, clientcmd.RecommendedSchemaName), fmt.Sprintf("If non-empty, load/store cached API schemas in this directory, default is '$HOME/%s/%s'", clientcmd.RecommendedHomeDir, clientcmd.RecommendedSchemaName)) + cmd.Flags().BoolVar(&options.UseOpenAPI, "openapi-validation", false, "If true, use openapi rather than swagger for validation") cmd.MarkFlagFilename("schema-cache-dir") } func AddOpenAPIFlags(cmd *cobra.Command) { + cmd.Flags().Bool("openapi-validation", false, "If true, use openapi rather than swagger for validation") cmd.Flags().String("schema-cache-dir", fmt.Sprintf("~/%s/%s", clientcmd.RecommendedHomeDir, clientcmd.RecommendedSchemaName), fmt.Sprintf("If non-empty, load/store cached API schemas in this directory, default is '$HOME/%s/%s'", @@ -448,6 +451,7 @@ func AddGeneratorFlags(cmd *cobra.Command, defaultGenerator string) { type ValidateOptions struct { EnableValidation bool + UseOpenAPI bool SchemaCacheDir string } diff --git a/pkg/kubectl/cmd/util/openapi/document.go b/pkg/kubectl/cmd/util/openapi/document.go index 23e83b3dda175..6b2f6782b948d 100644 --- a/pkg/kubectl/cmd/util/openapi/document.go +++ b/pkg/kubectl/cmd/util/openapi/document.go @@ -164,8 +164,9 @@ func (d *Definitions) parseReference(s *openapi_v2.Schema, path *Path) (Schema, if _, ok := d.models[reference]; !ok { return nil, newSchemaError(path, "unknown model in reference: %q", reference) } - return &Reference{ - Reference: reference, + return &Ref{ + BaseSchema: d.parseBaseSchema(s, path), + reference: reference, definitions: d, }, nil } @@ -303,38 +304,27 @@ func (d *Definitions) LookupResource(gvk schema.GroupVersionKind) Schema { return model } -// SchemaReference doesn't match a specific type. It's mostly a -// pass-through type. -type Reference struct { - Reference string +type Ref struct { + BaseSchema + reference string definitions *Definitions } -var _ Schema = &Reference{} +var _ Reference = &Ref{} -func (r *Reference) GetSubSchema() Schema { - return r.definitions.models[r.Reference] +func (r *Ref) Reference() string { + return r.reference } -func (r *Reference) Accept(s SchemaVisitor) { - r.GetSubSchema().Accept(s) +func (r *Ref) SubSchema() Schema { + return r.definitions.models[r.reference] } -func (r *Reference) GetDescription() string { - return r.GetSubSchema().GetDescription() +func (r *Ref) Accept(v SchemaVisitor) { + v.VisitReference(r) } -func (r *Reference) GetExtensions() map[string]interface{} { - return r.GetSubSchema().GetExtensions() -} - -func (*Reference) GetPath() *Path { - // Reference never has a path, because it can be referenced from - // multiple locations. - return &Path{} -} - -func (r *Reference) GetName() string { - return r.Reference +func (r *Ref) GetName() string { + return fmt.Sprintf("Reference to %q", r.reference) } diff --git a/pkg/kubectl/cmd/util/openapi/openapi.go b/pkg/kubectl/cmd/util/openapi/openapi.go index 81f1f0eae5bd3..8e6cf0639f44f 100644 --- a/pkg/kubectl/cmd/util/openapi/openapi.go +++ b/pkg/kubectl/cmd/util/openapi/openapi.go @@ -49,11 +49,13 @@ type Resources interface { // - Map is a map of string to one and only one given subtype // - Primitive can be string, integer, number and boolean. // - Kind is an object with specific fields mapping to specific types. +// - Reference is a link to another definition. type SchemaVisitor interface { VisitArray(*Array) VisitMap(*Map) VisitPrimitive(*Primitive) VisitKind(*Kind) + VisitReference(Reference) } // Schema is the base definition of an openapi type. @@ -219,3 +221,11 @@ func (p *Primitive) GetName() string { } return fmt.Sprintf("%s (%s)", p.Type, p.Format) } + +// Reference implementation depends on the type of document. +type Reference interface { + Schema + + Reference() string + SubSchema() Schema +} diff --git a/pkg/kubectl/cmd/util/openapi/openapi_test.go b/pkg/kubectl/cmd/util/openapi/openapi_test.go index e2eb2fa0c0dc4..93736c93ca796 100644 --- a/pkg/kubectl/cmd/util/openapi/openapi_test.go +++ b/pkg/kubectl/cmd/util/openapi/openapi_test.go @@ -78,20 +78,20 @@ var _ = Describe("Reading apps/v1beta1/Deployment from openAPIData", func() { It("should have a metadata key of type Reference", func() { Expect(deployment.Fields).To(HaveKey("metadata")) - key := deployment.Fields["metadata"].(*openapi.Reference) + key := deployment.Fields["metadata"].(openapi.Reference) Expect(key).ToNot(BeNil()) - Expect(key.Reference).To(Equal("io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta")) - subSchema := key.GetSubSchema().(*openapi.Kind) + Expect(key.Reference()).To(Equal("io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta")) + subSchema := key.SubSchema().(*openapi.Kind) Expect(subSchema).ToNot(BeNil()) }) var status *openapi.Kind It("should have a status key of type Reference", func() { Expect(deployment.Fields).To(HaveKey("status")) - key := deployment.Fields["status"].(*openapi.Reference) + key := deployment.Fields["status"].(openapi.Reference) Expect(key).ToNot(BeNil()) - Expect(key.Reference).To(Equal("io.k8s.api.apps.v1beta1.DeploymentStatus")) - status = key.GetSubSchema().(*openapi.Kind) + Expect(key.Reference()).To(Equal("io.k8s.api.apps.v1beta1.DeploymentStatus")) + status = key.SubSchema().(*openapi.Kind) Expect(status).ToNot(BeNil()) }) @@ -106,22 +106,22 @@ var _ = Describe("Reading apps/v1beta1/Deployment from openAPIData", func() { Expect(status.Fields).To(HaveKey("conditions")) conditions := status.Fields["conditions"].(*openapi.Array) Expect(conditions).ToNot(BeNil()) - Expect(conditions.GetName()).To(Equal("Array of io.k8s.api.apps.v1beta1.DeploymentCondition")) + Expect(conditions.GetName()).To(Equal(`Array of Reference to "io.k8s.api.apps.v1beta1.DeploymentCondition"`)) Expect(conditions.GetExtensions()).To(Equal(map[string]interface{}{ "x-kubernetes-patch-merge-key": "type", "x-kubernetes-patch-strategy": "merge", })) - condition := conditions.SubType.(*openapi.Reference) - Expect(condition.Reference).To(Equal("io.k8s.api.apps.v1beta1.DeploymentCondition")) + condition := conditions.SubType.(openapi.Reference) + Expect(condition.Reference()).To(Equal("io.k8s.api.apps.v1beta1.DeploymentCondition")) }) var spec *openapi.Kind It("should have a spec key of type Reference", func() { Expect(deployment.Fields).To(HaveKey("spec")) - key := deployment.Fields["spec"].(*openapi.Reference) + key := deployment.Fields["spec"].(openapi.Reference) Expect(key).ToNot(BeNil()) - Expect(key.Reference).To(Equal("io.k8s.api.apps.v1beta1.DeploymentSpec")) - spec = key.GetSubSchema().(*openapi.Kind) + Expect(key.Reference()).To(Equal("io.k8s.api.apps.v1beta1.DeploymentSpec")) + spec = key.SubSchema().(*openapi.Kind) Expect(spec).ToNot(BeNil()) }) @@ -132,9 +132,9 @@ var _ = Describe("Reading apps/v1beta1/Deployment from openAPIData", func() { It("should have a spec with a PodTemplateSpec sub-field", func() { Expect(spec.Fields).To(HaveKey("template")) - key := spec.Fields["template"].(*openapi.Reference) + key := spec.Fields["template"].(openapi.Reference) Expect(key).ToNot(BeNil()) - Expect(key.Reference).To(Equal("io.k8s.api.core.v1.PodTemplateSpec")) + Expect(key.Reference()).To(Equal("io.k8s.api.core.v1.PodTemplateSpec")) }) }) @@ -164,10 +164,10 @@ var _ = Describe("Reading authorization.k8s.io/v1/SubjectAccessReview from openA sar := schema.(*openapi.Kind) Expect(sar).ToNot(BeNil()) Expect(sar.Fields).To(HaveKey("spec")) - specRef := sar.Fields["spec"].(*openapi.Reference) + specRef := sar.Fields["spec"].(openapi.Reference) Expect(specRef).ToNot(BeNil()) - Expect(specRef.Reference).To(Equal("io.k8s.api.authorization.v1.SubjectAccessReviewSpec")) - sarspec = specRef.GetSubSchema().(*openapi.Kind) + Expect(specRef.Reference()).To(Equal("io.k8s.api.authorization.v1.SubjectAccessReviewSpec")) + sarspec = specRef.SubSchema().(*openapi.Kind) Expect(sarspec).ToNot(BeNil()) }) diff --git a/pkg/kubectl/cmd/util/openapi/validation/BUILD b/pkg/kubectl/cmd/util/openapi/validation/BUILD index 74eb5e9721890..77b98f39bb3e9 100644 --- a/pkg/kubectl/cmd/util/openapi/validation/BUILD +++ b/pkg/kubectl/cmd/util/openapi/validation/BUILD @@ -17,6 +17,7 @@ go_library( ], tags = ["automanaged"], deps = [ + "//pkg/api:go_default_library", "//pkg/api/util:go_default_library", "//pkg/kubectl/cmd/util/openapi:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", @@ -36,6 +37,7 @@ go_test( tags = ["automanaged"], deps = [ ":go_default_library", + "//pkg/api/testapi:go_default_library", "//pkg/kubectl/cmd/util/openapi:go_default_library", "//pkg/kubectl/cmd/util/openapi/testing:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", diff --git a/pkg/kubectl/cmd/util/openapi/validation/types.go b/pkg/kubectl/cmd/util/openapi/validation/types.go index e209930b4a95b..2c16c2b16fc3f 100644 --- a/pkg/kubectl/cmd/util/openapi/validation/types.go +++ b/pkg/kubectl/cmd/util/openapi/validation/types.go @@ -103,6 +103,9 @@ func (item *mapItem) VisitMap(schema *openapi.Map) { func (item *mapItem) VisitKind(schema *openapi.Kind) { // Verify each sub-field. for _, key := range item.sortedKeys() { + if item.Map[key] == nil { + continue + } subItem, err := itemFactory(item.Path().FieldPath(key), item.Map[key]) if err != nil { item.AddError(err) @@ -118,12 +121,17 @@ func (item *mapItem) VisitKind(schema *openapi.Kind) { // Verify that all required fields are present. for _, required := range schema.RequiredFields { - if _, ok := item.Map[required]; !ok { + if v, ok := item.Map[required]; !ok || v == nil { item.AddValidationError(MissingRequiredFieldError{Path: schema.GetPath().String(), Field: required}) } } } +func (item *mapItem) VisitReference(schema openapi.Reference) { + // passthrough + schema.SubSchema().Accept(item) +} + // arrayItem represents a yaml array. type arrayItem struct { baseItem @@ -139,7 +147,12 @@ func (item *arrayItem) VisitPrimitive(schema *openapi.Primitive) { func (item *arrayItem) VisitArray(schema *openapi.Array) { for i, v := range item.Array { - subItem, err := itemFactory(item.Path().ArrayPath(i), v) + path := item.Path().ArrayPath(i) + if v == nil { + item.AddValidationError(InvalidObjectTypeError{Type: "nil", Path: path.String()}) + continue + } + subItem, err := itemFactory(path, v) if err != nil { item.AddError(err) continue @@ -157,6 +170,11 @@ func (item *arrayItem) VisitKind(schema *openapi.Kind) { item.AddValidationError(InvalidTypeError{Path: schema.GetPath().String(), Expected: "array", Actual: "map"}) } +func (item *arrayItem) VisitReference(schema openapi.Reference) { + // passthrough + schema.SubSchema().Accept(item) +} + // primitiveItem represents a yaml value. type primitiveItem struct { baseItem @@ -208,6 +226,11 @@ func (item *primitiveItem) VisitKind(schema *openapi.Kind) { item.AddValidationError(InvalidTypeError{Path: schema.GetPath().String(), Expected: "map", Actual: item.Kind}) } +func (item *primitiveItem) VisitReference(schema openapi.Reference) { + // passthrough + schema.SubSchema().Accept(item) +} + // itemFactory creates the relevant item type/visitor based on the current yaml type. func itemFactory(path openapi.Path, v interface{}) (ValidationItem, error) { // We need to special case for no-type fields in yaml (e.g. empty item in list) diff --git a/pkg/kubectl/cmd/util/openapi/validation/validation.go b/pkg/kubectl/cmd/util/openapi/validation/validation.go index ed5e255c8bf9d..08bc6a573bda6 100644 --- a/pkg/kubectl/cmd/util/openapi/validation/validation.go +++ b/pkg/kubectl/cmd/util/openapi/validation/validation.go @@ -19,11 +19,13 @@ package validation import ( "errors" "fmt" + "strings" "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/kubernetes/pkg/api" apiutil "k8s.io/kubernetes/pkg/api/util" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" ) @@ -38,7 +40,7 @@ func NewSchemaValidation(resources openapi.Resources) *SchemaValidation { } } -func (v *SchemaValidation) Validate(data []byte) error { +func (v *SchemaValidation) ValidateBytes(data []byte) error { obj, err := parse(data) if err != nil { return err @@ -49,21 +51,48 @@ func (v *SchemaValidation) Validate(data []byte) error { return err } + if strings.HasSuffix(gvk.Kind, "List") { + return utilerrors.NewAggregate(v.validateList(obj)) + } + + return utilerrors.NewAggregate(v.validateResource(obj, gvk)) +} + +func (v *SchemaValidation) validateList(object interface{}) []error { + fields := object.(map[string]interface{}) + if fields == nil { + return []error{errors.New("invalid object to validate")} + } + + errs := []error{} + for _, item := range fields["items"].([]interface{}) { + if gvk, err := getObjectKind(item); err != nil { + errs = append(errs, err) + } else { + errs = append(errs, v.validateResource(item, gvk)...) + } + } + return errs +} + +func (v *SchemaValidation) validateResource(obj interface{}, gvk schema.GroupVersionKind) []error { + if !api.Registry.IsEnabledVersion(gvk.GroupVersion()) { + // if we don't have this in our scheme, just skip + // validation because its an object we don't recognize + return nil + } + resource := v.resources.LookupResource(gvk) if resource == nil { - return fmt.Errorf("unknown object type %q", gvk) + return []error{fmt.Errorf("unknown object type %#v", gvk)} } rootValidation, err := itemFactory(openapi.NewPath(gvk.Kind), obj) if err != nil { - return err + return []error{err} } resource.Accept(rootValidation) - errs := rootValidation.Errors() - if errs != nil { - return utilerrors.NewAggregate(errs) - } - return nil + return rootValidation.Errors() } func parse(data []byte) (interface{}, error) { @@ -91,6 +120,7 @@ func getObjectKind(object interface{}) (schema.GroupVersionKind, error) { return schema.GroupVersionKind{}, errors.New("apiVersion isn't string type") } version := apiutil.GetVersion(apiVersion.(string)) + group := apiutil.GetGroup(apiVersion.(string)) kind := fields["kind"] if kind == nil { return schema.GroupVersionKind{}, errors.New("kind not set") @@ -99,5 +129,5 @@ func getObjectKind(object interface{}) (schema.GroupVersionKind, error) { return schema.GroupVersionKind{}, errors.New("kind isn't string type") } - return schema.GroupVersionKind{Kind: kind.(string), Version: version}, nil + return schema.GroupVersionKind{Group: group, Version: version, Kind: kind.(string)}, nil } diff --git a/pkg/kubectl/cmd/util/openapi/validation/validation_test.go b/pkg/kubectl/cmd/util/openapi/validation/validation_test.go index cdcc0012f04bc..8454db4688080 100644 --- a/pkg/kubectl/cmd/util/openapi/validation/validation_test.go +++ b/pkg/kubectl/cmd/util/openapi/validation/validation_test.go @@ -23,6 +23,8 @@ import ( . "github.com/onsi/gomega" utilerrors "k8s.io/apimachinery/pkg/util/errors" + // This dependency is needed to register API types. + _ "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi" tst "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi/testing" "k8s.io/kubernetes/pkg/kubectl/cmd/util/openapi/validation" @@ -41,8 +43,30 @@ var _ = Describe("resource validation using OpenAPI Schema", func() { Expect(validator).ToNot(BeNil()) }) + It("finds Deployment in Schema and validates it", func() { + err := validator.ValidateBytes([]byte(` +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + name: redis-master + name: name +spec: + replicas: 1 + template: + metadata: + labels: + app: redis + spec: + containers: + - image: redis + name: redis +`)) + Expect(err).To(BeNil()) + }) + It("validates a valid pod", func() { - err := validator.Validate([]byte(` + err := validator.ValidateBytes([]byte(` apiVersion: v1 kind: Pod metadata: @@ -64,7 +88,7 @@ spec: }) It("finds invalid command (string instead of []string) in Json Pod", func() { - err := validator.Validate([]byte(` + err := validator.ValidateBytes([]byte(` { "kind": "Pod", "apiVersion": "v1", @@ -98,7 +122,7 @@ spec: }) It("fails because hostPort is string instead of int", func() { - err := validator.Validate([]byte(` + err := validator.ValidateBytes([]byte(` { "kind": "Pod", "apiVersion": "v1", @@ -150,7 +174,7 @@ spec: }) It("fails because volume is not an array of object", func() { - err := validator.Validate([]byte(` + err := validator.ValidateBytes([]byte(` { "kind": "Pod", "apiVersion": "v1", @@ -191,7 +215,7 @@ spec: }) It("fails because some string lists have empty strings", func() { - err := validator.Validate([]byte(` + err := validator.ValidateBytes([]byte(` apiVersion: v1 kind: Pod metadata: @@ -209,14 +233,129 @@ spec: `)) Expect(err).To(Equal(utilerrors.NewAggregate([]error{ - validation.InvalidObjectTypeError{ - Path: "Pod.spec.containers[0].args[0]", - Type: "nil", + validation.ValidationError{ + Path: "Pod.spec.containers[0].args", + Err: validation.InvalidObjectTypeError{ + Path: "Pod.spec.containers[0].args[0]", + Type: "nil", + }, + }, + validation.ValidationError{ + Path: "Pod.spec.containers[0].command", + Err: validation.InvalidObjectTypeError{ + Path: "Pod.spec.containers[0].command[0]", + Type: "nil", + }, + }, + }))) + }) + + It("fails if required fields are missing", func() { + err := validator.ValidateBytes([]byte(` +apiVersion: v1 +kind: Pod +metadata: + labels: + name: redis-master + name: name +spec: + containers: + - command: ["my", "command"] +`)) + + Expect(err).To(Equal(utilerrors.NewAggregate([]error{ + validation.ValidationError{ + Path: "Pod.spec.containers[0]", + Err: validation.MissingRequiredFieldError{ + Path: "io.k8s.api.core.v1.Container", + Field: "name", + }, }, - validation.InvalidObjectTypeError{ - Path: "Pod.spec.containers[0].command[0]", - Type: "nil", + validation.ValidationError{ + Path: "Pod.spec.containers[0]", + Err: validation.MissingRequiredFieldError{ + Path: "io.k8s.api.core.v1.Container", + Field: "image", + }, }, }))) }) + + It("fails if required fields are empty", func() { + err := validator.ValidateBytes([]byte(` +apiVersion: v1 +kind: Pod +metadata: + labels: + name: redis-master + name: name +spec: + containers: + - image: + name: +`)) + + Expect(err).To(Equal(utilerrors.NewAggregate([]error{ + validation.ValidationError{ + Path: "Pod.spec.containers[0]", + Err: validation.MissingRequiredFieldError{ + Path: "io.k8s.api.core.v1.Container", + Field: "name", + }, + }, + validation.ValidationError{ + Path: "Pod.spec.containers[0]", + Err: validation.MissingRequiredFieldError{ + Path: "io.k8s.api.core.v1.Container", + Field: "image", + }, + }, + }))) + }) + + It("is fine with empty non-mandatory fields", func() { + err := validator.ValidateBytes([]byte(` +apiVersion: v1 +kind: Pod +metadata: + labels: + name: redis-master + name: name +spec: + containers: + - image: image + name: name + command: +`)) + + Expect(err).To(BeNil()) + }) + + It("can validate lists", func() { + err := validator.ValidateBytes([]byte(` +apiVersion: v1 +kind: List +items: + - apiVersion: v1 + kind: Pod + metadata: + labels: + name: redis-master + name: name + spec: + containers: + - name: name +`)) + + Expect(err).To(Equal(utilerrors.NewAggregate([]error{ + validation.ValidationError{ + Path: "Pod.spec.containers[0]", + Err: validation.MissingRequiredFieldError{ + Path: "io.k8s.api.core.v1.Container", + Field: "image", + }, + }, + }))) + }) + }) diff --git a/pkg/kubectl/history.go b/pkg/kubectl/history.go index 5277abfbc89d2..7bf4eb5b88c37 100644 --- a/pkg/kubectl/history.go +++ b/pkg/kubectl/history.go @@ -153,9 +153,9 @@ type DaemonSetHistoryViewer struct { // ViewHistory returns a revision-to-history map as the revision history of a deployment // TODO: this should be a describer func (h *DaemonSetHistoryViewer) ViewHistory(namespace, name string, revision int64) (string, error) { - versionedExtensionsClient := versionedExtensionsClientV1beta1(h.c) versionedAppsClient := versionedAppsClientV1beta1(h.c) - ds, allHistory, err := controlledHistories(versionedExtensionsClient, versionedAppsClient, namespace, name) + versionedExtensionsClient := versionedExtensionsClientV1beta1(h.c) + versionedObj, allHistory, err := controlledHistories(versionedAppsClient, versionedExtensionsClient, namespace, name, "DaemonSet") if err != nil { return "", fmt.Errorf("unable to find history controlled by DaemonSet %s: %v", name, err) } @@ -175,7 +175,13 @@ func (h *DaemonSetHistoryViewer) ViewHistory(namespace, name string, revision in if !ok { return "", fmt.Errorf("unable to find the specified revision") } - dsOfHistory, err := applyHistory(ds, history) + + versionedDS, ok := versionedObj.(*extensionsv1beta1.DaemonSet) + if !ok { + return "", fmt.Errorf("unexpected non-DaemonSet object returned: %v", versionedDS) + } + + dsOfHistory, err := applyHistory(versionedDS, history) if err != nil { return "", fmt.Errorf("unable to parse history %s", history.Name) } @@ -256,29 +262,51 @@ func (h *StatefulSetHistoryViewer) ViewHistory(namespace, name string, revision }) } -// controlledHistories returns all ControllerRevisions controlled by the given DaemonSet -func controlledHistories(extensions clientextensionsv1beta1.ExtensionsV1beta1Interface, apps clientappsv1beta1.AppsV1beta1Interface, namespace, name string) (*extensionsv1beta1.DaemonSet, []*appsv1beta1.ControllerRevision, error) { - ds, err := extensions.DaemonSets(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return nil, nil, fmt.Errorf("failed to retrieve DaemonSet %s: %v", name, err) +// controlledHistories returns all ControllerRevisions controlled by the given API object +func controlledHistories(apps clientappsv1beta1.AppsV1beta1Interface, extensions clientextensionsv1beta1.ExtensionsV1beta1Interface, namespace, name, kind string) (runtime.Object, []*appsv1beta1.ControllerRevision, error) { + var obj runtime.Object + var labelSelector *metav1.LabelSelector + + switch kind { + case "DaemonSet": + ds, err := extensions.DaemonSets(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to retrieve DaemonSet %s: %v", name, err) + } + labelSelector = ds.Spec.Selector + obj = ds + case "StatefulSet": + ss, err := apps.StatefulSets(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to retrieve StatefulSet %s: %v", name, err) + } + labelSelector = ss.Spec.Selector + obj = ss + default: + return nil, nil, fmt.Errorf("unsupported API object kind: %s", kind) } + var result []*appsv1beta1.ControllerRevision - selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector) + selector, err := metav1.LabelSelectorAsSelector(labelSelector) if err != nil { return nil, nil, err } - historyList, err := apps.ControllerRevisions(ds.Namespace).List(metav1.ListOptions{LabelSelector: selector.String()}) + historyList, err := apps.ControllerRevisions(namespace).List(metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { return nil, nil, err } + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, nil, fmt.Errorf("failed to obtain accessor for %s named %s: %v", kind, name, err) + } for i := range historyList.Items { history := historyList.Items[i] - // Only add history that belongs to the DaemonSet - if metav1.IsControlledBy(&history, ds) { + // Only add history that belongs to the API object + if metav1.IsControlledBy(&history, accessor) { result = append(result, &history) } } - return ds, result, nil + return obj, result, nil } // applyHistory returns a specific revision of DaemonSet by applying the given history to a copy of the given DaemonSet diff --git a/pkg/kubectl/pdb.go b/pkg/kubectl/pdb.go index 43f54c4a7c6b0..08d8a74b8750c 100644 --- a/pkg/kubectl/pdb.go +++ b/pkg/kubectl/pdb.go @@ -214,7 +214,7 @@ func (s *PodDisruptionBudgetV2Generator) validate() error { return fmt.Errorf("a selector must be specified") } if len(s.MaxUnavailable) > 0 && len(s.MinAvailable) > 0 { - return fmt.Errorf("exactly one of min-available and max-available must be specified") + return fmt.Errorf("min-available and max-unavailable cannot be both specified") } return nil } diff --git a/pkg/kubectl/rollback.go b/pkg/kubectl/rollback.go index 957cbf111e413..31f74d81c06f6 100644 --- a/pkg/kubectl/rollback.go +++ b/pkg/kubectl/rollback.go @@ -39,6 +39,7 @@ import ( clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/controller/daemon" deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" + "k8s.io/kubernetes/pkg/controller/statefulset" sliceutil "k8s.io/kubernetes/pkg/kubectl/util/slice" printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" ) @@ -59,6 +60,8 @@ func RollbackerFor(kind schema.GroupKind, c clientset.Interface) (Rollbacker, er return &DeploymentRollbacker{c}, nil case extensions.Kind("DaemonSet"), apps.Kind("DaemonSet"): return &DaemonSetRollbacker{c}, nil + case apps.Kind("StatefulSet"): + return &StatefulSetRollbacker{c}, nil } return nil, fmt.Errorf("no rollbacker has been implemented for %q", kind) } @@ -221,32 +224,22 @@ func (r *DaemonSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations ma if !ok { return "", fmt.Errorf("passed object is not a DaemonSet: %#v", obj) } - versionedExtensionsClient := versionedExtensionsClientV1beta1(r.c) versionedAppsClient := versionedAppsClientV1beta1(r.c) - versionedDS, allHistory, err := controlledHistories(versionedExtensionsClient, versionedAppsClient, ds.Namespace, ds.Name) + versionedExtensionsClient := versionedExtensionsClientV1beta1(r.c) + versionedObj, allHistory, err := controlledHistories(versionedAppsClient, versionedExtensionsClient, ds.Namespace, ds.Name, "DaemonSet") if err != nil { return "", fmt.Errorf("unable to find history controlled by DaemonSet %s: %v", ds.Name, err) } + versionedDS, ok := versionedObj.(*externalextensions.DaemonSet) + if !ok { + return "", fmt.Errorf("unexpected non-DaemonSet object returned: %v", versionedDS) + } if toRevision == 0 && len(allHistory) <= 1 { return "", fmt.Errorf("no last revision to roll back to") } - // Find the history to rollback to - var toHistory *appsv1beta1.ControllerRevision - if toRevision == 0 { - // If toRevision == 0, find the latest revision (2nd max) - sort.Sort(historiesByRevision(allHistory)) - toHistory = allHistory[len(allHistory)-2] - } else { - for _, h := range allHistory { - if h.Revision == toRevision { - // If toRevision != 0, find the history with matching revision - toHistory = h - break - } - } - } + toHistory := findHistory(toRevision, allHistory) if toHistory == nil { return "", revisionNotFoundErr(toRevision) } @@ -256,14 +249,7 @@ func (r *DaemonSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations ma if err != nil { return "", err } - content := bytes.NewBuffer([]byte{}) - w := printersinternal.NewPrefixWriter(content) - internalTemplate := &api.PodTemplateSpec{} - if err := apiv1.Convert_v1_PodTemplateSpec_To_api_PodTemplateSpec(&appliedDS.Spec.Template, internalTemplate, nil); err != nil { - return "", fmt.Errorf("failed to convert podtemplate while printing: %v", err) - } - printersinternal.DescribePodTemplate(internalTemplate, w) - return fmt.Sprintf("will roll back to %s", content.String()), nil + return printPodTemplate(&appliedDS.Spec.Template) } // Skip if the revision already matches current DaemonSet @@ -283,6 +269,104 @@ func (r *DaemonSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations ma return rollbackSuccess, nil } +type StatefulSetRollbacker struct { + c clientset.Interface +} + +// toRevision is a non-negative integer, with 0 being reserved to indicate rolling back to previous configuration +func (r *StatefulSetRollbacker) Rollback(obj runtime.Object, updatedAnnotations map[string]string, toRevision int64, dryRun bool) (string, error) { + if toRevision < 0 { + return "", revisionNotFoundErr(toRevision) + } + + ss, ok := obj.(*apps.StatefulSet) + if !ok { + return "", fmt.Errorf("passed object is not a StatefulSet: %#v", obj) + } + versionedAppsClient := versionedAppsClientV1beta1(r.c) + versionedExtensionsClient := versionedExtensionsClientV1beta1(r.c) + versionedObj, allHistory, err := controlledHistories(versionedAppsClient, versionedExtensionsClient, ss.Namespace, ss.Name, "StatefulSet") + if err != nil { + return "", fmt.Errorf("unable to find history controlled by StatefulSet %s: %v", ss.Name, err) + } + + versionedSS, ok := versionedObj.(*appsv1beta1.StatefulSet) + if !ok { + return "", fmt.Errorf("unexpected non-StatefulSet object returned: %v", versionedSS) + } + + if toRevision == 0 && len(allHistory) <= 1 { + return "", fmt.Errorf("no last revision to roll back to") + } + + toHistory := findHistory(toRevision, allHistory) + if toHistory == nil { + return "", revisionNotFoundErr(toRevision) + } + + if dryRun { + appliedSS, err := statefulset.ApplyRevision(versionedSS, toHistory) + if err != nil { + return "", err + } + return printPodTemplate(&appliedSS.Spec.Template) + } + + // Skip if the revision already matches current StatefulSet + done, err := statefulset.Match(versionedSS, toHistory) + if err != nil { + return "", err + } + if done { + return fmt.Sprintf("%s (current template already matches revision %d)", rollbackSkipped, toRevision), nil + } + + // Restore revision + if _, err = versionedAppsClient.StatefulSets(ss.Namespace).Patch(ss.Name, types.StrategicMergePatchType, toHistory.Data.Raw); err != nil { + return "", fmt.Errorf("failed restoring revision %d: %v", toRevision, err) + } + + return rollbackSuccess, nil +} + +// findHistory returns a controllerrevision of a specific revision from the given controllerrevisions. +// It returns nil if no such controllerrevision exists. +// If toRevision is 0, the last previously used history is returned. +func findHistory(toRevision int64, allHistory []*appsv1beta1.ControllerRevision) *appsv1beta1.ControllerRevision { + if toRevision == 0 && len(allHistory) <= 1 { + return nil + } + + // Find the history to rollback to + var toHistory *appsv1beta1.ControllerRevision + if toRevision == 0 { + // If toRevision == 0, find the latest revision (2nd max) + sort.Sort(historiesByRevision(allHistory)) + toHistory = allHistory[len(allHistory)-2] + } else { + for _, h := range allHistory { + if h.Revision == toRevision { + // If toRevision != 0, find the history with matching revision + return h + } + } + } + + return toHistory +} + +// printPodTemplate converts a given pod template into a human-readable string. +func printPodTemplate(specTemplate *v1.PodTemplateSpec) (string, error) { + content := bytes.NewBuffer([]byte{}) + w := printersinternal.NewPrefixWriter(content) + internalTemplate := &api.PodTemplateSpec{} + if err := apiv1.Convert_v1_PodTemplateSpec_To_api_PodTemplateSpec(specTemplate, internalTemplate, nil); err != nil { + return "", fmt.Errorf("failed to convert podtemplate while printing: %v", err) + } + printersinternal.DescribePodTemplate(internalTemplate, w) + return fmt.Sprintf("will roll back to %s", content.String()), nil +} + func revisionNotFoundErr(r int64) error { return fmt.Errorf("unable to find specified revision %v in history", r) } diff --git a/pkg/kubectl/sorting_printer.go b/pkg/kubectl/sorting_printer.go index 9226a213e2c24..f1330e3582bfa 100644 --- a/pkg/kubectl/sorting_printer.go +++ b/pkg/kubectl/sorting_printer.go @@ -188,7 +188,8 @@ func isLess(i, j reflect.Value) (bool, error) { // sort metav1.Time in := i.Interface() if t, ok := in.(metav1.Time); ok { - return t.Before(j.Interface().(metav1.Time)), nil + time := j.Interface().(metav1.Time) + return t.Before(&time), nil } // fallback to the fields comparison for idx := 0; idx < i.NumField(); idx++ { diff --git a/pkg/kubectl/util/i18n/i18n.go b/pkg/kubectl/util/i18n/i18n.go index aac3a304958bd..b2095a2090fc7 100644 --- a/pkg/kubectl/util/i18n/i18n.go +++ b/pkg/kubectl/util/i18n/i18n.go @@ -38,6 +38,7 @@ var knownTranslations = map[string][]string{ "zh_CN", "ja_JP", "zh_TW", + "it_IT", }, // only used for unit tests. "test": { diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 1f0aa84dceeea..979ef5e2d22fa 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -12,7 +12,6 @@ go_library( "active_deadline.go", "doc.go", "kubelet.go", - "kubelet_cadvisor.go", "kubelet_getters.go", "kubelet_network.go", "kubelet_node_status.go", @@ -79,6 +78,7 @@ go_library( "//pkg/kubelet/server/remotecommand:go_default_library", "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/server/streaming:go_default_library", + "//pkg/kubelet/stats:go_default_library", "//pkg/kubelet/status:go_default_library", "//pkg/kubelet/sysctl:go_default_library", "//pkg/kubelet/types:go_default_library", @@ -145,7 +145,6 @@ go_test( name = "go_default_test", srcs = [ "active_deadline_test.go", - "kubelet_cadvisor_test.go", "kubelet_getters_test.go", "kubelet_network_test.go", "kubelet_node_status_test.go", @@ -170,6 +169,7 @@ go_test( "//pkg/api:go_default_library", "//pkg/api/install:go_default_library", "//pkg/capabilities:go_default_library", + "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/apis/kubeletconfig:go_default_library", "//pkg/kubelet/cadvisor/testing:go_default_library", "//pkg/kubelet/cm:go_default_library", @@ -192,6 +192,7 @@ go_test( "//pkg/kubelet/server/portforward:go_default_library", "//pkg/kubelet/server/remotecommand:go_default_library", "//pkg/kubelet/server/stats:go_default_library", + "//pkg/kubelet/stats:go_default_library", "//pkg/kubelet/status:go_default_library", "//pkg/kubelet/status/testing:go_default_library", "//pkg/kubelet/types:go_default_library", @@ -274,6 +275,7 @@ filegroup( "//pkg/kubelet/rktshim:all-srcs", "//pkg/kubelet/secret:all-srcs", "//pkg/kubelet/server:all-srcs", + "//pkg/kubelet/stats:all-srcs", "//pkg/kubelet/status:all-srcs", "//pkg/kubelet/sysctl:all-srcs", "//pkg/kubelet/types:all-srcs", diff --git a/pkg/kubelet/apis/kubeletconfig/BUILD b/pkg/kubelet/apis/kubeletconfig/BUILD index 92d82340d23f4..07c85a597d6ec 100644 --- a/pkg/kubelet/apis/kubeletconfig/BUILD +++ b/pkg/kubelet/apis/kubeletconfig/BUILD @@ -36,7 +36,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", - "//pkg/kubelet/apis/kubeletconfig/install:all-srcs", + "//pkg/kubelet/apis/kubeletconfig/scheme:all-srcs", "//pkg/kubelet/apis/kubeletconfig/v1alpha1:all-srcs", "//pkg/kubelet/apis/kubeletconfig/validation:all-srcs", ], diff --git a/pkg/kubelet/apis/kubeletconfig/install/install.go b/pkg/kubelet/apis/kubeletconfig/install/install.go deleted file mode 100644 index 731791f9d0837..0000000000000 --- a/pkg/kubelet/apis/kubeletconfig/install/install.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package install installs the experimental API group, making it available as -// an option to all of the API encoding/decoding machinery. -package install - -import ( - "k8s.io/apimachinery/pkg/apimachinery/announced" - "k8s.io/apimachinery/pkg/apimachinery/registered" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" - "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" -) - -func init() { - // TODO(mtaufen): probably want to create a kubelet scheme rather than reusing the api scheme, but need to ask lavalamp - Install(api.GroupFactoryRegistry, api.Registry, api.Scheme) -} - -// Install registers the API group and adds types to a scheme -func Install(groupFactoryRegistry announced.APIGroupFactoryRegistry, registry *registered.APIRegistrationManager, scheme *runtime.Scheme) { - if err := announced.NewGroupMetaFactory( - &announced.GroupMetaFactoryArgs{ - GroupName: kubeletconfig.GroupName, - VersionPreferenceOrder: []string{v1alpha1.SchemeGroupVersion.Version}, - AddInternalObjectsToScheme: kubeletconfig.AddToScheme, - }, - announced.VersionToSchemeFunc{ - v1alpha1.SchemeGroupVersion.Version: v1alpha1.AddToScheme, - }, - ).Announce(groupFactoryRegistry).RegisterAndEnable(registry, scheme); err != nil { - panic(err) - } -} diff --git a/pkg/kubelet/apis/kubeletconfig/install/BUILD b/pkg/kubelet/apis/kubeletconfig/scheme/BUILD similarity index 55% rename from pkg/kubelet/apis/kubeletconfig/install/BUILD rename to pkg/kubelet/apis/kubeletconfig/scheme/BUILD index 111801fa0b6b9..44660bf9818f4 100644 --- a/pkg/kubelet/apis/kubeletconfig/install/BUILD +++ b/pkg/kubelet/apis/kubeletconfig/scheme/BUILD @@ -1,23 +1,14 @@ -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["install.go"], - tags = ["automanaged"], + srcs = ["scheme.go"], + visibility = ["//visibility:public"], deps = [ - "//pkg/api:go_default_library", "//pkg/kubelet/apis/kubeletconfig:go_default_library", "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", ], ) @@ -32,4 +23,5 @@ filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], + visibility = ["//visibility:public"], ) diff --git a/pkg/kubelet/apis/kubeletconfig/scheme/scheme.go b/pkg/kubelet/apis/kubeletconfig/scheme/scheme.go new file mode 100644 index 0000000000000..76f042818e6d7 --- /dev/null +++ b/pkg/kubelet/apis/kubeletconfig/scheme/scheme.go @@ -0,0 +1,40 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheme + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" +) + +// Utility functions for the Kubelet's kubeletconfig API group + +// NewSchemeAndCodecs is a utility funciton that returns a Scheme and CodecFactory +// that understand the types in the kubeletconfig API group. +func NewSchemeAndCodecs() (*runtime.Scheme, *serializer.CodecFactory, error) { + scheme := runtime.NewScheme() + if err := kubeletconfig.AddToScheme(scheme); err != nil { + return nil, nil, err + } + if err := v1alpha1.AddToScheme(scheme); err != nil { + return nil, nil, err + } + codecs := serializer.NewCodecFactory(scheme) + return scheme, &codecs, nil +} diff --git a/pkg/kubelet/apis/kubeletconfig/types.go b/pkg/kubelet/apis/kubeletconfig/types.go index 3845a139e368b..2f44149bcf716 100644 --- a/pkg/kubelet/apis/kubeletconfig/types.go +++ b/pkg/kubelet/apis/kubeletconfig/types.go @@ -56,17 +56,8 @@ type KubeletConfiguration struct { metav1.TypeMeta // Only used for dynamic configuration. - // The length of the trial period for this configuration. If the Kubelet records CrashLoopThreshold or - // more startups during this period, the current configuration will be marked bad and the - // Kubelet will roll-back to the last-known-good. Default 10 minutes. - ConfigTrialDuration metav1.Duration - // Only used for dynamic configuration. - // If this number of Kubelet "crashes" during ConfigTrialDuration meets this threshold, - // the configuration fails the trial and the Kubelet rolls back to its last-known-good config. - // Crash-loops are detected by counting Kubelet startups, so one startup is implicitly added - // to this threshold to always allow a single restart per config change. - // Default 10, mimimum allowed is 0, maximum allowed is 10. - CrashLoopThreshold int32 + // The length of the trial period for this configuration. This configuration will become the last-known-good after this duration. + ConfigTrialDuration *metav1.Duration // podManifestPath is the path to the directory containing pod manifests to // run, or the path to a single manifest file PodManifestPath string @@ -149,9 +140,9 @@ type KubeletConfiguration struct { // maxContainerCount is the maximum number of old instances of containers // to retain globally. Each container takes up some disk space. MaxContainerCount int32 - // cAdvisorPort is the port of the localhost cAdvisor endpoint + // cAdvisorPort is the port of the localhost cAdvisor endpoint (set to 0 to disable) CAdvisorPort int32 - // healthzPort is the port of the localhost healthz endpoint + // healthzPort is the port of the localhost healthz endpoint (set to 0 to disable) HealthzPort int32 // healthzBindAddress is the IP address for the healthz server to serve // on. @@ -358,7 +349,7 @@ type KubeletConfiguration struct { SystemReserved ConfigurationMap // A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) pairs // that describe resources reserved for kubernetes system components. - // Currently only cpu and memory are supported. [default=none] + // Currently cpu, memory and local storage for root file system are supported. [default=none] // See http://kubernetes.io/docs/user-guide/compute-resources for more detail. KubeReserved ConfigurationMap // This flag helps kubelet identify absolute name of top level cgroup used to enforce `SystemReserved` compute resource reservation for OS system daemons. diff --git a/pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go b/pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go index 47d240c2af141..5d82438f94b1c 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go +++ b/pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go @@ -32,6 +32,10 @@ import ( const ( DefaultRootDir = "/var/lib/kubelet" + // DEPRECATED: auto detecting cloud providers goes against the initiative + // for out-of-tree cloud providers as we'll now depend on cAdvisor integrations + // with cloud providers instead of in the core repo. + // More details here: https://github.com/kubernetes/kubernetes/issues/50986 AutoDetectCloudProvider = "auto-detect" defaultIPTablesMasqueradeBit = 14 @@ -53,9 +57,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.ConfigTrialDuration == nil { obj.ConfigTrialDuration = &metav1.Duration{Duration: 10 * time.Minute} } - if obj.CrashLoopThreshold == nil { - obj.CrashLoopThreshold = utilpointer.Int32Ptr(10) - } if obj.Authentication.Anonymous.Enabled == nil { obj.Authentication.Anonymous.Enabled = boolVar(true) } @@ -115,8 +116,8 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.HealthzBindAddress == "" { obj.HealthzBindAddress = "127.0.0.1" } - if obj.HealthzPort == 0 { - obj.HealthzPort = 10248 + if obj.HealthzPort == nil { + obj.HealthzPort = utilpointer.Int32Ptr(10248) } if obj.HostNetworkSources == nil { obj.HostNetworkSources = []string{kubetypes.AllSource} @@ -177,8 +178,8 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.Port == 0 { obj.Port = ports.KubeletPort } - if obj.ReadOnlyPort == 0 { - obj.ReadOnlyPort = ports.KubeletReadOnlyPort + if obj.ReadOnlyPort == nil { + obj.ReadOnlyPort = utilpointer.Int32Ptr(ports.KubeletReadOnlyPort) } if obj.RegisterNode == nil { obj.RegisterNode = boolVar(true) diff --git a/pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go b/pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go index d2ad06f28011e..679ffa21e2162 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go +++ b/pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go @@ -51,17 +51,8 @@ type KubeletConfiguration struct { metav1.TypeMeta `json:",inline"` // Only used for dynamic configuration. - // The length of the trial period for this configuration. If the Kubelet records CrashLoopThreshold or - // more startups during this period, the current configuration will be marked bad and the - // Kubelet will roll-back to the last-known-good. Default 10 minutes. + // The length of the trial period for this configuration. This configuration will become the last-known-good after this duration. ConfigTrialDuration *metav1.Duration `json:"configTrialDuration"` - // Only used for dynamic configuration. - // If this number of Kubelet "crashes" during ConfigTrialDuration meets this threshold, - // the configuration fails the trial and the Kubelet rolls back to its last-known-good config. - // Crash-loops are detected by counting Kubelet startups, so one startup is implicitly added - // to this threshold to always allow a single restart per config change. - // Default 10, mimimum allowed is 0, maximum allowed is 10. - CrashLoopThreshold *int32 `json:"crashLoopThreshold"` // podManifestPath is the path to the directory containing pod manifests to // run, or the path to a single manifest file PodManifestPath string `json:"podManifestPath"` @@ -87,7 +78,7 @@ type KubeletConfiguration struct { Port int32 `json:"port"` // readOnlyPort is the read-only port for the Kubelet to serve on with // no authentication/authorization (set to 0 to disable) - ReadOnlyPort int32 `json:"readOnlyPort"` + ReadOnlyPort *int32 `json:"readOnlyPort"` // tlsCertFile is the file containing x509 Certificate for HTTPS. (CA cert, // if any, concatenated after server cert). If tlsCertFile and // tlsPrivateKeyFile are not provided, a self-signed certificate @@ -144,10 +135,10 @@ type KubeletConfiguration struct { // maxContainerCount is the maximum number of old instances of containers // to retain globally. Each container takes up some disk space. MaxContainerCount *int32 `json:"maxContainerCount"` - // cAdvisorPort is the port of the localhost cAdvisor endpoint + // cAdvisorPort is the port of the localhost cAdvisor endpoint (set to 0 to disable) CAdvisorPort *int32 `json:"cAdvisorPort"` - // healthzPort is the port of the localhost healthz endpoint - HealthzPort int32 `json:"healthzPort"` + // healthzPort is the port of the localhost healthz endpoint (set to 0 to disable) + HealthzPort *int32 `json:"healthzPort"` // healthzBindAddress is the IP address for the healthz server to serve // on. HealthzBindAddress string `json:"healthzBindAddress"` diff --git a/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go b/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go index c3244579e8e38..36bd5b9231881 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go @@ -142,12 +142,7 @@ func Convert_kubeletconfig_KubeletAuthorization_To_v1alpha1_KubeletAuthorization } func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfiguration(in *KubeletConfiguration, out *kubeletconfig.KubeletConfiguration, s conversion.Scope) error { - if err := v1.Convert_Pointer_v1_Duration_To_v1_Duration(&in.ConfigTrialDuration, &out.ConfigTrialDuration, s); err != nil { - return err - } - if err := v1.Convert_Pointer_int32_To_int32(&in.CrashLoopThreshold, &out.CrashLoopThreshold, s); err != nil { - return err - } + out.ConfigTrialDuration = (*v1.Duration)(unsafe.Pointer(in.ConfigTrialDuration)) out.PodManifestPath = in.PodManifestPath out.SyncFrequency = in.SyncFrequency out.FileCheckFrequency = in.FileCheckFrequency @@ -159,7 +154,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura } out.Address = in.Address out.Port = in.Port - out.ReadOnlyPort = in.ReadOnlyPort + if err := v1.Convert_Pointer_int32_To_int32(&in.ReadOnlyPort, &out.ReadOnlyPort, s); err != nil { + return err + } out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile if err := Convert_v1alpha1_KubeletAuthentication_To_kubeletconfig_KubeletAuthentication(&in.Authentication, &out.Authentication, s); err != nil { @@ -195,7 +192,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura if err := v1.Convert_Pointer_int32_To_int32(&in.CAdvisorPort, &out.CAdvisorPort, s); err != nil { return err } - out.HealthzPort = in.HealthzPort + if err := v1.Convert_Pointer_int32_To_int32(&in.HealthzPort, &out.HealthzPort, s); err != nil { + return err + } out.HealthzBindAddress = in.HealthzBindAddress if err := v1.Convert_Pointer_int32_To_int32(&in.OOMScoreAdj, &out.OOMScoreAdj, s); err != nil { return err @@ -306,12 +305,7 @@ func Convert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfiguration } func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfiguration(in *kubeletconfig.KubeletConfiguration, out *KubeletConfiguration, s conversion.Scope) error { - if err := v1.Convert_v1_Duration_To_Pointer_v1_Duration(&in.ConfigTrialDuration, &out.ConfigTrialDuration, s); err != nil { - return err - } - if err := v1.Convert_int32_To_Pointer_int32(&in.CrashLoopThreshold, &out.CrashLoopThreshold, s); err != nil { - return err - } + out.ConfigTrialDuration = (*v1.Duration)(unsafe.Pointer(in.ConfigTrialDuration)) out.PodManifestPath = in.PodManifestPath out.SyncFrequency = in.SyncFrequency out.FileCheckFrequency = in.FileCheckFrequency @@ -323,7 +317,9 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura } out.Address = in.Address out.Port = in.Port - out.ReadOnlyPort = in.ReadOnlyPort + if err := v1.Convert_int32_To_Pointer_int32(&in.ReadOnlyPort, &out.ReadOnlyPort, s); err != nil { + return err + } out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile if err := Convert_kubeletconfig_KubeletAuthentication_To_v1alpha1_KubeletAuthentication(&in.Authentication, &out.Authentication, s); err != nil { @@ -336,21 +332,9 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura if err := v1.Convert_bool_To_Pointer_bool(&in.AllowPrivileged, &out.AllowPrivileged, s); err != nil { return err } - if in.HostNetworkSources == nil { - out.HostNetworkSources = make([]string, 0) - } else { - out.HostNetworkSources = *(*[]string)(unsafe.Pointer(&in.HostNetworkSources)) - } - if in.HostPIDSources == nil { - out.HostPIDSources = make([]string, 0) - } else { - out.HostPIDSources = *(*[]string)(unsafe.Pointer(&in.HostPIDSources)) - } - if in.HostIPCSources == nil { - out.HostIPCSources = make([]string, 0) - } else { - out.HostIPCSources = *(*[]string)(unsafe.Pointer(&in.HostIPCSources)) - } + out.HostNetworkSources = *(*[]string)(unsafe.Pointer(&in.HostNetworkSources)) + out.HostPIDSources = *(*[]string)(unsafe.Pointer(&in.HostPIDSources)) + out.HostIPCSources = *(*[]string)(unsafe.Pointer(&in.HostIPCSources)) if err := v1.Convert_int32_To_Pointer_int32(&in.RegistryPullQPS, &out.RegistryPullQPS, s); err != nil { return err } @@ -371,7 +355,9 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura if err := v1.Convert_int32_To_Pointer_int32(&in.CAdvisorPort, &out.CAdvisorPort, s); err != nil { return err } - out.HealthzPort = in.HealthzPort + if err := v1.Convert_int32_To_Pointer_int32(&in.HealthzPort, &out.HealthzPort, s); err != nil { + return err + } out.HealthzBindAddress = in.HealthzBindAddress if err := v1.Convert_int32_To_Pointer_int32(&in.OOMScoreAdj, &out.OOMScoreAdj, s); err != nil { return err @@ -381,11 +367,7 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura } out.ClusterDomain = in.ClusterDomain out.MasterServiceNamespace = in.MasterServiceNamespace - if in.ClusterDNS == nil { - out.ClusterDNS = make([]string, 0) - } else { - out.ClusterDNS = *(*[]string)(unsafe.Pointer(&in.ClusterDNS)) - } + out.ClusterDNS = *(*[]string)(unsafe.Pointer(&in.ClusterDNS)) out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency out.ImageMinimumGCAge = in.ImageMinimumGCAge @@ -428,11 +410,7 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura if err := v1.Convert_bool_To_Pointer_bool(&in.RegisterSchedulable, &out.RegisterSchedulable, s); err != nil { return err } - if in.RegisterWithTaints == nil { - out.RegisterWithTaints = make([]core_v1.Taint, 0) - } else { - out.RegisterWithTaints = *(*[]core_v1.Taint)(unsafe.Pointer(&in.RegisterWithTaints)) - } + out.RegisterWithTaints = *(*[]core_v1.Taint)(unsafe.Pointer(&in.RegisterWithTaints)) out.ContentType = in.ContentType if err := v1.Convert_int32_To_Pointer_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil { return err @@ -479,11 +457,7 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura out.KubeReserved = *(*map[string]string)(unsafe.Pointer(&in.KubeReserved)) out.SystemReservedCgroup = in.SystemReservedCgroup out.KubeReservedCgroup = in.KubeReservedCgroup - if in.EnforceNodeAllocatable == nil { - out.EnforceNodeAllocatable = make([]string, 0) - } else { - out.EnforceNodeAllocatable = *(*[]string)(unsafe.Pointer(&in.EnforceNodeAllocatable)) - } + out.EnforceNodeAllocatable = *(*[]string)(unsafe.Pointer(&in.EnforceNodeAllocatable)) out.ExperimentalNodeAllocatableIgnoreEvictionThreshold = in.ExperimentalNodeAllocatableIgnoreEvictionThreshold return nil } diff --git a/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go index a33c9c2cb0591..a6ba484b4f294 100644 --- a/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go @@ -143,15 +143,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { **out = **in } } - if in.CrashLoopThreshold != nil { - in, out := &in.CrashLoopThreshold, &out.CrashLoopThreshold - if *in == nil { - *out = nil - } else { - *out = new(int32) - **out = **in - } - } out.SyncFrequency = in.SyncFrequency out.FileCheckFrequency = in.FileCheckFrequency out.HTTPCheckFrequency = in.HTTPCheckFrequency @@ -164,6 +155,15 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { **out = **in } } + if in.ReadOnlyPort != nil { + in, out := &in.ReadOnlyPort, &out.ReadOnlyPort + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } in.Authentication.DeepCopyInto(&out.Authentication) out.Authorization = in.Authorization if in.AllowPrivileged != nil { @@ -236,6 +236,15 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { **out = **in } } + if in.HealthzPort != nil { + in, out := &in.HealthzPort, &out.HealthzPort + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } if in.OOMScoreAdj != nil { in, out := &in.OOMScoreAdj, &out.OOMScoreAdj if *in == nil { diff --git a/pkg/kubelet/apis/kubeletconfig/validation/validation.go b/pkg/kubelet/apis/kubeletconfig/validation/validation.go index 2c78b3c24dd00..519cef7f8c955 100644 --- a/pkg/kubelet/apis/kubeletconfig/validation/validation.go +++ b/pkg/kubelet/apis/kubeletconfig/validation/validation.go @@ -23,18 +23,8 @@ import ( containermanager "k8s.io/kubernetes/pkg/kubelet/cm" ) -// MaxCrashLoopThreshold is the maximum allowed KubeletConfiguraiton.CrashLoopThreshold -const MaxCrashLoopThreshold = 10 - // ValidateKubeletConfiguration validates `kc` and returns an error if it is invalid func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error { - // restrict crashloop threshold to between 0 and `maxCrashLoopThreshold`, inclusive - // more than `maxStartups=maxCrashLoopThreshold` adds unnecessary bloat to the .startups.json file, - // and negative values would be silly. - if kc.CrashLoopThreshold < 0 || kc.CrashLoopThreshold > MaxCrashLoopThreshold { - return fmt.Errorf("field `CrashLoopThreshold` must be between 0 and %d, inclusive", MaxCrashLoopThreshold) - } - if !kc.CgroupsPerQOS && len(kc.EnforceNodeAllocatable) > 0 { return fmt.Errorf("node allocatable enforcement is not supported unless Cgroups Per QOS feature is turned on") } diff --git a/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go b/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go index a6f81c152e796..3f9c00c2dadda 100644 --- a/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package kubeletconfig import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" api "k8s.io/kubernetes/pkg/api" @@ -124,7 +125,15 @@ func (in *KubeletAuthorization) DeepCopy() *KubeletAuthorization { func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = *in out.TypeMeta = in.TypeMeta - out.ConfigTrialDuration = in.ConfigTrialDuration + if in.ConfigTrialDuration != nil { + in, out := &in.ConfigTrialDuration, &out.ConfigTrialDuration + if *in == nil { + *out = nil + } else { + *out = new(v1.Duration) + **out = **in + } + } out.SyncFrequency = in.SyncFrequency out.FileCheckFrequency = in.FileCheckFrequency out.HTTPCheckFrequency = in.HTTPCheckFrequency diff --git a/pkg/kubelet/cadvisor/cadvisor_linux.go b/pkg/kubelet/cadvisor/cadvisor_linux.go index 77135b2871b1b..bcde373dcfccc 100644 --- a/pkg/kubelet/cadvisor/cadvisor_linux.go +++ b/pkg/kubelet/cadvisor/cadvisor_linux.go @@ -24,6 +24,7 @@ import ( "net" "net/http" "os" + "path" "strconv" "time" @@ -77,21 +78,29 @@ func init() { } func containerLabels(c *cadvisorapi.ContainerInfo) map[string]string { - set := map[string]string{metrics.LabelID: c.Name} + // Prometheus requires that all metrics in the same family have the same labels, + // so we arrange to supply blank strings for missing labels + var name, image, podName, namespace, containerName string if len(c.Aliases) > 0 { - set[metrics.LabelName] = c.Aliases[0] - } - if image := c.Spec.Image; len(image) > 0 { - set[metrics.LabelImage] = image + name = c.Aliases[0] } + image = c.Spec.Image if v, ok := c.Spec.Labels[types.KubernetesPodNameLabel]; ok { - set["pod_name"] = v + podName = v } if v, ok := c.Spec.Labels[types.KubernetesPodNamespaceLabel]; ok { - set["namespace"] = v + namespace = v } if v, ok := c.Spec.Labels[types.KubernetesContainerNameLabel]; ok { - set["container_name"] = v + containerName = v + } + set := map[string]string{ + metrics.LabelID: c.Name, + metrics.LabelName: name, + metrics.LabelImage: image, + "pod_name": podName, + "namespace": namespace, + "container_name": containerName, } return set } @@ -108,7 +117,9 @@ func New(address string, port uint, runtime string, rootPath string) (Interface, if _, err := os.Stat(rootPath); err != nil { if os.IsNotExist(err) { - return nil, fmt.Errorf("rootDirectory %q does not exist", rootPath) + if err := os.MkdirAll(path.Clean(rootPath), 0750); err != nil { + return nil, fmt.Errorf("error creating root directory %q: %v", rootPath, err) + } } else { return nil, fmt.Errorf("failed to Stat %q: %v", rootPath, err) } diff --git a/pkg/kubelet/cadvisor/util.go b/pkg/kubelet/cadvisor/util.go index e1d0c90f27c88..a95a8fb19e29a 100644 --- a/pkg/kubelet/cadvisor/util.go +++ b/pkg/kubelet/cadvisor/util.go @@ -35,18 +35,9 @@ func CapacityFromMachineInfo(info *cadvisorapi.MachineInfo) v1.ResourceList { return c } -func StorageScratchCapacityFromFsInfo(info cadvisorapi2.FsInfo) v1.ResourceList { +func EphemeralStorageCapacityFromFsInfo(info cadvisorapi2.FsInfo) v1.ResourceList { c := v1.ResourceList{ - v1.ResourceStorageScratch: *resource.NewQuantity( - int64(info.Capacity), - resource.BinarySI), - } - return c -} - -func StorageOverlayCapacityFromFsInfo(info cadvisorapi2.FsInfo) v1.ResourceList { - c := v1.ResourceList{ - v1.ResourceStorageOverlay: *resource.NewQuantity( + v1.ResourceEphemeralStorage: *resource.NewQuantity( int64(info.Capacity), resource.BinarySI), } diff --git a/pkg/kubelet/certificate/certificate_store.go b/pkg/kubelet/certificate/certificate_store.go index 5230f1ab9b343..a98a9031978d0 100644 --- a/pkg/kubelet/certificate/certificate_store.go +++ b/pkg/kubelet/certificate/certificate_store.go @@ -101,7 +101,7 @@ func (s *fileStore) recover() error { } return err } else if fi.Mode()&os.ModeSymlink != os.ModeSymlink { - return fmt.Errorf("expected %q to be a symlink but it is a file.", updatedPath) + return fmt.Errorf("expected %q to be a symlink but it is a file", updatedPath) } // Move the 'updated' symlink to 'current'. @@ -184,7 +184,7 @@ func loadCertKeyBlocks(pairFile string) (cert *pem.Block, key *pem.Block, err er if certBlock == nil { return nil, nil, fmt.Errorf("could not decode the first block from %q from expected PEM format", pairFile) } - keyBlock, rest := pem.Decode(rest) + keyBlock, _ := pem.Decode(rest) if keyBlock == nil { return nil, nil, fmt.Errorf("could not decode the second block from %q from expected PEM format", pairFile) } diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index cfd2f6e60c806..51b892c222ccb 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -55,7 +55,6 @@ go_library( "//pkg/util/procfs:go_default_library", "//pkg/util/sysctl:go_default_library", "//pkg/util/version:go_default_library", - "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs:go_default_library", "//vendor/github.com/opencontainers/runc/libcontainer/cgroups/systemd:go_default_library", @@ -109,6 +108,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//pkg/kubelet/cm/cpuset:all-srcs", "//pkg/kubelet/cm/util:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/kubelet/cm/cgroup_manager_linux.go b/pkg/kubelet/cm/cgroup_manager_linux.go index 7657dcca15e65..c210dc2f60534 100644 --- a/pkg/kubelet/cm/cgroup_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_manager_linux.go @@ -233,7 +233,7 @@ func (m *cgroupManagerImpl) Exists(name CgroupName) bool { // scoped to the set control groups it understands. this is being discussed // in https://github.com/opencontainers/runc/issues/1440 // once resolved, we can remove this code. - whitelistControllers := sets.NewString("cpu", "cpuacct", "cpuset", "memory", "hugetlb", "systemd") + whitelistControllers := sets.NewString("cpu", "cpuacct", "cpuset", "memory", "systemd") // If even one cgroup path doesn't exist, then the cgroup doesn't exist. for controller, path := range cgroupPaths { diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 8ba7d91942914..66128398e3bd7 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -30,7 +30,6 @@ import ( "time" "github.com/golang/glog" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/cgroups/fs" "github.com/opencontainers/runc/libcontainer/configs" @@ -552,24 +551,11 @@ func (cm *containerManagerImpl) setFsCapacity() error { if err != nil { return fmt.Errorf("Fail to get rootfs information %v", err) } - hasDedicatedImageFs, _ := cm.cadvisorInterface.HasDedicatedImageFs() - var imagesfs cadvisorapiv2.FsInfo - if hasDedicatedImageFs { - imagesfs, err = cm.cadvisorInterface.ImagesFsInfo() - if err != nil { - return fmt.Errorf("Fail to get imagefs information %v", err) - } - } cm.Lock() - for rName, rCap := range cadvisor.StorageScratchCapacityFromFsInfo(rootfs) { + for rName, rCap := range cadvisor.EphemeralStorageCapacityFromFsInfo(rootfs) { cm.capacity[rName] = rCap } - if hasDedicatedImageFs { - for rName, rCap := range cadvisor.StorageOverlayCapacityFromFsInfo(imagesfs) { - cm.capacity[rName] = rCap - } - } cm.Unlock() return nil } diff --git a/pkg/kubelet/cm/cpuset/BUILD b/pkg/kubelet/cm/cpuset/BUILD new file mode 100644 index 0000000000000..84f1beebd153c --- /dev/null +++ b/pkg/kubelet/cm/cpuset/BUILD @@ -0,0 +1,36 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["cpuset_test.go"], + library = ":go_default_library", + tags = ["automanaged"], +) + +go_library( + name = "go_default_library", + srcs = ["cpuset.go"], + tags = ["automanaged"], + deps = ["//vendor/github.com/golang/glog:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/kubelet/cm/cpuset/OWNERS b/pkg/kubelet/cm/cpuset/OWNERS new file mode 100644 index 0000000000000..02f72bf6cc1ab --- /dev/null +++ b/pkg/kubelet/cm/cpuset/OWNERS @@ -0,0 +1,5 @@ +approvers: +- derekwaynecarr +- vishh +- ConnorDoyle +- sjenning diff --git a/pkg/kubelet/cm/cpuset/cpuset.go b/pkg/kubelet/cm/cpuset/cpuset.go new file mode 100644 index 0000000000000..ccf577d819cb7 --- /dev/null +++ b/pkg/kubelet/cm/cpuset/cpuset.go @@ -0,0 +1,280 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cpuset + +import ( + "bytes" + "fmt" + "github.com/golang/glog" + "reflect" + "sort" + "strconv" + "strings" +) + +// Builder is a mutable builder for CPUSet. Functions that mutate instances +// of this type are not thread-safe. +type Builder struct { + result CPUSet + done bool +} + +// NewBuilder returns a mutable CPUSet builder. +func NewBuilder() Builder { + return Builder{ + result: CPUSet{ + elems: map[int]struct{}{}, + }, + } +} + +// Add adds the supplied elements to the result. Calling Add after calling +// Result has no effect. +func (b Builder) Add(elems ...int) { + if b.done { + return + } + for _, elem := range elems { + b.result.elems[elem] = struct{}{} + } +} + +// Result returns the result CPUSet containing all elements that were +// previously added to this builder. Subsequent calls to Add have no effect. +func (b Builder) Result() CPUSet { + b.done = true + return b.result +} + +// CPUSet is a thread-safe, immutable set-like data structure for CPU IDs. +type CPUSet struct { + elems map[int]struct{} +} + +// NewCPUSet returns a new CPUSet containing the supplied elements. +func NewCPUSet(cpus ...int) CPUSet { + b := NewBuilder() + for _, c := range cpus { + b.Add(c) + } + return b.Result() +} + +// Size returns the number of elements in this set. +func (s CPUSet) Size() int { + return len(s.elems) +} + +// IsEmpty returns true if there are zero elements in this set. +func (s CPUSet) IsEmpty() bool { + return s.Size() == 0 +} + +// Contains returns true if the supplied element is present in this set. +func (s CPUSet) Contains(cpu int) bool { + _, found := s.elems[cpu] + return found +} + +// Equals returns true if the supplied set contains exactly the same elements +// as this set (s IsSubsetOf s2 and s2 IsSubsetOf s). +func (s CPUSet) Equals(s2 CPUSet) bool { + return reflect.DeepEqual(s.elems, s2.elems) +} + +// Filter returns a new CPU set that contains all of the elements from this +// set that match the supplied predicate, without mutating the source set. +func (s CPUSet) Filter(predicate func(int) bool) CPUSet { + b := NewBuilder() + for cpu := range s.elems { + if predicate(cpu) { + b.Add(cpu) + } + } + return b.Result() +} + +// FilterNot returns a new CPU set that contains all of the elements from this +// set that do not match the supplied predicate, without mutating the source +// set. +func (s CPUSet) FilterNot(predicate func(int) bool) CPUSet { + b := NewBuilder() + for cpu := range s.elems { + if !predicate(cpu) { + b.Add(cpu) + } + } + return b.Result() +} + +// IsSubsetOf returns true if the supplied set contains all the elements +func (s CPUSet) IsSubsetOf(s2 CPUSet) bool { + result := true + for cpu := range s.elems { + if !s2.Contains(cpu) { + result = false + break + } + } + return result +} + +// Union returns a new CPU set that contains all of the elements from this +// set and all of the elements from the supplied set, without mutating +// either source set. +func (s CPUSet) Union(s2 CPUSet) CPUSet { + b := NewBuilder() + for cpu := range s.elems { + b.Add(cpu) + } + for cpu := range s2.elems { + b.Add(cpu) + } + return b.Result() +} + +// Intersection returns a new CPU set that contains all of the elements +// that are present in both this set and the supplied set, without mutating +// either source set. +func (s CPUSet) Intersection(s2 CPUSet) CPUSet { + return s.Filter(func(cpu int) bool { return s2.Contains(cpu) }) +} + +// Difference returns a new CPU set that contains all of the elements that +// are present in this set and not the supplied set, without mutating either +// source set. +func (s CPUSet) Difference(s2 CPUSet) CPUSet { + return s.FilterNot(func(cpu int) bool { return s2.Contains(cpu) }) +} + +// ToSlice returns a slice of integers that contains all elements from +// this set. +func (s CPUSet) ToSlice() []int { + result := []int{} + for cpu := range s.elems { + result = append(result, cpu) + } + sort.Ints(result) + return result +} + +// String returns a new string representation of the elements in this CPU set +// in canonical linux CPU list format. +// +// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS +func (s CPUSet) String() string { + if s.IsEmpty() { + return "" + } + + elems := s.ToSlice() + + type rng struct { + start int + end int + } + + ranges := []rng{{elems[0], elems[0]}} + + for i := 1; i < len(elems); i++ { + lastRange := &ranges[len(ranges)-1] + // if this element is adjacent to the high end of the last range + if elems[i] == lastRange.end+1 { + // then extend the last range to include this element + lastRange.end = elems[i] + continue + } + // otherwise, start a new range beginning with this element + ranges = append(ranges, rng{elems[i], elems[i]}) + } + + // construct string from ranges + var result bytes.Buffer + for _, r := range ranges { + if r.start == r.end { + result.WriteString(strconv.Itoa(r.start)) + } else { + result.WriteString(fmt.Sprintf("%d-%d", r.start, r.end)) + } + result.WriteString(",") + } + return strings.TrimRight(result.String(), ",") +} + +// MustParse CPUSet constructs a new CPU set from a Linux CPU list formatted +// string. Unlike Parse, it does not return an error but rather panics if the +// input cannot be used to construct a CPU set. +func MustParse(s string) CPUSet { + res, err := Parse(s) + if err != nil { + glog.Fatalf("unable to parse [%s] as CPUSet: %v", s, err) + } + return res +} + +// Parse CPUSet constructs a new CPU set from a Linux CPU list formatted string. +// +// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS +func Parse(s string) (CPUSet, error) { + b := NewBuilder() + + // Handle empty string. + if s == "" { + return b.Result(), nil + } + + // Split CPU list string: + // "0-5,34,46-48 => ["0-5", "34", "46-48"] + ranges := strings.Split(s, ",") + + for _, r := range ranges { + boundaries := strings.Split(r, "-") + if len(boundaries) == 1 { + // Handle ranges that consist of only one element like "34". + elem, err := strconv.Atoi(boundaries[0]) + if err != nil { + return NewCPUSet(), err + } + b.Add(elem) + } else if len(boundaries) == 2 { + // Handle multi-element ranges like "0-5". + start, err := strconv.Atoi(boundaries[0]) + if err != nil { + return NewCPUSet(), err + } + end, err := strconv.Atoi(boundaries[1]) + if err != nil { + return NewCPUSet(), err + } + // Add all elements to the result. + // e.g. "0-5", "46-48" => [0, 1, 2, 3, 4, 5, 46, 47, 48]. + for e := start; e <= end; e++ { + b.Add(e) + } + } + } + return b.Result(), nil +} + +// Clone returns a copy of this CPU set. +func (s CPUSet) Clone() CPUSet { + b := NewBuilder() + for elem := range s.elems { + b.Add(elem) + } + return b.Result() +} diff --git a/pkg/kubelet/cm/cpuset/cpuset_test.go b/pkg/kubelet/cm/cpuset/cpuset_test.go new file mode 100644 index 0000000000000..e2d2410bbca9e --- /dev/null +++ b/pkg/kubelet/cm/cpuset/cpuset_test.go @@ -0,0 +1,324 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cpuset + +import ( + "reflect" + "testing" +) + +func TestCPUSetBuilder(t *testing.T) { + b := NewBuilder() + elems := []int{1, 2, 3, 4, 5} + for _, elem := range elems { + b.Add(elem) + } + result := b.Result() + for _, elem := range elems { + if !result.Contains(elem) { + t.Fatalf("expected cpuset to contain element %d: [%v]", elem, result) + } + } + if len(elems) != result.Size() { + t.Fatalf("expected cpuset %s to have the same size as %v", result, elems) + } +} + +func TestCPUSetSize(t *testing.T) { + testCases := []struct { + cpuset CPUSet + expected int + }{ + {NewCPUSet(), 0}, + {NewCPUSet(5), 1}, + {NewCPUSet(1, 2, 3, 4, 5), 5}, + } + + for _, c := range testCases { + actual := c.cpuset.Size() + if actual != c.expected { + t.Fatalf("expected: %d, actual: %d, cpuset: [%v]", c.expected, actual, c.cpuset) + } + } +} + +func TestCPUSetIsEmpty(t *testing.T) { + testCases := []struct { + cpuset CPUSet + expected bool + }{ + {NewCPUSet(), true}, + {NewCPUSet(5), false}, + {NewCPUSet(1, 2, 3, 4, 5), false}, + } + + for _, c := range testCases { + actual := c.cpuset.IsEmpty() + if actual != c.expected { + t.Fatalf("expected: %t, IsEmpty() returned: %t, cpuset: [%v]", c.expected, actual, c.cpuset) + } + } +} + +func TestCPUSetContains(t *testing.T) { + testCases := []struct { + cpuset CPUSet + mustContain []int + mustNotContain []int + }{ + {NewCPUSet(), []int{}, []int{1, 2, 3, 4, 5}}, + {NewCPUSet(5), []int{5}, []int{1, 2, 3, 4}}, + {NewCPUSet(1, 2, 4, 5), []int{1, 2, 4, 5}, []int{0, 3, 6}}, + } + + for _, c := range testCases { + for _, elem := range c.mustContain { + if !c.cpuset.Contains(elem) { + t.Fatalf("expected cpuset to contain element %d: [%v]", elem, c.cpuset) + } + } + for _, elem := range c.mustNotContain { + if c.cpuset.Contains(elem) { + t.Fatalf("expected cpuset not to contain element %d: [%v]", elem, c.cpuset) + } + } + } +} + +func TestCPUSetEqual(t *testing.T) { + shouldEqual := []struct { + s1 CPUSet + s2 CPUSet + }{ + {NewCPUSet(), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + } + + shouldNotEqual := []struct { + s1 CPUSet + s2 CPUSet + }{ + {NewCPUSet(), NewCPUSet(5)}, + {NewCPUSet(5), NewCPUSet()}, + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5)}, + } + + for _, c := range shouldEqual { + if !c.s1.Equals(c.s2) { + t.Fatalf("expected cpusets to be equal: s1: [%v], s2: [%v]", c.s1, c.s2) + } + } + for _, c := range shouldNotEqual { + if c.s1.Equals(c.s2) { + t.Fatalf("expected cpusets to not be equal: s1: [%v], s2: [%v]", c.s1, c.s2) + } + } +} + +func TestCPUSetIsSubsetOf(t *testing.T) { + shouldBeSubset := []struct { + s1 CPUSet + s2 CPUSet + }{ + // A set is a subset of itself + {NewCPUSet(), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + + // Empty set is a subset of every set + {NewCPUSet(), NewCPUSet(5)}, + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)}, + + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(2, 3), NewCPUSet(1, 2, 3, 4, 5)}, + } + + shouldNotBeSubset := []struct { + s1 CPUSet + s2 CPUSet + }{} + + for _, c := range shouldBeSubset { + if !c.s1.IsSubsetOf(c.s2) { + t.Fatalf("expected s1 to be a subset of s2: s1: [%v], s2: [%v]", c.s1, c.s2) + } + } + for _, c := range shouldNotBeSubset { + if c.s1.IsSubsetOf(c.s2) { + t.Fatalf("expected s1 to not be a subset of s2: s1: [%v], s2: [%v]", c.s1, c.s2) + } + } +} + +func TestCPUSetUnion(t *testing.T) { + testCases := []struct { + s1 CPUSet + s2 CPUSet + expected CPUSet + }{ + {NewCPUSet(), NewCPUSet(), NewCPUSet()}, + + {NewCPUSet(), NewCPUSet(5), NewCPUSet(5)}, + {NewCPUSet(5), NewCPUSet(), NewCPUSet(5)}, + {NewCPUSet(5), NewCPUSet(5), NewCPUSet(5)}, + + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5)}, + + {NewCPUSet(1, 2), NewCPUSet(3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3), NewCPUSet(3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + } + + for _, c := range testCases { + result := c.s1.Union(c.s2) + if !result.Equals(c.expected) { + t.Fatalf("expected the union of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2) + } + } +} + +func TestCPUSetIntersection(t *testing.T) { + testCases := []struct { + s1 CPUSet + s2 CPUSet + expected CPUSet + }{ + {NewCPUSet(), NewCPUSet(), NewCPUSet()}, + + {NewCPUSet(), NewCPUSet(5), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(5), NewCPUSet(5)}, + + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet()}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5), NewCPUSet(5)}, + + {NewCPUSet(1, 2), NewCPUSet(3, 4, 5), NewCPUSet()}, + {NewCPUSet(1, 2, 3), NewCPUSet(3, 4, 5), NewCPUSet(3)}, + } + + for _, c := range testCases { + result := c.s1.Intersection(c.s2) + if !result.Equals(c.expected) { + t.Fatalf("expected the intersection of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2) + } + } +} + +func TestCPUSetDifference(t *testing.T) { + testCases := []struct { + s1 CPUSet + s2 CPUSet + expected CPUSet + }{ + {NewCPUSet(), NewCPUSet(), NewCPUSet()}, + + {NewCPUSet(), NewCPUSet(5), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(), NewCPUSet(5)}, + {NewCPUSet(5), NewCPUSet(5), NewCPUSet()}, + + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5), NewCPUSet(1, 2, 3, 4)}, + + {NewCPUSet(1, 2), NewCPUSet(3, 4, 5), NewCPUSet(1, 2)}, + {NewCPUSet(1, 2, 3), NewCPUSet(3, 4, 5), NewCPUSet(1, 2)}, + } + + for _, c := range testCases { + result := c.s1.Difference(c.s2) + if !result.Equals(c.expected) { + t.Fatalf("expected the difference of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2) + } + } +} + +func TestCPUSetToSlice(t *testing.T) { + testCases := []struct { + set CPUSet + expected []int + }{ + {NewCPUSet(), []int{}}, + {NewCPUSet(5), []int{5}}, + {NewCPUSet(1, 2, 3, 4, 5), []int{1, 2, 3, 4, 5}}, + } + + for _, c := range testCases { + result := c.set.ToSlice() + if !reflect.DeepEqual(result, c.expected) { + t.Fatalf("expected set as slice to be [%v] (got [%v]), s: [%v]", c.expected, result, c.set) + } + } +} + +func TestCPUSetString(t *testing.T) { + testCases := []struct { + set CPUSet + expected string + }{ + {NewCPUSet(), ""}, + {NewCPUSet(5), "5"}, + {NewCPUSet(1, 2, 3, 4, 5), "1-5"}, + {NewCPUSet(1, 2, 3, 5, 6, 8), "1-3,5-6,8"}, + } + + for _, c := range testCases { + result := c.set.String() + if result != c.expected { + t.Fatalf("expected set as string to be %s (got \"%s\"), s: [%v]", c.expected, result, c.set) + } + } +} + +func TestParse(t *testing.T) { + testCases := []struct { + cpusetString string + expected CPUSet + }{ + {"", NewCPUSet()}, + {"5", NewCPUSet(5)}, + {"1,2,3,4,5", NewCPUSet(1, 2, 3, 4, 5)}, + {"1-5", NewCPUSet(1, 2, 3, 4, 5)}, + {"1-2,3-5", NewCPUSet(1, 2, 3, 4, 5)}, + } + + for _, c := range testCases { + result, err := Parse(c.cpusetString) + if err != nil { + t.Fatalf("expected error not to have occurred: %v", err) + } + if !result.Equals(c.expected) { + t.Fatalf("expected string \"%s\" to parse as [%v] (got [%v])", c.cpusetString, c.expected, result) + } + } +} diff --git a/pkg/kubelet/cm/node_container_manager.go b/pkg/kubelet/cm/node_container_manager.go index 7c817984b0712..a96f9c5403790 100644 --- a/pkg/kubelet/cm/node_container_manager.go +++ b/pkg/kubelet/cm/node_container_manager.go @@ -218,9 +218,9 @@ func hardEvictionReservation(thresholds []evictionapi.Threshold, capacity v1.Res value := evictionapi.GetThresholdQuantity(threshold.Value, &memoryCapacity) ret[v1.ResourceMemory] = *value case evictionapi.SignalNodeFsAvailable: - storageCapacity := capacity[v1.ResourceStorageScratch] + storageCapacity := capacity[v1.ResourceEphemeralStorage] value := evictionapi.GetThresholdQuantity(threshold.Value, &storageCapacity) - ret[v1.ResourceStorageScratch] = *value + ret[v1.ResourceEphemeralStorage] = *value } } return ret diff --git a/pkg/kubelet/cm/node_container_manager_test.go b/pkg/kubelet/cm/node_container_manager_test.go index 29208186abc68..c06b9aa85be57 100644 --- a/pkg/kubelet/cm/node_container_manager_test.go +++ b/pkg/kubelet/cm/node_container_manager_test.go @@ -316,17 +316,17 @@ func TestNodeAllocatableInputValidation(t *testing.T) { invalidConfiguration bool }{ { - kubeReserved: getScratchResourceList("100Mi"), - systemReserved: getScratchResourceList("50Mi"), - capacity: getScratchResourceList("500Mi"), + kubeReserved: getEphemeralStorageResourceList("100Mi"), + systemReserved: getEphemeralStorageResourceList("50Mi"), + capacity: getEphemeralStorageResourceList("500Mi"), }, { - kubeReserved: getScratchResourceList("10Gi"), - systemReserved: getScratchResourceList("10Gi"), + kubeReserved: getEphemeralStorageResourceList("10Gi"), + systemReserved: getEphemeralStorageResourceList("10Gi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &storageEvictionThreshold, }, - capacity: getScratchResourceList("20Gi"), + capacity: getEphemeralStorageResourceList("20Gi"), invalidConfiguration: true, }, } @@ -359,12 +359,12 @@ func TestNodeAllocatableInputValidation(t *testing.T) { } } -// getScratchResourceList returns a ResourceList with the -// specified scratch storage resource values -func getScratchResourceList(storage string) v1.ResourceList { +// getEphemeralStorageResourceList returns a ResourceList with the +// specified ephemeral storage resource values +func getEphemeralStorageResourceList(storage string) v1.ResourceList { res := v1.ResourceList{} if storage != "" { - res[v1.ResourceStorageScratch] = resource.MustParse(storage) + res[v1.ResourceEphemeralStorage] = resource.MustParse(storage) } return res } diff --git a/pkg/kubelet/container/BUILD b/pkg/kubelet/container/BUILD index 6f68454ab62b1..78c840846a9fb 100644 --- a/pkg/kubelet/container/BUILD +++ b/pkg/kubelet/container/BUILD @@ -1,10 +1,4 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -27,6 +21,7 @@ go_library( ], "//conditions:default": [], }), + visibility = ["//visibility:public"], deps = [ "//pkg/api:go_default_library", "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library", @@ -96,4 +91,5 @@ filegroup( "//pkg/kubelet/container/testing:all-srcs", ], tags = ["automanaged"], + visibility = ["//visibility:public"], ) diff --git a/pkg/kubelet/container/testing/BUILD b/pkg/kubelet/container/testing/BUILD index 3efa80948c6e8..deb39b6aa59d9 100644 --- a/pkg/kubelet/container/testing/BUILD +++ b/pkg/kubelet/container/testing/BUILD @@ -1,9 +1,4 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", @@ -11,10 +6,12 @@ go_library( "fake_cache.go", "fake_runtime.go", "fake_runtime_helper.go", + "mock_runtime_cache.go", "mockfileinfo.go", "os.go", "runtime_mock.go", ], + visibility = ["//visibility:public"], deps = [ "//pkg/kubelet/container:go_default_library", "//pkg/volume:go_default_library", @@ -38,4 +35,5 @@ filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], + visibility = ["//visibility:public"], ) diff --git a/pkg/kubelet/container/testing/mock_runtime_cache.go b/pkg/kubelet/container/testing/mock_runtime_cache.go new file mode 100644 index 0000000000000..0267764c90b58 --- /dev/null +++ b/pkg/kubelet/container/testing/mock_runtime_cache.go @@ -0,0 +1,64 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by mockery v1.0.0 +package testing + +import container "k8s.io/kubernetes/pkg/kubelet/container" +import mock "github.com/stretchr/testify/mock" +import time "time" + +// MockRuntimeCache is an autogenerated mock type for the RuntimeCache type +type MockRuntimeCache struct { + mock.Mock +} + +// ForceUpdateIfOlder provides a mock function with given fields: _a0 +func (_m *MockRuntimeCache) ForceUpdateIfOlder(_a0 time.Time) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(time.Time) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetPods provides a mock function with given fields: +func (_m *MockRuntimeCache) GetPods() ([]*container.Pod, error) { + ret := _m.Called() + + var r0 []*container.Pod + if rf, ok := ret.Get(0).(func() []*container.Pod); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*container.Pod) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/pkg/kubelet/dockershim/exec.go b/pkg/kubelet/dockershim/exec.go index 6d4b229cd8652..a861bb47f0e04 100644 --- a/pkg/kubelet/dockershim/exec.go +++ b/pkg/kubelet/dockershim/exec.go @@ -135,6 +135,9 @@ func (*NsenterExecHandler) ExecInContainer(client libdocker.Interface, container type NativeExecHandler struct{} func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container *dockertypes.ContainerJSON, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { + done := make(chan struct{}) + defer close(done) + createOpts := dockertypes.ExecConfig{ Cmd: cmd, AttachStdin: stdin != nil, @@ -149,9 +152,23 @@ func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container // Have to start this before the call to client.StartExec because client.StartExec is a blocking // call :-( Otherwise, resize events don't get processed and the terminal never resizes. - kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) { - client.ResizeExecTTY(execObj.ID, uint(size.Height), uint(size.Width)) - }) + // + // We also have to delay attempting to send a terminal resize request to docker until after the + // exec has started; otherwise, the initial resize request will fail. + execStarted := make(chan struct{}) + go func() { + select { + case <-execStarted: + // client.StartExec has started the exec, so we can start resizing + case <-done: + // ExecInContainer has returned, so short-circuit + return + } + + kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) { + client.ResizeExecTTY(execObj.ID, uint(size.Height), uint(size.Width)) + }) + }() startOpts := dockertypes.ExecStartCheck{Detach: false, Tty: tty} streamOpts := libdocker.StreamOptions{ @@ -159,6 +176,7 @@ func (*NativeExecHandler) ExecInContainer(client libdocker.Interface, container OutputStream: stdout, ErrorStream: stderr, RawTerminal: tty, + ExecStarted: execStarted, } err = client.StartExec(execObj.ID, startOpts, streamOpts) if err != nil { diff --git a/pkg/kubelet/dockershim/libdocker/kube_docker_client.go b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go index 7a2f4e6d66bc5..64a974dda4866 100644 --- a/pkg/kubelet/dockershim/libdocker/kube_docker_client.go +++ b/pkg/kubelet/dockershim/libdocker/kube_docker_client.go @@ -464,6 +464,15 @@ func (d *kubeDockerClient) StartExec(startExec string, opts dockertypes.ExecStar return err } defer resp.Close() + + if sopts.ExecStarted != nil { + // Send a message to the channel indicating that the exec has started. This is needed so + // interactive execs can handle resizing correctly - the request to resize the TTY has to happen + // after the call to d.client.ContainerExecAttach, and because d.holdHijackedConnection below + // blocks, we use sopts.ExecStarted to signal the caller that it's ok to resize. + sopts.ExecStarted <- struct{}{} + } + return d.holdHijackedConnection(sopts.RawTerminal || opts.Tty, sopts.InputStream, sopts.OutputStream, sopts.ErrorStream, resp) } @@ -594,6 +603,7 @@ type StreamOptions struct { InputStream io.Reader OutputStream io.Writer ErrorStream io.Writer + ExecStarted chan struct{} } // operationTimeout is the error returned when the docker operations are timeout. diff --git a/pkg/kubelet/events/event.go b/pkg/kubelet/events/event.go index 28cddf4a901c3..b8cae2cfad955 100644 --- a/pkg/kubelet/events/event.go +++ b/pkg/kubelet/events/event.go @@ -46,6 +46,7 @@ const ( FailedDetachVolume = "FailedDetachVolume" FailedMountVolume = "FailedMount" FailedUnMountVolume = "FailedUnMount" + WarnAlreadyMountedVolume = "AlreadyMountedVolume" SuccessfulDetachVolume = "SuccessfulDetachVolume" SuccessfulMountVolume = "SuccessfulMountVolume" SuccessfulUnMountVolume = "SuccessfulUnMountVolume" @@ -62,6 +63,7 @@ const ( SuccessfulNodeAllocatableEnforcement = "NodeAllocatableEnforced" UnsupportedMountOption = "UnsupportedMountOption" SandboxChanged = "SandboxChanged" + FailedCreatePodSandBox = "FailedCreatePodSandBox" // Image manager event reason list InvalidDiskCapacity = "InvalidDiskCapacity" diff --git a/pkg/kubelet/eviction/BUILD b/pkg/kubelet/eviction/BUILD index 0d796fdd3a7db..69b373533fd53 100644 --- a/pkg/kubelet/eviction/BUILD +++ b/pkg/kubelet/eviction/BUILD @@ -48,6 +48,7 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/api/v1/helper/qos:go_default_library", + "//pkg/api/v1/resource:go_default_library", "//pkg/features:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cm:go_default_library", diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index 5841afe7cdd39..f430b82502bcb 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -31,6 +31,7 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" v1qos "k8s.io/kubernetes/pkg/api/v1/helper/qos" + apiv1resource "k8s.io/kubernetes/pkg/api/v1/resource" "k8s.io/kubernetes/pkg/features" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm" @@ -472,7 +473,12 @@ func (m *managerImpl) localStorageEviction(pods []*v1.Pod) []*v1.Pod { continue } - if m.containerOverlayLimitEviction(podStats, pod) { + if m.podEphemeralStorageLimitEviction(podStats, pod) { + evicted = append(evicted, pod) + continue + } + + if m.containerEphemeralStorageLimitEviction(podStats, pod) { evicted = append(evicted, pod) } } @@ -496,23 +502,56 @@ func (m *managerImpl) emptyDirLimitEviction(podStats statsapi.PodStats, pod *v1. } } } + + return false +} + +func (m *managerImpl) podEphemeralStorageLimitEviction(podStats statsapi.PodStats, pod *v1.Pod) bool { + _, podLimits := apiv1resource.PodRequestsAndLimits(pod) + _, found := podLimits[v1.ResourceEphemeralStorage] + if !found { + return false + } + + podEphemeralStorageTotalUsage := &resource.Quantity{} + fsStatsSet := []fsStatsType{} + if *m.dedicatedImageFs { + fsStatsSet = []fsStatsType{fsStatsLogs, fsStatsLocalVolumeSource} + } else { + fsStatsSet = []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource} + } + podUsage, err := podDiskUsage(podStats, pod, fsStatsSet) + if err != nil { + glog.Errorf("eviction manager: error getting pod disk usage %v", err) + return false + } + + podEphemeralStorageTotalUsage.Add(podUsage[resourceDisk]) + if podEphemeralStorageTotalUsage.Cmp(podLimits[v1.ResourceEphemeralStorage]) > 0 { + // the total usage of pod exceeds the total size limit of containers, evict the pod + return m.evictPod(pod, v1.ResourceEphemeralStorage, fmt.Sprintf("pod ephemeral local storage usage exceeds the total limit of containers %v", podLimits[v1.ResourceEphemeralStorage])) + } return false } -func (m *managerImpl) containerOverlayLimitEviction(podStats statsapi.PodStats, pod *v1.Pod) bool { +func (m *managerImpl) containerEphemeralStorageLimitEviction(podStats statsapi.PodStats, pod *v1.Pod) bool { thresholdsMap := make(map[string]*resource.Quantity) for _, container := range pod.Spec.Containers { - overlayLimit := container.Resources.Limits.StorageOverlay() - if overlayLimit != nil && overlayLimit.Value() != 0 { - thresholdsMap[container.Name] = overlayLimit + ephemeralLimit := container.Resources.Limits.StorageEphemeral() + if ephemeralLimit != nil && ephemeralLimit.Value() != 0 { + thresholdsMap[container.Name] = ephemeralLimit } } for _, containerStat := range podStats.Containers { - rootfs := diskUsage(containerStat.Rootfs) - if overlayThreshold, ok := thresholdsMap[containerStat.Name]; ok { - if overlayThreshold.Cmp(*rootfs) < 0 { - return m.evictPod(pod, v1.ResourceName("containerOverlay"), fmt.Sprintf("container's overlay usage exceeds the limit %q", overlayThreshold.String())) + containerUsed := diskUsage(containerStat.Logs) + if !*m.dedicatedImageFs { + containerUsed.Add(*diskUsage(containerStat.Rootfs)) + } + + if ephemeralStorageThreshold, ok := thresholdsMap[containerStat.Name]; ok { + if ephemeralStorageThreshold.Cmp(*containerUsed) < 0 { + return m.evictPod(pod, v1.ResourceEphemeralStorage, fmt.Sprintf("container's ephemeral local storage usage exceeds the limit %q", ephemeralStorageThreshold.String())) } } diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index 859b8d21a4592..6cef960232a7b 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -54,8 +54,6 @@ const ( resourceNodeFs v1.ResourceName = "nodefs" // nodefs inodes, number. internal to this module, used to account for local node root filesystem inodes. resourceNodeFsInodes v1.ResourceName = "nodefsInodes" - // container overlay storage, in bytes. internal to this module, used to account for local disk usage for container overlay. - resourceOverlay v1.ResourceName = "overlay" ) var ( @@ -400,12 +398,10 @@ func localVolumeNames(pod *v1.Pod) []string { func podDiskUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsStatsType) (v1.ResourceList, error) { disk := resource.Quantity{Format: resource.BinarySI} inodes := resource.Quantity{Format: resource.BinarySI} - overlay := resource.Quantity{Format: resource.BinarySI} for _, container := range podStats.Containers { if hasFsStatsType(statsToMeasure, fsStatsRoot) { disk.Add(*diskUsage(container.Rootfs)) inodes.Add(*inodeUsage(container.Rootfs)) - overlay.Add(*diskUsage(container.Rootfs)) } if hasFsStatsType(statsToMeasure, fsStatsLogs) { disk.Add(*diskUsage(container.Logs)) @@ -425,9 +421,8 @@ func podDiskUsage(podStats statsapi.PodStats, pod *v1.Pod, statsToMeasure []fsSt } } return v1.ResourceList{ - resourceDisk: disk, - resourceInodes: inodes, - resourceOverlay: overlay, + resourceDisk: disk, + resourceInodes: inodes, }, nil } @@ -727,7 +722,7 @@ func makeSignalObservations(summaryProvider stats.SummaryProvider, capacityProvi } } - storageScratchCapacity, storageScratchAllocatable, exist := getResourceAllocatable(nodeCapacity, allocatableReservation, v1.ResourceStorageScratch) + ephemeralStorageCapacity, ephemeralStorageAllocatable, exist := getResourceAllocatable(nodeCapacity, allocatableReservation, v1.ResourceEphemeralStorage) if exist { for _, pod := range pods { podStat, ok := statsFunc(pod) @@ -735,25 +730,23 @@ func makeSignalObservations(summaryProvider stats.SummaryProvider, capacityProvi continue } - usage, err := podDiskUsage(podStat, pod, []fsStatsType{fsStatsLogs, fsStatsLocalVolumeSource, fsStatsRoot}) + fsStatsSet := []fsStatsType{} + if withImageFs { + fsStatsSet = []fsStatsType{fsStatsLogs, fsStatsLocalVolumeSource} + } else { + fsStatsSet = []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource} + } + + usage, err := podDiskUsage(podStat, pod, fsStatsSet) if err != nil { glog.Warningf("eviction manager: error getting pod disk usage %v", err) continue } - // If there is a seperate imagefs set up for container runtimes, the scratch disk usage from nodefs should exclude the overlay usage - if withImageFs { - diskUsage := usage[resourceDisk] - diskUsageP := &diskUsage - diskUsagep := diskUsageP.Copy() - diskUsagep.Sub(usage[resourceOverlay]) - storageScratchAllocatable.Sub(*diskUsagep) - } else { - storageScratchAllocatable.Sub(usage[resourceDisk]) - } + ephemeralStorageAllocatable.Sub(usage[resourceDisk]) } result[evictionapi.SignalAllocatableNodeFsAvailable] = signalObservation{ - available: storageScratchAllocatable, - capacity: storageScratchCapacity, + available: ephemeralStorageAllocatable, + capacity: ephemeralStorageCapacity, } } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 80fd03f26a163..9edfabf400c23 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -88,8 +88,9 @@ import ( "k8s.io/kubernetes/pkg/kubelet/rkt" "k8s.io/kubernetes/pkg/kubelet/secret" "k8s.io/kubernetes/pkg/kubelet/server" - "k8s.io/kubernetes/pkg/kubelet/server/stats" + serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats" "k8s.io/kubernetes/pkg/kubelet/server/streaming" + "k8s.io/kubernetes/pkg/kubelet/stats" "k8s.io/kubernetes/pkg/kubelet/status" "k8s.io/kubernetes/pkg/kubelet/sysctl" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" @@ -243,6 +244,7 @@ type Dependencies struct { Recorder record.EventRecorder Writer kubeio.Writer VolumePlugins []volume.VolumePlugin + DynamicPluginProber volume.DynamicPluginProber TLSOptions *server.TLSOptions KubeletConfigController *kubeletconfig.Controller } @@ -559,6 +561,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, MTU: int(crOptions.NetworkPluginMTU), } + klet.resourceAnalyzer = serverstats.NewResourceAnalyzer(klet, kubeCfg.VolumeStatsAggPeriod.Duration) + // Remote runtime shim just cannot talk back to kubelet, so it doesn't // support bandwidth shaping or hostports till #35457. To enable legacy // features, replace with networkHost. @@ -641,6 +645,12 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } klet.containerRuntime = runtime klet.runner = runtime + klet.StatsProvider = stats.NewCadvisorStatsProvider( + klet.cadvisor, + klet.resourceAnalyzer, + klet.podManager, + klet.runtimeCache, + klet.containerRuntime) } else { // rkt uses the legacy, non-CRI, integration. Configure it the old way. // TODO: Include hairpin mode settings in rkt? @@ -673,11 +683,14 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } klet.containerRuntime = runtime klet.runner = kubecontainer.DirectStreamingRunner(runtime) + klet.StatsProvider = stats.NewCadvisorStatsProvider( + klet.cadvisor, + klet.resourceAnalyzer, + klet.podManager, + klet.runtimeCache, + klet.containerRuntime) } - // TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency - klet.resourceAnalyzer = stats.NewResourceAnalyzer(klet, kubeCfg.VolumeStatsAggPeriod.Duration, klet.containerRuntime) - klet.pleg = pleg.NewGenericPLEG(klet.containerRuntime, plegChannelCapacity, plegRelistPeriod, klet.podCache, clock.RealClock{}) klet.runtimeState = newRuntimeState(maxWaitForContainerRuntime) klet.runtimeState.addHealthCheck("PLEG", klet.pleg.Healthy) @@ -736,7 +749,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps.Recorder) klet.volumePluginMgr, err = - NewInitializedVolumePluginMgr(klet, secretManager, configMapManager, kubeDeps.VolumePlugins) + NewInitializedVolumePluginMgr(klet, secretManager, configMapManager, kubeDeps.VolumePlugins, kubeDeps.DynamicPluginProber) if err != nil { return nil, err } @@ -952,7 +965,11 @@ type Kubelet struct { volumeManager volumemanager.VolumeManager // Cloud provider interface. - cloud cloudprovider.Interface + cloud cloudprovider.Interface + // DEPRECATED: auto detecting cloud providers goes against the initiative + // for out-of-tree cloud providers as we'll now depend on cAdvisor integrations + // with cloud providers instead of in the core repo. + // More details here: https://github.com/kubernetes/kubernetes/issues/50986 autoDetectCloudProvider bool // Indicates that the node initialization happens in an external cloud controller externalCloudProvider bool @@ -992,7 +1009,7 @@ type Kubelet struct { oomWatcher OOMWatcher // Monitor resource usage - resourceAnalyzer stats.ResourceAnalyzer + resourceAnalyzer serverstats.ResourceAnalyzer // Whether or not we should have the QOS cgroup hierarchy for resource management cgroupsPerQOS bool @@ -1103,6 +1120,9 @@ type Kubelet struct { // dockerLegacyService contains some legacy methods for backward compatibility. // It should be set only when docker is using non json-file logging driver. dockerLegacyService dockershim.DockerLegacyService + + // StatsProvider provides the node and the container stats. + *stats.StatsProvider } func allLocalIPsWithoutLoopback() ([]net.IP, error) { diff --git a/pkg/kubelet/kubelet_cadvisor.go b/pkg/kubelet/kubelet_cadvisor.go deleted file mode 100644 index 2ff74f03cbc1e..0000000000000 --- a/pkg/kubelet/kubelet_cadvisor.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - cadvisorapi "github.com/google/cadvisor/info/v1" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" - "k8s.io/apimachinery/pkg/types" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" -) - -// GetContainerInfo returns stats (from Cadvisor) for a container. -func (kl *Kubelet) GetContainerInfo(podFullName string, podUID types.UID, containerName string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) { - - // Resolve and type convert back again. - // We need the static pod UID but the kubecontainer API works with types.UID. - podUID = types.UID(kl.podManager.TranslatePodUID(podUID)) - - pods, err := kl.runtimeCache.GetPods() - if err != nil { - return nil, err - } - pod := kubecontainer.Pods(pods).FindPod(podFullName, podUID) - container := pod.FindContainerByName(containerName) - if container == nil { - return nil, kubecontainer.ErrContainerNotFound - } - - ci, err := kl.cadvisor.DockerContainer(container.ID.ID, req) - if err != nil { - return nil, err - } - return &ci, nil -} - -// GetContainerInfoV2 returns stats (from Cadvisor) for containers. -func (kl *Kubelet) GetContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) { - return kl.cadvisor.ContainerInfoV2(name, options) -} - -// ImagesFsInfo returns information about docker image fs usage from -// cadvisor. -func (kl *Kubelet) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) { - return kl.cadvisor.ImagesFsInfo() -} - -// RootFsInfo returns info about the root fs from cadvisor. -func (kl *Kubelet) RootFsInfo() (cadvisorapiv2.FsInfo, error) { - return kl.cadvisor.RootFsInfo() -} - -// GetRawContainerInfo returns stats (from Cadvisor) for a non-Kubernetes container. -func (kl *Kubelet) GetRawContainerInfo(containerName string, req *cadvisorapi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorapi.ContainerInfo, error) { - if subcontainers { - return kl.cadvisor.SubcontainerInfo(containerName, req) - } - - containerInfo, err := kl.cadvisor.ContainerInfo(containerName, req) - if err != nil { - return nil, err - } - return map[string]*cadvisorapi.ContainerInfo{ - containerInfo.Name: containerInfo, - }, nil -} - -// GetVersionInfo returns information about the version of cAdvisor in use. -func (kl *Kubelet) GetVersionInfo() (*cadvisorapi.VersionInfo, error) { - return kl.cadvisor.VersionInfo() -} - -// GetCachedMachineInfo assumes that the machine info can't change without a reboot -func (kl *Kubelet) GetCachedMachineInfo() (*cadvisorapi.MachineInfo, error) { - if kl.machineInfo == nil { - info, err := kl.cadvisor.MachineInfo() - if err != nil { - return nil, err - } - kl.machineInfo = info - } - return kl.machineInfo, nil -} - -// GetCachedRootFsInfo assumes that the rootfs info can't change without a reboot -func (kl *Kubelet) GetCachedRootFsInfo() (cadvisorapiv2.FsInfo, error) { - if kl.rootfsInfo == nil { - info, err := kl.cadvisor.RootFsInfo() - if err != nil { - return cadvisorapiv2.FsInfo{}, err - } - kl.rootfsInfo = &info - } - return *kl.rootfsInfo, nil -} diff --git a/pkg/kubelet/kubelet_cadvisor_test.go b/pkg/kubelet/kubelet_cadvisor_test.go deleted file mode 100644 index a107dc1df0904..0000000000000 --- a/pkg/kubelet/kubelet_cadvisor_test.go +++ /dev/null @@ -1,252 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - cadvisorapi "github.com/google/cadvisor/info/v1" - "k8s.io/apimachinery/pkg/types" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing" -) - -func TestGetContainerInfo(t *testing.T) { - cadvisorAPIFailure := fmt.Errorf("cAdvisor failure") - runtimeError := fmt.Errorf("List containers error") - tests := []struct { - name string - containerID string - containerPath string - cadvisorContainerInfo cadvisorapi.ContainerInfo - runtimeError error - podList []*kubecontainertest.FakePod - requestedPodFullName string - requestedPodUID types.UID - requestedContainerName string - expectDockerContainerCall bool - mockError error - expectedError error - expectStats bool - }{ - { - name: "get container info", - containerID: "ab2cdf", - containerPath: "/docker/ab2cdf", - cadvisorContainerInfo: cadvisorapi.ContainerInfo{ - ContainerReference: cadvisorapi.ContainerReference{ - Name: "/docker/ab2cdf", - }, - }, - runtimeError: nil, - podList: []*kubecontainertest.FakePod{ - { - Pod: &kubecontainer.Pod{ - ID: "12345678", - Name: "qux", - Namespace: "ns", - Containers: []*kubecontainer.Container{ - { - Name: "foo", - ID: kubecontainer.ContainerID{Type: "test", ID: "ab2cdf"}, - }, - }, - }, - }, - }, - requestedPodFullName: "qux_ns", - requestedPodUID: "", - requestedContainerName: "foo", - expectDockerContainerCall: true, - mockError: nil, - expectedError: nil, - expectStats: true, - }, - { - name: "get container info when cadvisor failed", - containerID: "ab2cdf", - containerPath: "/docker/ab2cdf", - cadvisorContainerInfo: cadvisorapi.ContainerInfo{}, - runtimeError: nil, - podList: []*kubecontainertest.FakePod{ - { - Pod: &kubecontainer.Pod{ - ID: "uuid", - Name: "qux", - Namespace: "ns", - Containers: []*kubecontainer.Container{ - { - Name: "foo", - ID: kubecontainer.ContainerID{Type: "test", ID: "ab2cdf"}, - }, - }, - }, - }, - }, - requestedPodFullName: "qux_ns", - requestedPodUID: "uuid", - requestedContainerName: "foo", - expectDockerContainerCall: true, - mockError: cadvisorAPIFailure, - expectedError: cadvisorAPIFailure, - expectStats: false, - }, - { - name: "get container info on non-existent container", - containerID: "", - containerPath: "", - cadvisorContainerInfo: cadvisorapi.ContainerInfo{}, - runtimeError: nil, - podList: []*kubecontainertest.FakePod{}, - requestedPodFullName: "qux", - requestedPodUID: "", - requestedContainerName: "foo", - expectDockerContainerCall: false, - mockError: nil, - expectedError: kubecontainer.ErrContainerNotFound, - expectStats: false, - }, - { - name: "get container info when container runtime failed", - containerID: "", - containerPath: "", - cadvisorContainerInfo: cadvisorapi.ContainerInfo{}, - runtimeError: runtimeError, - podList: []*kubecontainertest.FakePod{}, - requestedPodFullName: "qux", - requestedPodUID: "", - requestedContainerName: "foo", - mockError: nil, - expectedError: runtimeError, - expectStats: false, - }, - { - name: "get container info with no containers", - containerID: "", - containerPath: "", - cadvisorContainerInfo: cadvisorapi.ContainerInfo{}, - runtimeError: nil, - podList: []*kubecontainertest.FakePod{}, - requestedPodFullName: "qux_ns", - requestedPodUID: "", - requestedContainerName: "foo", - mockError: nil, - expectedError: kubecontainer.ErrContainerNotFound, - expectStats: false, - }, - { - name: "get container info with no matching containers", - containerID: "", - containerPath: "", - cadvisorContainerInfo: cadvisorapi.ContainerInfo{}, - runtimeError: nil, - podList: []*kubecontainertest.FakePod{ - { - Pod: &kubecontainer.Pod{ - ID: "12345678", - Name: "qux", - Namespace: "ns", - Containers: []*kubecontainer.Container{ - { - Name: "bar", - ID: kubecontainer.ContainerID{Type: "test", ID: "fakeID"}, - }, - }, - }, - }, - }, - requestedPodFullName: "qux_ns", - requestedPodUID: "", - requestedContainerName: "foo", - mockError: nil, - expectedError: kubecontainer.ErrContainerNotFound, - expectStats: false, - }, - } - - for _, tc := range tests { - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnablec */) - defer testKubelet.Cleanup() - fakeRuntime := testKubelet.fakeRuntime - kubelet := testKubelet.kubelet - cadvisorReq := &cadvisorapi.ContainerInfoRequest{} - mockCadvisor := testKubelet.fakeCadvisor - if tc.expectDockerContainerCall { - mockCadvisor.On("DockerContainer", tc.containerID, cadvisorReq).Return(tc.cadvisorContainerInfo, tc.mockError) - } - fakeRuntime.Err = tc.runtimeError - fakeRuntime.PodList = tc.podList - - stats, err := kubelet.GetContainerInfo(tc.requestedPodFullName, tc.requestedPodUID, tc.requestedContainerName, cadvisorReq) - assert.Equal(t, tc.expectedError, err) - - if tc.expectStats { - require.NotNil(t, stats) - } - mockCadvisor.AssertExpectations(t) - } -} - -func TestGetRawContainerInfoRoot(t *testing.T) { - containerPath := "/" - containerInfo := &cadvisorapi.ContainerInfo{ - ContainerReference: cadvisorapi.ContainerReference{ - Name: containerPath, - }, - } - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - kubelet := testKubelet.kubelet - mockCadvisor := testKubelet.fakeCadvisor - cadvisorReq := &cadvisorapi.ContainerInfoRequest{} - mockCadvisor.On("ContainerInfo", containerPath, cadvisorReq).Return(containerInfo, nil) - - _, err := kubelet.GetRawContainerInfo(containerPath, cadvisorReq, false) - assert.NoError(t, err) - mockCadvisor.AssertExpectations(t) -} - -func TestGetRawContainerInfoSubcontainers(t *testing.T) { - containerPath := "/kubelet" - containerInfo := map[string]*cadvisorapi.ContainerInfo{ - containerPath: { - ContainerReference: cadvisorapi.ContainerReference{ - Name: containerPath, - }, - }, - "/kubelet/sub": { - ContainerReference: cadvisorapi.ContainerReference{ - Name: "/kubelet/sub", - }, - }, - } - testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) - defer testKubelet.Cleanup() - kubelet := testKubelet.kubelet - mockCadvisor := testKubelet.fakeCadvisor - cadvisorReq := &cadvisorapi.ContainerInfoRequest{} - mockCadvisor.On("SubcontainerInfo", containerPath, cadvisorReq).Return(containerInfo, nil) - - result, err := kubelet.GetRawContainerInfo(containerPath, cadvisorReq, true) - assert.NoError(t, err) - assert.Len(t, result, 2) - mockCadvisor.AssertExpectations(t) -} diff --git a/pkg/kubelet/kubelet_getters.go b/pkg/kubelet/kubelet_getters.go index dd55bd10df2e9..7737229507ee1 100644 --- a/pkg/kubelet/kubelet_getters.go +++ b/pkg/kubelet/kubelet_getters.go @@ -23,6 +23,8 @@ import ( "path/filepath" "github.com/golang/glog" + cadvisorapiv1 "github.com/google/cadvisor/info/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/cmd/kubelet/app/options" @@ -239,3 +241,20 @@ func (kl *Kubelet) getPodVolumePathListFromDisk(podUID types.UID) ([]string, err } return volumes, nil } + +// GetVersionInfo returns information about the version of cAdvisor in use. +func (kl *Kubelet) GetVersionInfo() (*cadvisorapiv1.VersionInfo, error) { + return kl.cadvisor.VersionInfo() +} + +// GetCachedMachineInfo assumes that the machine info can't change without a reboot +func (kl *Kubelet) GetCachedMachineInfo() (*cadvisorapiv1.MachineInfo, error) { + if kl.machineInfo == nil { + info, err := kl.cadvisor.MachineInfo() + if err != nil { + return nil, err + } + kl.machineInfo = info + } + return kl.machineInfo, nil +} diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index 9a20a75600ad3..2b1ec95831411 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -137,6 +137,7 @@ func (kl *Kubelet) tryRegisterWithAPIServer(node *v1.Node) bool { // the value of the controller-managed attach-detach // annotation. requiresUpdate := kl.reconcileCMADAnnotationWithExistingNode(node, existingNode) + requiresUpdate = kl.updateDefaultLabels(node, existingNode) || requiresUpdate if requiresUpdate { if _, err := nodeutil.PatchNodeStatus(kl.kubeClient, types.NodeName(kl.nodeName), originalNode, existingNode); err != nil { @@ -161,6 +162,33 @@ func (kl *Kubelet) tryRegisterWithAPIServer(node *v1.Node) bool { return false } +// updateDefaultLabels will set the default labels on the node +func (kl *Kubelet) updateDefaultLabels(initialNode, existingNode *v1.Node) bool { + defaultLabels := []string{ + kubeletapis.LabelHostname, + kubeletapis.LabelZoneFailureDomain, + kubeletapis.LabelZoneRegion, + kubeletapis.LabelInstanceType, + kubeletapis.LabelOS, + kubeletapis.LabelArch, + } + + var needsUpdate bool = false + //Set default labels but make sure to not set labels with empty values + for _, label := range defaultLabels { + if existingNode.Labels[label] != initialNode.Labels[label] { + existingNode.Labels[label] = initialNode.Labels[label] + needsUpdate = true + } + + if existingNode.Labels[label] == "" { + delete(existingNode.Labels, label) + } + } + + return needsUpdate +} + // reconcileCMADAnnotationWithExistingNode reconciles the controller-managed // attach-detach annotation on a new node and the existing node, returning // whether the existing node must be updated. @@ -564,11 +592,7 @@ func (kl *Kubelet) setNodeStatusMachineInfo(node *v1.Node) { // capacity for every node status request initialCapacity := kl.containerManager.GetCapacity() if initialCapacity != nil { - node.Status.Capacity[v1.ResourceStorageScratch] = initialCapacity[v1.ResourceStorageScratch] - imageCapacity, ok := initialCapacity[v1.ResourceStorageOverlay] - if ok { - node.Status.Capacity[v1.ResourceStorageOverlay] = imageCapacity - } + node.Status.Capacity[v1.ResourceEphemeralStorage] = initialCapacity[v1.ResourceEphemeralStorage] } } } diff --git a/pkg/kubelet/kubelet_node_status_test.go b/pkg/kubelet/kubelet_node_status_test.go index cd26a352cfe81..ae005ea20eb6e 100644 --- a/pkg/kubelet/kubelet_node_status_test.go +++ b/pkg/kubelet/kubelet_node_status_test.go @@ -43,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/util/sliceutils" @@ -675,8 +676,15 @@ func TestRegisterWithApiServer(t *testing.T) { kubeClient.AddReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) { // Return an existing (matching) node on get. return true, &v1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, - Spec: v1.NodeSpec{ExternalID: testKubeletHostname}, + ObjectMeta: metav1.ObjectMeta{ + Name: testKubeletHostname, + Labels: map[string]string{ + kubeletapis.LabelHostname: testKubeletHostname, + kubeletapis.LabelOS: goruntime.GOOS, + kubeletapis.LabelArch: goruntime.GOARCH, + }, + }, + Spec: v1.NodeSpec{ExternalID: testKubeletHostname}, }, nil }) kubeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) { @@ -731,7 +739,13 @@ func TestTryRegisterWithApiServer(t *testing.T) { newNode := func(cmad bool, externalID string) *v1.Node { node := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kubeletapis.LabelHostname: testKubeletHostname, + kubeletapis.LabelOS: goruntime.GOOS, + kubeletapis.LabelArch: goruntime.GOARCH, + }, + }, Spec: v1.NodeSpec{ ExternalID: externalID, }, @@ -961,3 +975,164 @@ func TestUpdateNewNodeStatusTooLargeReservation(t *testing.T) { assert.NoError(t, err) assert.True(t, apiequality.Semantic.DeepEqual(expectedNode.Status.Allocatable, updatedNode.Status.Allocatable), "%s", diff.ObjectDiff(expectedNode.Status.Allocatable, updatedNode.Status.Allocatable)) } + +func TestUpdateDefaultLabels(t *testing.T) { + testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) + + cases := []struct { + name string + initialNode *v1.Node + existingNode *v1.Node + needsUpdate bool + finalLabels map[string]string + }{ + { + name: "make sure default labels exist", + initialNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + }, + }, + }, + existingNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{}, + }, + }, + needsUpdate: true, + finalLabels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + }, + }, + { + name: "make sure default labels are up to date", + initialNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + }, + }, + }, + existingNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kubeletapis.LabelHostname: "old-hostname", + kubeletapis.LabelZoneFailureDomain: "old-zone-failure-domain", + kubeletapis.LabelZoneRegion: "old-zone-region", + kubeletapis.LabelInstanceType: "old-instance-type", + kubeletapis.LabelOS: "old-os", + kubeletapis.LabelArch: "old-arch", + }, + }, + }, + needsUpdate: true, + finalLabels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + }, + }, + { + name: "make sure existing labels do not get deleted", + initialNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + }, + }, + }, + existingNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + "please-persist": "foo", + }, + }, + }, + needsUpdate: false, + finalLabels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + "please-persist": "foo", + }, + }, + { + name: "no update needed", + initialNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + }, + }, + }, + existingNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + }, + }, + }, + needsUpdate: false, + finalLabels: map[string]string{ + kubeletapis.LabelHostname: "new-hostname", + kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain", + kubeletapis.LabelZoneRegion: "new-zone-region", + kubeletapis.LabelInstanceType: "new-instance-type", + kubeletapis.LabelOS: "new-os", + kubeletapis.LabelArch: "new-arch", + }, + }, + } + + for _, tc := range cases { + defer testKubelet.Cleanup() + kubelet := testKubelet.kubelet + + needsUpdate := kubelet.updateDefaultLabels(tc.initialNode, tc.existingNode) + assert.Equal(t, tc.needsUpdate, needsUpdate, tc.name) + assert.Equal(t, tc.finalLabels, tc.existingNode.Labels, tc.name) + } +} diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 86b004565c093..8720902a1fa88 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -233,7 +233,7 @@ func ensureHostsFile(fileName, hostIP, hostName, hostDomainName string, hostAlia // if Pod is using host network, read hosts file from the node's filesystem. // `etcHostsPath` references the location of the hosts file on the node. // `/etc/hosts` for *nix systems. - hostsFileContent, err = nodeHostsFileContent(etcHostsPath) + hostsFileContent, err = nodeHostsFileContent(etcHostsPath, hostAliases) if err != nil { return err } @@ -246,8 +246,13 @@ func ensureHostsFile(fileName, hostIP, hostName, hostDomainName string, hostAlia } // nodeHostsFileContent reads the content of node's hosts file. -func nodeHostsFileContent(hostsFilePath string) ([]byte, error) { - return ioutil.ReadFile(hostsFilePath) +func nodeHostsFileContent(hostsFilePath string, hostAliases []v1.HostAlias) ([]byte, error) { + hostsFileContent, err := ioutil.ReadFile(hostsFilePath) + if err != nil { + return nil, err + } + hostsFileContent = append(hostsFileContent, hostsEntriesFromHostAliases(hostAliases)...) + return hostsFileContent, nil } // managedHostsFileContent generates the content of the managed etc hosts based on Pod IP and other @@ -266,6 +271,19 @@ func managedHostsFileContent(hostIP, hostName, hostDomainName string, hostAliase } else { buffer.WriteString(fmt.Sprintf("%s\t%s\n", hostIP, hostName)) } + hostsFileContent := buffer.Bytes() + hostsFileContent = append(hostsFileContent, hostsEntriesFromHostAliases(hostAliases)...) + return hostsFileContent +} + +func hostsEntriesFromHostAliases(hostAliases []v1.HostAlias) []byte { + if len(hostAliases) == 0 { + return []byte{} + } + + var buffer bytes.Buffer + buffer.WriteString("\n") + buffer.WriteString("# Entries added by HostAliases.\n") // write each IP/hostname pair as an entry into hosts file for _, hostAlias := range hostAliases { for _, hostname := range hostAlias.Hostnames { @@ -333,7 +351,6 @@ func (kl *Kubelet) GetPodCgroupParent(pod *v1.Pod) string { // GenerateRunContainerOptions generates the RunContainerOptions, which can be used by // the container runtime to set parameters for launching a container. func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, bool, error) { - var err error useClusterFirstPolicy := false cgroupParent := kl.GetPodCgroupParent(pod) opts := &kubecontainer.RunContainerOptions{CgroupParent: cgroupParent} diff --git a/pkg/kubelet/kubelet_pods_test.go b/pkg/kubelet/kubelet_pods_test.go index eb9558706ae2f..413f266b84c78 100644 --- a/pkg/kubelet/kubelet_pods_test.go +++ b/pkg/kubelet/kubelet_pods_test.go @@ -184,11 +184,23 @@ func TestMakeMounts(t *testing.T) { func TestNodeHostsFileContent(t *testing.T) { testCases := []struct { - hostsFileName string - expectedContent string + hostsFileName string + hostAliases []v1.HostAlias + rawHostsFileContent string + expectedHostsFileContent string }{ { "hosts_test_file1", + []v1.HostAlias{}, + `# hosts file for testing. +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +fe00::0 ip6-mcastprefix +fe00::1 ip6-allnodes +fe00::2 ip6-allrouters +123.45.67.89 some.domain +`, `# hosts file for testing. 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback @@ -201,6 +213,70 @@ fe00::2 ip6-allrouters }, { "hosts_test_file2", + []v1.HostAlias{}, + `# another hosts file for testing. +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +fe00::0 ip6-mcastprefix +fe00::1 ip6-allnodes +fe00::2 ip6-allrouters +12.34.56.78 another.domain +`, + `# another hosts file for testing. +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +fe00::0 ip6-mcastprefix +fe00::1 ip6-allnodes +fe00::2 ip6-allrouters +12.34.56.78 another.domain +`, + }, + { + "hosts_test_file1_with_host_aliases", + []v1.HostAlias{ + {IP: "123.45.67.89", Hostnames: []string{"foo", "bar", "baz"}}, + }, + `# hosts file for testing. +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +fe00::0 ip6-mcastprefix +fe00::1 ip6-allnodes +fe00::2 ip6-allrouters +123.45.67.89 some.domain +`, + `# hosts file for testing. +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +fe00::0 ip6-mcastprefix +fe00::1 ip6-allnodes +fe00::2 ip6-allrouters +123.45.67.89 some.domain + +# Entries added by HostAliases. +123.45.67.89 foo +123.45.67.89 bar +123.45.67.89 baz +`, + }, + { + "hosts_test_file2_with_host_aliases", + []v1.HostAlias{ + {IP: "123.45.67.89", Hostnames: []string{"foo", "bar", "baz"}}, + {IP: "456.78.90.123", Hostnames: []string{"park", "doo", "boo"}}, + }, + `# another hosts file for testing. +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +fe00::0 ip6-mcastprefix +fe00::1 ip6-allnodes +fe00::2 ip6-allrouters +12.34.56.78 another.domain +`, `# another hosts file for testing. 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback @@ -209,18 +285,26 @@ fe00::0 ip6-mcastprefix fe00::1 ip6-allnodes fe00::2 ip6-allrouters 12.34.56.78 another.domain + +# Entries added by HostAliases. +123.45.67.89 foo +123.45.67.89 bar +123.45.67.89 baz +456.78.90.123 park +456.78.90.123 doo +456.78.90.123 boo `, }, } for _, testCase := range testCases { - tmpdir, err := writeHostsFile(testCase.hostsFileName, testCase.expectedContent) + tmpdir, err := writeHostsFile(testCase.hostsFileName, testCase.rawHostsFileContent) require.NoError(t, err, "could not create a temp hosts file") defer os.RemoveAll(tmpdir) - actualContent, fileReadErr := nodeHostsFileContent(filepath.Join(tmpdir, testCase.hostsFileName)) + actualContent, fileReadErr := nodeHostsFileContent(filepath.Join(tmpdir, testCase.hostsFileName), testCase.hostAliases) require.NoError(t, fileReadErr, "could not create read hosts file") - assert.Equal(t, testCase.expectedContent, string(actualContent), "hosts file content not expected") + assert.Equal(t, testCase.expectedHostsFileContent, string(actualContent), "hosts file content not expected") } } @@ -287,6 +371,8 @@ fe00::0 ip6-mcastprefix fe00::1 ip6-allnodes fe00::2 ip6-allrouters 203.0.113.1 podFoo.domainFoo podFoo + +# Entries added by HostAliases. 123.45.67.89 foo 123.45.67.89 bar 123.45.67.89 baz @@ -308,6 +394,8 @@ fe00::0 ip6-mcastprefix fe00::1 ip6-allnodes fe00::2 ip6-allrouters 203.0.113.1 podFoo.domainFoo podFoo + +# Entries added by HostAliases. 123.45.67.89 foo 123.45.67.89 bar 123.45.67.89 baz diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 657d304f438f1..86cac081edf5b 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -60,7 +60,8 @@ import ( proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results" probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing" "k8s.io/kubernetes/pkg/kubelet/secret" - "k8s.io/kubernetes/pkg/kubelet/server/stats" + serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats" + "k8s.io/kubernetes/pkg/kubelet/stats" "k8s.io/kubernetes/pkg/kubelet/status" statustest "k8s.io/kubernetes/pkg/kubelet/status/testing" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" @@ -115,6 +116,11 @@ func (tk *TestKubelet) Cleanup() { } } +func (tk *TestKubelet) chainMock() { + tk.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + tk.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) +} + // newTestKubelet returns test kubelet with two images. func newTestKubelet(t *testing.T, controllerAttachDetachEnabled bool) *TestKubelet { imageList := []kubecontainer.Image{ @@ -256,7 +262,7 @@ func newTestKubeletWithImageList( // TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency volumeStatsAggPeriod := time.Second * 10 - kubelet.resourceAnalyzer = stats.NewResourceAnalyzer(kubelet, volumeStatsAggPeriod, kubelet.containerRuntime) + kubelet.resourceAnalyzer = serverstats.NewResourceAnalyzer(kubelet, volumeStatsAggPeriod) nodeRef := &v1.ObjectReference{ Kind: "Node", Name: string(kubelet.nodeName), @@ -272,8 +278,9 @@ func newTestKubeletWithImageList( kubelet.admitHandlers.AddPodAdmitHandler(lifecycle.NewPredicateAdmitHandler(kubelet.getNodeAnyWay, lifecycle.NewAdmissionFailureHandlerStub())) plug := &volumetest.FakeVolumePlugin{PluginName: "fake", Host: nil} + var prober volume.DynamicPluginProber = nil // TODO (#51147) inject mock kubelet.volumePluginMgr, err = - NewInitializedVolumePluginMgr(kubelet, kubelet.secretManager, kubelet.configMapManager, []volume.VolumePlugin{plug}) + NewInitializedVolumePluginMgr(kubelet, kubelet.secretManager, kubelet.configMapManager, []volume.VolumePlugin{plug}, prober) require.NoError(t, err, "Failed to initialize VolumePluginMgr") kubelet.mounter = &mount.FakeMounter{} @@ -298,6 +305,12 @@ func newTestKubeletWithImageList( kubelet.AddPodSyncLoopHandler(activeDeadlineHandler) kubelet.AddPodSyncHandler(activeDeadlineHandler) kubelet.gpuManager = gpu.NewGPUManagerStub() + kubelet.StatsProvider = stats.NewCadvisorStatsProvider( + kubelet.cadvisor, + kubelet.resourceAnalyzer, + kubelet.podManager, + kubelet.runtimeCache, + fakeRuntime) return &TestKubelet{kubelet, fakeRuntime, mockCadvisor, fakeKubeClient, fakeMirrorClient, fakeClock, nil, plug} } @@ -322,7 +335,6 @@ var emptyPodUIDs map[types.UID]kubetypes.SyncPodType func TestSyncLoopAbort(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) kubelet := testKubelet.kubelet kubelet.runtimeState.setRuntimeSync(time.Now()) // The syncLoop waits on time.After(resyncInterval), set it really big so that we don't race for @@ -343,10 +355,6 @@ func TestSyncLoopAbort(t *testing.T) { func TestSyncPodsStartPod(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet fakeRuntime := testKubelet.fakeRuntime pods := []*v1.Pod{ @@ -367,9 +375,6 @@ func TestSyncPodsDeletesWhenSourcesAreReady(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() fakeRuntime := testKubelet.fakeRuntime - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) kubelet := testKubelet.kubelet kubelet.sourcesReady = config.NewSourcesReady(func(_ sets.String) bool { return ready }) @@ -415,14 +420,18 @@ func (ls testNodeLister) List(selector labels.Selector) ([]*v1.Node, error) { return ls.nodes, nil } +func checkPodStatus(t *testing.T, kl *Kubelet, pod *v1.Pod, phase v1.PodPhase) { + status, found := kl.statusManager.GetPodStatus(pod.UID) + require.True(t, found, "Status of pod %q is not found in the status map", pod.UID) + require.Equal(t, phase, status.Phase) +} + // Tests that we handle port conflicts correctly by setting the failed status in status map. func TestHandlePortConflicts(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() + testKubelet.chainMock() kl := testKubelet.kubelet - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) kl.nodeInfo = testNodeInfo{nodes: []*v1.Node{ { @@ -448,26 +457,18 @@ func TestHandlePortConflicts(t *testing.T) { fittingPod := pods[1] kl.HandlePodAdditions(pods) - // Check pod status stored in the status map. - // notfittingPod should be Failed - status, found := kl.statusManager.GetPodStatus(notfittingPod.UID) - require.True(t, found, "Status of pod %q is not found in the status map", notfittingPod.UID) - require.Equal(t, v1.PodFailed, status.Phase) - // fittingPod should be Pending - status, found = kl.statusManager.GetPodStatus(fittingPod.UID) - require.True(t, found, "Status of pod %q is not found in the status map", fittingPod.UID) - require.Equal(t, v1.PodPending, status.Phase) + // Check pod status stored in the status map. + checkPodStatus(t, kl, notfittingPod, v1.PodFailed) + checkPodStatus(t, kl, fittingPod, v1.PodPending) } // Tests that we handle host name conflicts correctly by setting the failed status in status map. func TestHandleHostNameConflicts(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() + testKubelet.chainMock() kl := testKubelet.kubelet - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) kl.nodeInfo = testNodeInfo{nodes: []*v1.Node{ { @@ -490,22 +491,17 @@ func TestHandleHostNameConflicts(t *testing.T) { fittingPod := pods[1] kl.HandlePodAdditions(pods) - // Check pod status stored in the status map. - // notfittingPod should be Failed - status, found := kl.statusManager.GetPodStatus(notfittingPod.UID) - require.True(t, found, "Status of pod %q is not found in the status map", notfittingPod.UID) - require.Equal(t, v1.PodFailed, status.Phase) - // fittingPod should be Pending - status, found = kl.statusManager.GetPodStatus(fittingPod.UID) - require.True(t, found, "Status of pod %q is not found in the status map", fittingPod.UID) - require.Equal(t, v1.PodPending, status.Phase) + // Check pod status stored in the status map. + checkPodStatus(t, kl, notfittingPod, v1.PodFailed) + checkPodStatus(t, kl, fittingPod, v1.PodPending) } // Tests that we handle not matching labels selector correctly by setting the failed status in status map. func TestHandleNodeSelector(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() + testKubelet.chainMock() kl := testKubelet.kubelet nodes := []*v1.Node{ { @@ -518,9 +514,6 @@ func TestHandleNodeSelector(t *testing.T) { }, } kl.nodeInfo = testNodeInfo{nodes: nodes} - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) pods := []*v1.Pod{ podWithUIDNameNsSpec("123456789", "podA", "foo", v1.PodSpec{NodeSelector: map[string]string{"key": "A"}}), podWithUIDNameNsSpec("987654321", "podB", "foo", v1.PodSpec{NodeSelector: map[string]string{"key": "B"}}), @@ -530,22 +523,17 @@ func TestHandleNodeSelector(t *testing.T) { fittingPod := pods[1] kl.HandlePodAdditions(pods) - // Check pod status stored in the status map. - // notfittingPod should be Failed - status, found := kl.statusManager.GetPodStatus(notfittingPod.UID) - require.True(t, found, "Status of pod %q is not found in the status map", notfittingPod.UID) - require.Equal(t, v1.PodFailed, status.Phase) - // fittingPod should be Pending - status, found = kl.statusManager.GetPodStatus(fittingPod.UID) - require.True(t, found, "Status of pod %q is not found in the status map", fittingPod.UID) - require.Equal(t, v1.PodPending, status.Phase) + // Check pod status stored in the status map. + checkPodStatus(t, kl, notfittingPod, v1.PodFailed) + checkPodStatus(t, kl, fittingPod, v1.PodPending) } // Tests that we handle exceeded resources correctly by setting the failed status in status map. func TestHandleMemExceeded(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() + testKubelet.chainMock() kl := testKubelet.kubelet nodes := []*v1.Node{ {ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, @@ -556,14 +544,11 @@ func TestHandleMemExceeded(t *testing.T) { }}}, } kl.nodeInfo = testNodeInfo{nodes: nodes} - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) spec := v1.PodSpec{NodeName: string(kl.nodeName), Containers: []v1.Container{{Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "memory": resource.MustParse("90"), + v1.ResourceMemory: resource.MustParse("90"), }, }}}, } @@ -579,25 +564,17 @@ func TestHandleMemExceeded(t *testing.T) { fittingPod := pods[1] kl.HandlePodAdditions(pods) - // Check pod status stored in the status map. - // notfittingPod should be Failed - status, found := kl.statusManager.GetPodStatus(notfittingPod.UID) - require.True(t, found, "Status of pod %q is not found in the status map", notfittingPod.UID) - require.Equal(t, v1.PodFailed, status.Phase) - // fittingPod should be Pending - status, found = kl.statusManager.GetPodStatus(fittingPod.UID) - require.True(t, found, "Status of pod %q is not found in the status map", fittingPod.UID) - require.Equal(t, v1.PodPending, status.Phase) + // Check pod status stored in the status map. + checkPodStatus(t, kl, notfittingPod, v1.PodFailed) + checkPodStatus(t, kl, fittingPod, v1.PodPending) } // TODO(filipg): This test should be removed once StatusSyncer can do garbage collection without external signal. func TestPurgingObsoleteStatusMapEntries(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() versionInfo := &cadvisorapi.VersionInfo{ KernelVersion: "3.16.0-0.bpo.4-amd64", ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)", @@ -735,11 +712,7 @@ func TestCreateMirrorPod(t *testing.T) { for _, updateType := range []kubetypes.SyncPodType{kubetypes.SyncPodCreate, kubetypes.SyncPodUpdate} { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kl := testKubelet.kubelet manager := testKubelet.fakeMirrorClient @@ -762,11 +735,7 @@ func TestCreateMirrorPod(t *testing.T) { func TestDeleteOutdatedMirrorPod(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kl := testKubelet.kubelet manager := testKubelet.fakeMirrorClient @@ -805,10 +774,7 @@ func TestDeleteOutdatedMirrorPod(t *testing.T) { func TestDeleteOrphanedMirrorPods(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kl := testKubelet.kubelet manager := testKubelet.fakeMirrorClient @@ -928,11 +894,7 @@ func TestGetContainerInfoForMirrorPods(t *testing.T) { func TestHostNetworkAllowed(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet @@ -961,11 +923,7 @@ func TestHostNetworkAllowed(t *testing.T) { func TestHostNetworkDisallowed(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet @@ -993,11 +951,7 @@ func TestHostNetworkDisallowed(t *testing.T) { func TestHostPIDAllowed(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet @@ -1026,11 +980,7 @@ func TestHostPIDAllowed(t *testing.T) { func TestHostPIDDisallowed(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet @@ -1058,11 +1008,7 @@ func TestHostPIDDisallowed(t *testing.T) { func TestHostIPCAllowed(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet @@ -1091,11 +1037,7 @@ func TestHostIPCAllowed(t *testing.T) { func TestHostIPCDisallowed(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet @@ -1123,11 +1065,7 @@ func TestHostIPCDisallowed(t *testing.T) { func TestPrivilegeContainerAllowed(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet @@ -1153,10 +1091,7 @@ func TestPrivilegeContainerAllowed(t *testing.T) { func TestPrivilegedContainerDisallowed(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet capabilities.SetForTests(capabilities.Capabilities{ @@ -1180,10 +1115,7 @@ func TestPrivilegedContainerDisallowed(t *testing.T) { func TestNetworkErrorsWithoutHostNetwork(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet kubelet.runtimeState.setNetworkState(fmt.Errorf("simulated network error")) @@ -1298,11 +1230,7 @@ func TestSyncPodsDoesNotSetPodsThatDidNotRunTooLongToFailed(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() fakeRuntime := testKubelet.fakeRuntime - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet @@ -1367,11 +1295,7 @@ func podWithUIDNameNsSpec(uid types.UID, name, namespace string, spec v1.PodSpec func TestDeletePodDirsForDeletedPods(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kl := testKubelet.kubelet pods := []*v1.Pod{ podWithUIDNameNs("12345678", "pod1", "ns"), @@ -1407,11 +1331,7 @@ func syncAndVerifyPodDir(t *testing.T, testKubelet *TestKubelet, pods []*v1.Pod, func TestDoesNotDeletePodDirsForTerminatedPods(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kl := testKubelet.kubelet pods := []*v1.Pod{ podWithUIDNameNs("12345678", "pod1", "ns"), @@ -1430,11 +1350,7 @@ func TestDoesNotDeletePodDirsForTerminatedPods(t *testing.T) { func TestDoesNotDeletePodDirsIfContainerIsRunning(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("Start").Return(nil) - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() runningPod := &kubecontainer.Pod{ ID: "12345678", Name: "pod1", @@ -1494,10 +1410,7 @@ func TestGetPodsToSync(t *testing.T) { func TestGenerateAPIPodStatusWithSortedContainers(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet numContainers := 10 expectedOrder := []string{} @@ -1557,10 +1470,7 @@ func TestGenerateAPIPodStatusWithReasonCache(t *testing.T) { emptyContainerID := (&kubecontainer.ContainerID{}).String() testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet pod := podWithUIDNameNs("12345678", "foo", "new") pod.Spec = v1.PodSpec{RestartPolicy: v1.RestartPolicyOnFailure} @@ -1747,10 +1657,7 @@ func TestGenerateAPIPodStatusWithDifferentRestartPolicies(t *testing.T) { emptyContainerID := (&kubecontainer.ContainerID{}).String() testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() - testKubelet.fakeCadvisor.On("VersionInfo").Return(&cadvisorapi.VersionInfo{}, nil) - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) + testKubelet.chainMock() kubelet := testKubelet.kubelet pod := podWithUIDNameNs("12345678", "foo", "new") containers := []v1.Container{{Name: "succeed"}, {Name: "failed"}} @@ -1914,6 +1821,7 @@ func (a *testPodAdmitHandler) Admit(attrs *lifecycle.PodAdmitAttributes) lifecyc func TestHandlePodAdditionsInvokesPodAdmitHandlers(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() + testKubelet.chainMock() kl := testKubelet.kubelet kl.nodeInfo = testNodeInfo{nodes: []*v1.Node{ { @@ -1925,9 +1833,6 @@ func TestHandlePodAdditionsInvokesPodAdmitHandlers(t *testing.T) { }, }, }} - testKubelet.fakeCadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil) - testKubelet.fakeCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) - testKubelet.fakeCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{}, nil) pods := []*v1.Pod{ { @@ -1952,16 +1857,10 @@ func TestHandlePodAdditionsInvokesPodAdmitHandlers(t *testing.T) { kl.admitHandlers.AddPodAdmitHandler(&testPodAdmitHandler{podsToReject: podsToReject}) kl.HandlePodAdditions(pods) - // Check pod status stored in the status map. - // podToReject should be Failed - status, found := kl.statusManager.GetPodStatus(podToReject.UID) - require.True(t, found, "Status of pod %q is not found in the status map", podToAdmit.UID) - require.Equal(t, v1.PodFailed, status.Phase) - // podToAdmit should be Pending - status, found = kl.statusManager.GetPodStatus(podToAdmit.UID) - require.True(t, found, "Status of pod %q is not found in the status map", podToAdmit.UID) - require.Equal(t, v1.PodPending, status.Phase) + // Check pod status stored in the status map. + checkPodStatus(t, kl, podToReject, v1.PodFailed) + checkPodStatus(t, kl, podToAdmit, v1.PodPending) } // testPodSyncLoopHandler is a lifecycle.PodSyncLoopHandler that is used for testing. @@ -2065,10 +1964,9 @@ func TestSyncPodKillPod(t *testing.T) { }, }) require.NoError(t, err) + // Check pod status stored in the status map. - status, found := kl.statusManager.GetPodStatus(pod.UID) - require.True(t, found, "Status of pod %q is not found in the status map", pod.UID) - require.Equal(t, v1.PodFailed, status.Phase) + checkPodStatus(t, kl, pod, v1.PodFailed) } func waitForVolumeUnmount( diff --git a/pkg/kubelet/kubelet_volumes.go b/pkg/kubelet/kubelet_volumes.go index 2baaf667472ac..2bb3c73c3744f 100644 --- a/pkg/kubelet/kubelet_volumes.go +++ b/pkg/kubelet/kubelet_volumes.go @@ -69,6 +69,7 @@ func (kl *Kubelet) newVolumeMounterFromPlugins(spec *volume.Spec, pod *v1.Pod, o if err != nil { return nil, fmt.Errorf("can't use volume plugins for %s: %v", spec.Name(), err) } + opts.Containerized = kl.kubeletConfiguration.Containerized physicalMounter, err := plugin.NewMounter(spec, pod, opts) if err != nil { return nil, fmt.Errorf("failed to instantiate mounter for volume: %s using plugin: %s with a root cause: %v", spec.Name(), plugin.GetPluginName(), err) diff --git a/pkg/kubelet/kubeletconfig/BUILD b/pkg/kubelet/kubeletconfig/BUILD index d9b14812703c1..e3f94317a73bd 100644 --- a/pkg/kubelet/kubeletconfig/BUILD +++ b/pkg/kubelet/kubeletconfig/BUILD @@ -16,17 +16,14 @@ go_library( deps = [ "//pkg/kubelet/apis/kubeletconfig:go_default_library", "//pkg/kubelet/apis/kubeletconfig/validation:go_default_library", - "//pkg/kubelet/kubeletconfig/badconfig:go_default_library", "//pkg/kubelet/kubeletconfig/checkpoint:go_default_library", "//pkg/kubelet/kubeletconfig/checkpoint/store:go_default_library", "//pkg/kubelet/kubeletconfig/configfiles:go_default_library", - "//pkg/kubelet/kubeletconfig/startups:go_default_library", "//pkg/kubelet/kubeletconfig/status:go_default_library", "//pkg/kubelet/kubeletconfig/util/equal:go_default_library", "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library", "//pkg/kubelet/kubeletconfig/util/log:go_default_library", "//pkg/kubelet/kubeletconfig/util/panic:go_default_library", - "//pkg/version:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", @@ -49,10 +46,8 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", - "//pkg/kubelet/kubeletconfig/badconfig:all-srcs", "//pkg/kubelet/kubeletconfig/checkpoint:all-srcs", "//pkg/kubelet/kubeletconfig/configfiles:all-srcs", - "//pkg/kubelet/kubeletconfig/startups:all-srcs", "//pkg/kubelet/kubeletconfig/status:all-srcs", "//pkg/kubelet/kubeletconfig/util/codec:all-srcs", "//pkg/kubelet/kubeletconfig/util/equal:all-srcs", diff --git a/pkg/kubelet/kubeletconfig/badconfig/BUILD b/pkg/kubelet/kubeletconfig/badconfig/BUILD deleted file mode 100644 index 82fecc726786d..0000000000000 --- a/pkg/kubelet/kubeletconfig/badconfig/BUILD +++ /dev/null @@ -1,47 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = [ - "badconfig_test.go", - "fstracker_test.go", - ], - library = ":go_default_library", - deps = [ - "//pkg/kubelet/kubeletconfig/util/files:go_default_library", - "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library", - "//pkg/kubelet/kubeletconfig/util/test:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = [ - "badconfig.go", - "fstracker.go", - ], - deps = [ - "//pkg/kubelet/kubeletconfig/util/files:go_default_library", - "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library", - "//pkg/kubelet/kubeletconfig/util/log:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/pkg/kubelet/kubeletconfig/badconfig/badconfig.go b/pkg/kubelet/kubeletconfig/badconfig/badconfig.go deleted file mode 100644 index 0374192695ac6..0000000000000 --- a/pkg/kubelet/kubeletconfig/badconfig/badconfig.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package badconfig - -import ( - "encoding/json" - "fmt" - "time" -) - -// Tracker tracks "bad" configurations in a storage layer -type Tracker interface { - // Initialize sets up the storage layer - Initialize() error - // MarkBad marks `uid` as a bad config and records `reason` as the reason for marking it bad - MarkBad(uid, reason string) error - // Entry returns the Entry for `uid` if it exists in the tracker, otherise nil - Entry(uid string) (*Entry, error) -} - -// Entry describes when a configuration was marked bad and why -type Entry struct { - Time string `json:"time"` - Reason string `json:"reason"` -} - -// markBad makes an entry in `m` for the config with `uid` and reason `reason` -func markBad(m map[string]Entry, uid, reason string) { - now := time.Now() - entry := Entry{ - Time: now.Format(time.RFC3339), // use RFC3339 time format - Reason: reason, - } - m[uid] = entry -} - -// getEntry returns the Entry for `uid` in `m`, or nil if no such entry exists -func getEntry(m map[string]Entry, uid string) *Entry { - entry, ok := m[uid] - if ok { - return &entry - } - return nil -} - -// encode retuns a []byte representation of `m`, for saving `m` to a storage layer -func encode(m map[string]Entry) ([]byte, error) { - data, err := json.Marshal(m) - if err != nil { - return nil, err - } - return data, nil -} - -// decode transforms a []byte into a `map[string]Entry`, or returns an error if it can't produce said map -// if `data` is empty, returns an empty map -func decode(data []byte) (map[string]Entry, error) { - // create the map - m := map[string]Entry{} - // if the data is empty, just return the empty map - if len(data) == 0 { - return m, nil - } - // otherwise unmarshal the json - if err := json.Unmarshal(data, &m); err != nil { - return nil, fmt.Errorf("failed to unmarshal, error: %v", err) - } - return m, nil -} diff --git a/pkg/kubelet/kubeletconfig/badconfig/badconfig_test.go b/pkg/kubelet/kubeletconfig/badconfig/badconfig_test.go deleted file mode 100644 index 617219eeb6cbf..0000000000000 --- a/pkg/kubelet/kubeletconfig/badconfig/badconfig_test.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package badconfig - -import ( - "fmt" - "reflect" - "testing" - "time" -) - -func TestMarkBad(t *testing.T) { - // build a map with one entry - m := map[string]Entry{} - uid := "uid" - reason := "reason" - markBad(m, uid, reason) - - // the entry should exist for uid - entry, ok := m[uid] - if !ok { - t.Fatalf("expect entry for uid %q, but none exists", uid) - } - - // the entry's reason should match the reason it was marked bad with - if entry.Reason != reason { - t.Errorf("expect Entry.Reason %q, but got %q", reason, entry.Reason) - } - - // the entry's timestamp should be in RFC3339 format - if err := assertRFC3339(entry.Time); err != nil { - t.Errorf("expect Entry.Time to use RFC3339 format, but got %q, error: %v", entry.Time, err) - } - - // it should be the only entry in the map thus far - if n := len(m); n != 1 { - t.Errorf("expect one entry in the map, but got %d", n) - } - -} - -func TestGetEntry(t *testing.T) { - nowstamp := time.Now().Format(time.RFC3339) - uid := "uid" - expect := &Entry{ - Time: nowstamp, - Reason: "reason", - } - m := map[string]Entry{uid: *expect} - - // should return nil for entries that don't exist - bogus := "bogus-uid" - if e := getEntry(m, bogus); e != nil { - t.Errorf("expect nil for entries that don't exist (uid: %q), but got %#v", bogus, e) - } - - // should return non-nil for entries that exist - if e := getEntry(m, uid); e == nil { - t.Errorf("expect non-nil for entries that exist (uid: %q), but got nil", uid) - } else if !reflect.DeepEqual(expect, e) { - // entry should match what we inserted for the given UID - t.Errorf("expect entry for uid %q to match %#v, but got %#v", uid, expect, e) - } -} - -func TestEncode(t *testing.T) { - nowstamp := time.Now().Format(time.RFC3339) - uid := "uid" - expect := fmt.Sprintf(`{"%s":{"time":"%s","reason":"reason"}}`, uid, nowstamp) - m := map[string]Entry{uid: { - Time: nowstamp, - Reason: "reason", - }} - - data, err := encode(m) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - json := string(data) - - if json != expect { - t.Errorf("expect encoding of %#v to match %q, but got %q", m, expect, json) - } -} - -func TestDecode(t *testing.T) { - nowstamp := time.Now().Format(time.RFC3339) - uid := "uid" - valid := []byte(fmt.Sprintf(`{"%s":{"time":"%s","reason":"reason"}}`, uid, nowstamp)) - expect := map[string]Entry{uid: { - Time: nowstamp, - Reason: "reason", - }} - - // decoding valid json should result in an object with the correct values - if m, err := decode(valid); err != nil { - t.Errorf("expect decoding valid json %q to produce a map, but got error: %v", valid, err) - } else if !reflect.DeepEqual(expect, m) { - // m should equal expected decoded object - t.Errorf("expect decoding valid json %q to produce %#v, but got %#v", valid, expect, m) - } - - // decoding invalid json should return an error - invalid := []byte(`invalid`) - if m, err := decode(invalid); err == nil { - t.Errorf("expect decoding invalid json %q to return an error, but decoded to %#v", invalid, m) - } -} - -func TestRoundTrip(t *testing.T) { - nowstamp := time.Now().Format(time.RFC3339) - uid := "uid" - expect := map[string]Entry{uid: { - Time: nowstamp, - Reason: "reason", - }} - - // test that encoding and decoding an object results in the same value - data, err := encode(expect) - if err != nil { - t.Fatalf("failed to encode %#v, error: %v", expect, err) - } - after, err := decode(data) - if err != nil { - t.Fatalf("failed to decode %q, error: %v", string(data), err) - } - if !reflect.DeepEqual(expect, after) { - t.Errorf("expect round-tripping %#v to result in the same value, but got %#v", expect, after) - } -} - -func assertRFC3339(s string) error { - tm, err := time.Parse(time.RFC3339, s) - if err != nil { - return fmt.Errorf("expect RFC3339 format, but failed to parse, error: %v", err) - } - // parsing succeeded, now finish round-trip and compare - rt := tm.Format(time.RFC3339) - if rt != s { - return fmt.Errorf("expect RFC3339 format, but failed to round trip unchanged, original %q, round-trip %q", s, rt) - } - return nil -} diff --git a/pkg/kubelet/kubeletconfig/badconfig/fstracker.go b/pkg/kubelet/kubeletconfig/badconfig/fstracker.go deleted file mode 100644 index a90ec26dd5c2d..0000000000000 --- a/pkg/kubelet/kubeletconfig/badconfig/fstracker.go +++ /dev/null @@ -1,105 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package badconfig - -import ( - "path/filepath" - - utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files" - utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem" - utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log" -) - -const ( - badConfigsFile = "bad-configs.json" -) - -// fsTracker tracks bad config in the local filesystem -type fsTracker struct { - // fs is the filesystem to use for storage operations; can be mocked for testing - fs utilfs.Filesystem - // trackingDir is the absolute path to the storage directory for fsTracker - trackingDir string -} - -// NewFsTracker returns a new Tracker that will store information in the `trackingDir` -func NewFsTracker(fs utilfs.Filesystem, trackingDir string) Tracker { - return &fsTracker{ - fs: fs, - trackingDir: trackingDir, - } -} - -func (tracker *fsTracker) Initialize() error { - utillog.Infof("initializing bad config tracking directory %q", tracker.trackingDir) - if err := utilfiles.EnsureDir(tracker.fs, tracker.trackingDir); err != nil { - return err - } - if err := utilfiles.EnsureFile(tracker.fs, filepath.Join(tracker.trackingDir, badConfigsFile)); err != nil { - return err - } - return nil -} - -func (tracker *fsTracker) MarkBad(uid, reason string) error { - m, err := tracker.load() - if err != nil { - return err - } - // create the bad config entry in the map - markBad(m, uid, reason) - // save the file - if err := tracker.save(m); err != nil { - return err - } - return nil -} - -func (tracker *fsTracker) Entry(uid string) (*Entry, error) { - m, err := tracker.load() - if err != nil { - return nil, err - } - // return the entry, or nil if it doesn't exist - return getEntry(m, uid), nil -} - -// load loads the bad-config-tracking file from disk and decodes the map encoding it contains -func (tracker *fsTracker) load() (map[string]Entry, error) { - path := filepath.Join(tracker.trackingDir, badConfigsFile) - // load the file - data, err := tracker.fs.ReadFile(path) - if err != nil { - return nil, err - } - return decode(data) -} - -// save replaces the contents of the bad-config-tracking file with the encoding of `m` -func (tracker *fsTracker) save(m map[string]Entry) error { - // encode the map - data, err := encode(m) - if err != nil { - return err - } - // save the file - path := filepath.Join(tracker.trackingDir, badConfigsFile) - if err := utilfiles.ReplaceFile(tracker.fs, path, data); err != nil { - return err - } - return nil -} diff --git a/pkg/kubelet/kubeletconfig/badconfig/fstracker_test.go b/pkg/kubelet/kubeletconfig/badconfig/fstracker_test.go deleted file mode 100644 index 25b234b18b079..0000000000000 --- a/pkg/kubelet/kubeletconfig/badconfig/fstracker_test.go +++ /dev/null @@ -1,255 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package badconfig - -import ( - "fmt" - "path/filepath" - "reflect" - "testing" - "time" - - utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files" - utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem" - utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test" -) - -const testTrackingDir = "/test-tracking-dir" - -// TODO(mtaufen): this file reuses a lot of test code from badconfig_test.go, should consolidate - -func newInitializedFakeFsTracker() (*fsTracker, error) { - fs := utilfs.NewFakeFs() - tracker := NewFsTracker(fs, testTrackingDir) - if err := tracker.Initialize(); err != nil { - return nil, err - } - return tracker.(*fsTracker), nil -} - -func TestFsTrackerInitialize(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("fsTracker.Initialize() failed with error: %v", err) - } - - // check that testTrackingDir exists - _, err = tracker.fs.Stat(testTrackingDir) - if err != nil { - t.Fatalf("expect %q to exist, but stat failed with error: %v", testTrackingDir, err) - } - - // check that testTrackingDir contains the badConfigsFile - path := filepath.Join(testTrackingDir, badConfigsFile) - _, err = tracker.fs.Stat(path) - if err != nil { - t.Fatalf("expect %q to exist, but stat failed with error: %v", path, err) - } -} - -func TestFsTrackerMarkBad(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - // create a bad config entry in the fs - uid := "uid" - reason := "reason" - tracker.MarkBad(uid, reason) - - // load the map from the fs - m, err := tracker.load() - if err != nil { - t.Fatalf("failed to load bad-config data, error: %v", err) - } - - // the entry should exist for uid - entry, ok := m[uid] - if !ok { - t.Fatalf("expect entry for uid %q, but none exists", uid) - } - - // the entry's reason should match the reason it was marked bad with - if entry.Reason != reason { - t.Errorf("expect Entry.Reason %q, but got %q", reason, entry.Reason) - } - - // the entry's timestamp should be in RFC3339 format - if err := assertRFC3339(entry.Time); err != nil { - t.Errorf("expect Entry.Time to use RFC3339 format, but got %q, error: %v", entry.Time, err) - } - - // it should be the only entry in the map thus far - if n := len(m); n != 1 { - t.Errorf("expect one entry in the map, but got %d", n) - } -} - -func TestFsTrackerEntry(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - // manually save a correct entry to fs - nowstamp := time.Now().Format(time.RFC3339) - uid := "uid" - expect := &Entry{ - Time: nowstamp, - Reason: "reason", - } - m := map[string]Entry{uid: *expect} - err = tracker.save(m) - if err != nil { - t.Fatalf("failed to save bad-config data, error: %v", err) - } - - // should return nil for entries that don't exist - bogus := "bogus-uid" - e, err := tracker.Entry(bogus) - if err != nil { - t.Errorf("expect nil for entries that don't exist (uid: %q), but got error: %v", bogus, err) - } else if e != nil { - t.Errorf("expect nil for entries that don't exist (uid: %q), but got %#v", bogus, e) - } - - // should return non-nil for entries that exist - e, err = tracker.Entry(uid) - if err != nil { - t.Errorf("expect non-nil for entries that exist (uid: %q), but got error: %v", uid, err) - } else if e == nil { - t.Errorf("expect non-nil for entries that exist (uid: %q), but got nil", uid) - } else if !reflect.DeepEqual(expect, e) { - // entry should match what we inserted for the given UID - t.Errorf("expect entry for uid %q to match %#v, but got %#v", uid, expect, e) - } -} - -// TODO(mtaufen): test loading invalid json (see startups/fstracker_test.go for example) -func TestFsTrackerLoad(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - uid := "uid" - nowstamp := time.Now().Format(time.RFC3339) - cases := []struct { - desc string - data []byte - expect map[string]Entry - err string - }{ - // empty file - {"empty file", []byte(""), map[string]Entry{}, ""}, - // empty map - {"empty map", []byte("{}"), map[string]Entry{}, ""}, - // valid json - {"valid json", []byte(fmt.Sprintf(`{"%s":{"time":"%s","reason":"reason"}}`, uid, nowstamp)), - map[string]Entry{uid: { - Time: nowstamp, - Reason: "reason", - }}, ""}, - // invalid json - {"invalid json", []byte(`*`), map[string]Entry{}, "failed to unmarshal"}, - } - - for _, c := range cases { - // save a file containing the correct serialization - utilfiles.ReplaceFile(tracker.fs, filepath.Join(testTrackingDir, badConfigsFile), c.data) - - // loading valid json should result in an object with the correct values - m, err := tracker.load() - if utiltest.SkipRest(t, c.desc, err, c.err) { - continue - } - if !reflect.DeepEqual(c.expect, m) { - // m should equal expected decoded object - t.Errorf("case %q, expect %#v but got %#v", c.desc, c.expect, m) - } - } -} - -func TestFsTrackerSave(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - uid := "uid" - nowstamp := time.Now().Format(time.RFC3339) - cases := []struct { - desc string - m map[string]Entry - expect string - err string - }{ - // empty map - {"empty map", map[string]Entry{}, "{}", ""}, - // 1-entry map - {"1-entry map", - map[string]Entry{uid: { - Time: nowstamp, - Reason: "reason", - }}, - fmt.Sprintf(`{"%s":{"time":"%s","reason":"reason"}}`, uid, nowstamp), ""}, - } - - for _, c := range cases { - if err := tracker.save(c.m); utiltest.SkipRest(t, c.desc, err, c.err) { - continue - } - - data, err := tracker.fs.ReadFile(filepath.Join(testTrackingDir, badConfigsFile)) - if err != nil { - t.Fatalf("failed to read bad-config file, error: %v", err) - } - json := string(data) - - if json != c.expect { - t.Errorf("case %q, expect %q but got %q", c.desc, c.expect, json) - } - } -} - -func TestFsTrackerRoundTrip(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - nowstamp := time.Now().Format(time.RFC3339) - uid := "uid" - expect := map[string]Entry{uid: { - Time: nowstamp, - Reason: "reason", - }} - - // test that saving and loading an object results in the same value - err = tracker.save(expect) - if err != nil { - t.Fatalf("failed to save bad-config data, error: %v", err) - } - after, err := tracker.load() - if err != nil { - t.Fatalf("failed to load bad-config data, error: %v", err) - } - if !reflect.DeepEqual(expect, after) { - t.Errorf("expect round-tripping %#v to result in the same value, but got %#v", expect, after) - } -} diff --git a/pkg/kubelet/kubeletconfig/checkpoint/BUILD b/pkg/kubelet/kubeletconfig/checkpoint/BUILD index 8e2844f81c451..c4d1fa6f88491 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/BUILD +++ b/pkg/kubelet/kubeletconfig/checkpoint/BUILD @@ -15,8 +15,8 @@ go_test( ], library = ":go_default_library", deps = [ - "//pkg/api:go_default_library", "//pkg/kubelet/apis/kubeletconfig:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", "//pkg/kubelet/kubeletconfig/util/codec:go_default_library", "//pkg/kubelet/kubeletconfig/util/test:go_default_library", @@ -40,12 +40,14 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/kubelet/apis/kubeletconfig:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", "//pkg/kubelet/kubeletconfig/util/codec:go_default_library", "//pkg/kubelet/kubeletconfig/util/log:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", ], ) diff --git a/pkg/kubelet/kubeletconfig/checkpoint/checkpoint.go b/pkg/kubelet/kubeletconfig/checkpoint/checkpoint.go index e9a6c29d70142..852ea71488906 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/checkpoint.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/checkpoint.go @@ -30,7 +30,7 @@ import ( type Checkpoint interface { // UID returns the UID of the config source object behind the Checkpoint UID() string - // Parse parses the checkpoint into the internal KubeletConfiguration type + // Parse extracts the KubeletConfiguration from the checkpoint, applies defaults, and converts to the internal type Parse() (*kubeletconfig.KubeletConfiguration, error) // Encode returns a []byte representation of the config source object behind the Checkpoint Encode() ([]byte, error) @@ -56,6 +56,7 @@ func DecodeCheckpoint(data []byte) (Checkpoint, error) { if err != nil { return nil, fmt.Errorf("failed to convert decoded object into a v1 ConfigMap, error: %v", err) } + return NewConfigMapCheckpoint(cm) } diff --git a/pkg/kubelet/kubeletconfig/checkpoint/checkpoint_test.go b/pkg/kubelet/kubeletconfig/checkpoint/checkpoint_test.go index d9cc259524986..abcf9981b1c26 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/checkpoint_test.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/checkpoint_test.go @@ -30,20 +30,6 @@ import ( utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test" ) -// newUnsupportedEncoded returns an encoding of an object that does not have a Checkpoint implementation -func newUnsupportedEncoded(t *testing.T) []byte { - encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName) - if err != nil { - t.Fatalf("could not create an encoder, error: %v", err) - } - unsupported := &apiv1.Node{} - data, err := runtime.Encode(encoder, unsupported) - if err != nil { - t.Fatalf("could not encode object, error: %v", err) - } - return data -} - func TestDecodeCheckpoint(t *testing.T) { // generate correct Checkpoint for v1/ConfigMap test case cm, err := NewConfigMapCheckpoint(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: types.UID("uid")}}) @@ -87,3 +73,17 @@ func TestDecodeCheckpoint(t *testing.T) { } } } + +// newUnsupportedEncoded returns an encoding of an object that does not have a Checkpoint implementation +func newUnsupportedEncoded(t *testing.T) []byte { + encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName) + if err != nil { + t.Fatalf("could not create an encoder, error: %v", err) + } + unsupported := &apiv1.Node{} + data, err := runtime.Encode(encoder, unsupported) + if err != nil { + t.Fatalf("could not encode object, error: %v", err) + } + return data +} diff --git a/pkg/kubelet/kubeletconfig/checkpoint/configmap.go b/pkg/kubelet/kubeletconfig/checkpoint/configmap.go index b0a8ecdea98de..7472384889f5f 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/configmap.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/configmap.go @@ -21,7 +21,9 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec" ) @@ -29,7 +31,8 @@ const configMapConfigKey = "kubelet" // configMapCheckpoint implements Checkpoint, backed by a v1/ConfigMap config source object type configMapCheckpoint struct { - configMap *apiv1.ConfigMap + kubeletCodecs *serializer.CodecFactory // codecs for the KubeletConfiguration + configMap *apiv1.ConfigMap } // NewConfigMapCheckpoint returns a Checkpoint backed by `cm`. `cm` must be non-nil @@ -40,7 +43,13 @@ func NewConfigMapCheckpoint(cm *apiv1.ConfigMap) (Checkpoint, error) { } else if len(cm.ObjectMeta.UID) == 0 { return nil, fmt.Errorf("ConfigMap must have a UID to be treated as a Checkpoint") } - return &configMapCheckpoint{cm}, nil + + _, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + return nil, err + } + + return &configMapCheckpoint{kubeletCodecs, cm}, nil } // UID returns the UID of a configMapCheckpoint @@ -48,25 +57,23 @@ func (c *configMapCheckpoint) UID() string { return string(c.configMap.UID) } -// implements Parse for v1/ConfigMap checkpoints +// Parse extracts the KubeletConfiguration from v1/ConfigMap checkpoints, applies defaults, and converts to the internal type func (c *configMapCheckpoint) Parse() (*kubeletconfig.KubeletConfiguration, error) { const emptyCfgErr = "config was empty, but some parameters are required" - cm := c.configMap - - if len(cm.Data) == 0 { + if len(c.configMap.Data) == 0 { return nil, fmt.Errorf(emptyCfgErr) } // TODO(mtaufen): Once the KubeletConfiguration type is decomposed, extend this to a key for each sub-object - config, ok := cm.Data[configMapConfigKey] + config, ok := c.configMap.Data[configMapConfigKey] if !ok { return nil, fmt.Errorf("key %q not found in ConfigMap", configMapConfigKey) } else if len(config) == 0 { return nil, fmt.Errorf(emptyCfgErr) } - return utilcodec.DecodeKubeletConfiguration([]byte(config)) + return utilcodec.DecodeKubeletConfiguration(c.kubeletCodecs, []byte(config)) } // Encode encodes a configMapCheckpoint diff --git a/pkg/kubelet/kubeletconfig/checkpoint/configmap_test.go b/pkg/kubelet/kubeletconfig/checkpoint/configmap_test.go index 07092e4446d6c..98d8a37eb2adc 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/configmap_test.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/configmap_test.go @@ -26,8 +26,8 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test" ) @@ -66,9 +66,15 @@ func TestNewConfigMapCheckpoint(t *testing.T) { } func TestConfigMapCheckpointUID(t *testing.T) { + _, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cases := []string{"", "uid", "376dfb73-56db-11e7-a01e-42010a800002"} for _, uidIn := range cases { cpt := &configMapCheckpoint{ + kubeletCodecs, &apiv1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{UID: types.UID(uidIn)}, }, @@ -82,11 +88,16 @@ func TestConfigMapCheckpointUID(t *testing.T) { } func TestConfigMapCheckpointParse(t *testing.T) { + kubeletScheme, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // get the built-in default configuration external := &kubeletconfigv1alpha1.KubeletConfiguration{} - api.Scheme.Default(external) + kubeletScheme.Default(external) defaultConfig := &kubeletconfig.KubeletConfiguration{} - err := api.Scheme.Convert(external, defaultConfig, nil) + err = kubeletScheme.Convert(external, defaultConfig, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -123,7 +134,7 @@ apiVersion: kubeletconfig/v1alpha1`}}, defaultConfig, ""}, "kubelet": `{"kind":"KubeletConfiguration","apiVersion":"kubeletconfig/v1alpha1"}`}}, defaultConfig, ""}, } for _, c := range cases { - cpt := &configMapCheckpoint{c.cm} + cpt := &configMapCheckpoint{kubeletCodecs, c.cm} kc, err := cpt.Parse() if utiltest.SkipRest(t, c.desc, err, c.err) { continue @@ -136,6 +147,11 @@ apiVersion: kubeletconfig/v1alpha1`}}, defaultConfig, ""}, } func TestConfigMapCheckpointEncode(t *testing.T) { + _, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // only one case, based on output from the existing encoder, and since // this is hard to test (key order isn't guaranteed), we should probably // just stick to this test case and mostly rely on the round-trip test. @@ -146,7 +162,7 @@ func TestConfigMapCheckpointEncode(t *testing.T) { }{ // we expect Checkpoints to be encoded as a json representation of the underlying API object {"one-key", - &configMapCheckpoint{&apiv1.ConfigMap{ + &configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "one-key"}, Data: map[string]string{"one": ""}}}, `{"kind":"ConfigMap","apiVersion":"v1","metadata":{"name":"one-key","creationTimestamp":null},"data":{"one":""}} @@ -166,6 +182,11 @@ func TestConfigMapCheckpointEncode(t *testing.T) { } func TestConfigMapCheckpointRoundTrip(t *testing.T) { + _, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cases := []struct { desc string cpt *configMapCheckpoint @@ -173,7 +194,7 @@ func TestConfigMapCheckpointRoundTrip(t *testing.T) { }{ // empty data {"empty data", - &configMapCheckpoint{&apiv1.ConfigMap{ + &configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "empty-data-sha256-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", UID: "uid", @@ -182,7 +203,7 @@ func TestConfigMapCheckpointRoundTrip(t *testing.T) { ""}, // two keys {"two keys", - &configMapCheckpoint{&apiv1.ConfigMap{ + &configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "two-keys-sha256-2bff03d6249c8a9dc9a1436d087c124741361ccfac6615b81b67afcff5c42431", UID: "uid", @@ -191,7 +212,7 @@ func TestConfigMapCheckpointRoundTrip(t *testing.T) { ""}, // missing uid {"missing uid", - &configMapCheckpoint{&apiv1.ConfigMap{ + &configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "two-keys-sha256-2bff03d6249c8a9dc9a1436d087c124741361ccfac6615b81b67afcff5c42431", UID: "", diff --git a/pkg/kubelet/kubeletconfig/checkpoint/download.go b/pkg/kubelet/kubeletconfig/checkpoint/download.go index b61b394ab4b80..14374a4cbf3f5 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/download.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/download.go @@ -113,7 +113,6 @@ func (r *remoteConfigMap) UID() string { func (r *remoteConfigMap) Download(client clientset.Interface) (Checkpoint, string, error) { var reason string - uid := string(r.source.ConfigMapRef.UID) utillog.Infof("attempting to download ConfigMap with UID %q", uid) @@ -131,8 +130,14 @@ func (r *remoteConfigMap) Download(client clientset.Interface) (Checkpoint, stri return nil, reason, fmt.Errorf(reason) } + checkpoint, err := NewConfigMapCheckpoint(cm) + if err != nil { + reason = fmt.Sprintf("invalid downloaded object") + return nil, reason, fmt.Errorf("%s, error: %v", reason, err) + } + utillog.Infof("successfully downloaded ConfigMap with UID %q", uid) - return &configMapCheckpoint{cm}, "", nil + return checkpoint, "", nil } func (r *remoteConfigMap) Encode() ([]byte, error) { diff --git a/pkg/kubelet/kubeletconfig/checkpoint/download_test.go b/pkg/kubelet/kubeletconfig/checkpoint/download_test.go index 496353919f42f..da3f60e343034 100644 --- a/pkg/kubelet/kubeletconfig/checkpoint/download_test.go +++ b/pkg/kubelet/kubeletconfig/checkpoint/download_test.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" fakeclient "k8s.io/client-go/kubernetes/fake" + kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test" ) @@ -92,6 +93,11 @@ func TestRemoteConfigMapUID(t *testing.T) { } func TestRemoteConfigMapDownload(t *testing.T) { + _, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cm := &apiv1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "name", @@ -118,7 +124,7 @@ func TestRemoteConfigMapDownload(t *testing.T) { // successful download {"object exists and reference is correct", &remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}}, - &configMapCheckpoint{cm}, ""}, + &configMapCheckpoint{kubeletCodecs, cm}, ""}, } for _, c := range cases { diff --git a/pkg/kubelet/kubeletconfig/configfiles/BUILD b/pkg/kubelet/kubeletconfig/configfiles/BUILD index 94a32a7191c24..bba05666bbe99 100644 --- a/pkg/kubelet/kubeletconfig/configfiles/BUILD +++ b/pkg/kubelet/kubeletconfig/configfiles/BUILD @@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( @@ -10,8 +11,10 @@ go_library( srcs = ["configfiles.go"], deps = [ "//pkg/kubelet/apis/kubeletconfig:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", "//pkg/kubelet/kubeletconfig/util/codec:go_default_library", "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", ], ) @@ -27,3 +30,19 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) + +go_test( + name = "go_default_test", + srcs = ["configfiles_test.go"], + library = ":go_default_library", + deps = [ + "//pkg/kubelet/apis/kubeletconfig:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", + "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", + "//pkg/kubelet/kubeletconfig/util/files:go_default_library", + "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library", + "//pkg/kubelet/kubeletconfig/util/test:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", + ], +) diff --git a/pkg/kubelet/kubeletconfig/configfiles/configfiles.go b/pkg/kubelet/kubeletconfig/configfiles/configfiles.go index 449c5c8112ee2..68a118aea20dd 100644 --- a/pkg/kubelet/kubeletconfig/configfiles/configfiles.go +++ b/pkg/kubelet/kubeletconfig/configfiles/configfiles.go @@ -20,11 +20,15 @@ import ( "fmt" "path/filepath" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec" utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem" ) +const kubeletFile = "kubelet" + // Loader loads configuration from a storage layer type Loader interface { // Load loads and returns the KubeletConfiguration from the storage layer, or an error if a configuration could not be loaded @@ -35,23 +39,29 @@ type Loader interface { type fsLoader struct { // fs is the filesystem where the config files exist; can be mocked for testing fs utilfs.Filesystem + // kubeletCodecs is the scheme used to decode config files + kubeletCodecs *serializer.CodecFactory // configDir is the absolute path to the directory containing the configuration files configDir string } -// NewFSLoader returns a Loader that loads a KubeletConfiguration from the files in `configDir` -func NewFSLoader(fs utilfs.Filesystem, configDir string) Loader { - return &fsLoader{ - fs: fs, - configDir: configDir, +// NewFsLoader returns a Loader that loads a KubeletConfiguration from the files in `configDir` +func NewFsLoader(fs utilfs.Filesystem, configDir string) (Loader, error) { + _, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + return nil, err } + + return &fsLoader{ + fs: fs, + kubeletCodecs: kubeletCodecs, + configDir: configDir, + }, nil } func (loader *fsLoader) Load() (*kubeletconfig.KubeletConfiguration, error) { - errfmt := fmt.Sprintf("failed to load Kubelet config files from %q, error: ", loader.configDir) + "%v" - // require the config be in a file called "kubelet" - path := filepath.Join(loader.configDir, "kubelet") + path := filepath.Join(loader.configDir, kubeletFile) data, err := loader.fs.ReadFile(path) if err != nil { return nil, fmt.Errorf("failed to read init config file %q, error: %v", path, err) @@ -59,8 +69,8 @@ func (loader *fsLoader) Load() (*kubeletconfig.KubeletConfiguration, error) { // no configuration is an error, some parameters are required if len(data) == 0 { - return nil, fmt.Errorf(errfmt, fmt.Errorf("config file was empty, but some parameters are required")) + return nil, fmt.Errorf("init config file %q was empty, but some parameters are required", path) } - return utilcodec.DecodeKubeletConfiguration(data) + return utilcodec.DecodeKubeletConfiguration(loader.kubeletCodecs, data) } diff --git a/pkg/kubelet/kubeletconfig/configfiles/configfiles_test.go b/pkg/kubelet/kubeletconfig/configfiles/configfiles_test.go new file mode 100644 index 0000000000000..2e8017a47cf76 --- /dev/null +++ b/pkg/kubelet/kubeletconfig/configfiles/configfiles_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configfiles + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/davecgh/go-spew/spew" + + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" + kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" + kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" + utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files" + utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem" + utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test" +) + +func addFile(fs utilfs.Filesystem, path string, file string) error { + if err := utilfiles.EnsureDir(fs, filepath.Dir(path)); err != nil { + return err + } + if err := utilfiles.ReplaceFile(fs, path, []byte(file)); err != nil { + return err + } + return nil +} + +func TestLoad(t *testing.T) { + kubeletScheme, _, err := kubeletscheme.NewSchemeAndCodecs() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // get the built-in default configuration + external := &kubeletconfigv1alpha1.KubeletConfiguration{} + kubeletScheme.Default(external) + defaultConfig := &kubeletconfig.KubeletConfiguration{} + err = kubeletScheme.Convert(external, defaultConfig, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + cases := []struct { + desc string + file string + expect *kubeletconfig.KubeletConfiguration + err string + }{ + {"empty data", ``, nil, "was empty"}, + // invalid format + {"invalid yaml", `*`, nil, "failed to decode"}, + {"invalid json", `{*`, nil, "failed to decode"}, + // invalid object + {"missing kind", `{"apiVersion":"kubeletconfig/v1alpha1"}`, nil, "failed to decode"}, + {"missing version", `{"kind":"KubeletConfiguration"}`, nil, "failed to decode"}, + {"unregistered kind", `{"kind":"BogusKind","apiVersion":"kubeletconfig/v1alpha1"}`, nil, "failed to decode"}, + {"unregistered version", `{"kind":"KubeletConfiguration","apiVersion":"bogusversion"}`, nil, "failed to decode"}, + // empty object with correct kind and version should result in the defaults for that kind and version + {"default from yaml", `kind: KubeletConfiguration +apiVersion: kubeletconfig/v1alpha1`, defaultConfig, ""}, + {"default from json", `{"kind":"KubeletConfiguration","apiVersion":"kubeletconfig/v1alpha1"}`, defaultConfig, ""}, + } + + fs := utilfs.NewFakeFs() + for i := range cases { + dir := fmt.Sprintf("/%d", i) + if err := addFile(fs, filepath.Join(dir, kubeletFile), cases[i].file); err != nil { + t.Fatalf("unexpected error: %v", err) + } + loader, err := NewFsLoader(fs, dir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + kc, err := loader.Load() + if utiltest.SkipRest(t, cases[i].desc, err, cases[i].err) { + continue + } + // we expect the parsed configuration to match what we described in the ConfigMap + if !apiequality.Semantic.DeepEqual(cases[i].expect, kc) { + t.Errorf("case %q, expect config %s but got %s", cases[i].desc, spew.Sdump(cases[i].expect), spew.Sdump(kc)) + } + } + + // finally test for a missing file + desc := "missing kubelet file" + contains := "failed to read" + loader, err := NewFsLoader(fs, "/fake") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + _, err = loader.Load() + if err == nil { + t.Errorf("case %q, expect error to contain %q but got nil error", desc, contains) + } else if !strings.Contains(err.Error(), contains) { + t.Errorf("case %q, expect error to contain %q but got %q", desc, contains, err.Error()) + } +} diff --git a/pkg/kubelet/kubeletconfig/configsync.go b/pkg/kubelet/kubeletconfig/configsync.go index 6f2f74f28cc9a..fb94b0dad4cef 100644 --- a/pkg/kubelet/kubeletconfig/configsync.go +++ b/pkg/kubelet/kubeletconfig/configsync.go @@ -75,32 +75,8 @@ func (cc *Controller) syncConfigSource(client clientset.Interface, nodeName stri // If we get here: // - there is no need to restart to update the current config // - there was no error trying to sync configuration - // - if, previously, there was an error trying to sync configuration, we need to update to the correct condition - errfmt := `sync succeeded but unable to clear "failed to sync" message from ConfigOK, error: %v` - - currentUID := "" - if currentSource, err := cc.checkpointStore.Current(); err != nil { - utillog.Errorf(errfmt, err) - return - } else if currentSource != nil { - currentUID = currentSource.UID() - } - - lkgUID := "" - if lkgSource, err := cc.checkpointStore.LastKnownGood(); err != nil { - utillog.Errorf(errfmt, err) - return - } else if lkgSource != nil { - lkgUID = lkgSource.UID() - } - - currentBadReason := "" - if entry, err := cc.badConfigTracker.Entry(currentUID); err != nil { - utillog.Errorf(errfmt, err) - } else if entry != nil { - currentBadReason = entry.Reason - } - cc.configOK.ClearFailedSyncCondition(currentUID, lkgUID, currentBadReason, cc.initConfig != nil) + // - if, previously, there was an error trying to sync configuration, we need to clear that error from the condition + cc.configOK.ClearFailedSyncCondition() } // doSyncConfigSource checkpoints and sets the store's current config to the new config or resets config, diff --git a/pkg/kubelet/kubeletconfig/controller.go b/pkg/kubelet/kubeletconfig/controller.go index f6f0c34e6d2eb..af08192a5374f 100644 --- a/pkg/kubelet/kubeletconfig/controller.go +++ b/pkg/kubelet/kubeletconfig/controller.go @@ -27,12 +27,9 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/validation" - "k8s.io/kubernetes/pkg/version" - "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/badconfig" "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store" "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles" - "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/startups" "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status" utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem" utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log" @@ -40,10 +37,8 @@ import ( ) const ( - badConfigTrackingDir = "bad-config-tracking" - startupTrackingDir = "startup-tracking" - checkpointsDir = "checkpoints" - initConfigDir = "init" + checkpointsDir = "checkpoints" + initConfigDir = "init" ) // Controller is the controller which, among other things: @@ -51,9 +46,8 @@ const ( // - checkpoints configuration to disk // - downloads new configuration from the API server // - validates configuration -// - monitors for potential crash-loops caused by new configurations // - tracks the last-known-good configuration, and rolls-back to last-known-good when necessary -// For more information, see the proposal: https://github.com/kubernetes/kubernetes/pull/29459 +// For more information, see the proposal: https://github.com/kubernetes/community/blob/master/contributors/design-proposals/dynamic-kubelet-configuration.md type Controller struct { // dynamicConfig, if true, indicates that we should sync config from the API server dynamicConfig bool @@ -78,38 +72,31 @@ type Controller struct { // checkpointStore persists config source checkpoints to a storage layer checkpointStore store.Store - - // badConfigTracker persists bad-config records to a storage layer - badConfigTracker badconfig.Tracker - - // startupTracker persists Kubelet startup records, used for crash-loop detection, to a storage layer - startupTracker startups.Tracker } // NewController constructs a new Controller object and returns it. Directory paths must be absolute. // If the `initConfigDir` is an empty string, skips trying to load the init config. // If the `dynamicConfigDir` is an empty string, skips trying to load checkpoints or download new config, // but will still sync the ConfigOK condition if you call StartSync with a non-nil client. -func NewController(initConfigDir string, dynamicConfigDir string, defaultConfig *kubeletconfig.KubeletConfiguration) *Controller { +func NewController(initConfigDir string, + dynamicConfigDir string, + defaultConfig *kubeletconfig.KubeletConfiguration) (*Controller, error) { + var err error + fs := utilfs.DefaultFs{} var initLoader configfiles.Loader if len(initConfigDir) > 0 { - initLoader = configfiles.NewFSLoader(fs, initConfigDir) + initLoader, err = configfiles.NewFsLoader(fs, initConfigDir) + if err != nil { + return nil, err + } } dynamicConfig := false if len(dynamicConfigDir) > 0 { dynamicConfig = true } - // Get the current kubelet version; bad-config and startup-tracking information can be kubelet-version specific, - // e.g. a bug that crash loops an old Kubelet under a given config might be fixed in a new Kubelet or vice-versa, - // validation might be relaxed in a new Kubelet, etc. - // We also don't want a change in a file format to break Kubelet upgrades; this makes sure a new kubelet gets - // a fresh dir to put its config health data in. - // Note that config checkpoints use the api machinery to store ConfigMaps, and thus get file format versioning for free. - kubeletVersion := version.Get().String() - return &Controller{ dynamicConfig: dynamicConfig, defaultConfig: defaultConfig, @@ -117,10 +104,8 @@ func NewController(initConfigDir string, dynamicConfigDir string, defaultConfig pendingConfigSource: make(chan bool, 1), configOK: status.NewConfigOKCondition(), checkpointStore: store.NewFsStore(fs, filepath.Join(dynamicConfigDir, checkpointsDir)), - badConfigTracker: badconfig.NewFsTracker(fs, filepath.Join(dynamicConfigDir, badConfigTrackingDir, kubeletVersion)), - startupTracker: startups.NewFsTracker(fs, filepath.Join(dynamicConfigDir, startupTrackingDir, kubeletVersion)), initLoader: initLoader, - } + }, nil } // Bootstrap attempts to return a valid KubeletConfiguration based on the configuration of the Controller, @@ -163,11 +148,6 @@ func (cc *Controller) Bootstrap() (*kubeletconfig.KubeletConfiguration, error) { return nil, err } - // record the kubelet startup time, used for crashloop detection - if err := cc.startupTracker.RecordStartup(); err != nil { - return nil, err - } - // determine UID of the current config source curUID := "" if curSource, err := cc.checkpointStore.Current(); err != nil { @@ -181,14 +161,6 @@ func (cc *Controller) Bootstrap() (*kubeletconfig.KubeletConfiguration, error) { return cc.localConfig(), nil } // Assert: we will not use the local configurations, unless we roll back to lkg; curUID is non-empty - // check whether the current config is marked bad - if entry, err := cc.badConfigTracker.Entry(curUID); err != nil { - return nil, err - } else if entry != nil { - utillog.Infof("current config %q was marked bad for reason %q at time %q", curUID, entry.Reason, entry.Time) - return cc.lkgRollback(entry.Reason) - } - // TODO(mtaufen): consider re-verifying integrity and re-attempting download when a load/verify/parse/validate // error happens outside trial period, we already made it past the trial so it's probably filesystem corruption // or something else scary (unless someone is using a 0-length trial period) @@ -196,33 +168,26 @@ func (cc *Controller) Bootstrap() (*kubeletconfig.KubeletConfiguration, error) { // load the current config checkpoint, err := cc.checkpointStore.Load(curUID) if err != nil { - // TODO(mtaufen): rollback and mark bad for now, but this could reasonably be handled by re-attempting a download, + // TODO(mtaufen): rollback for now, but this could reasonably be handled by re-attempting a download, // it probably indicates some sort of corruption - return cc.badRollback(curUID, fmt.Sprintf(status.CurFailLoadReasonFmt, curUID), fmt.Sprintf("error: %v", err)) + return cc.lkgRollback(fmt.Sprintf(status.CurFailLoadReasonFmt, curUID), fmt.Sprintf("error: %v", err)) } // parse the checkpoint into a KubeletConfiguration cur, err := checkpoint.Parse() if err != nil { - return cc.badRollback(curUID, fmt.Sprintf(status.CurFailParseReasonFmt, curUID), fmt.Sprintf("error: %v", err)) + return cc.lkgRollback(fmt.Sprintf(status.CurFailParseReasonFmt, curUID), fmt.Sprintf("error: %v", err)) } // validate current config if err := validation.ValidateKubeletConfiguration(cur); err != nil { - return cc.badRollback(curUID, fmt.Sprintf(status.CurFailValidateReasonFmt, curUID), fmt.Sprintf("error: %v", err)) + return cc.lkgRollback(fmt.Sprintf(status.CurFailValidateReasonFmt, curUID), fmt.Sprintf("error: %v", err)) } - // check for crash loops if we're still in the trial period + // when the trial period is over, the current config becomes the last-known-good if trial, err := cc.inTrial(cur.ConfigTrialDuration.Duration); err != nil { return nil, err - } else if trial { - if crashing, err := cc.crashLooping(cur.CrashLoopThreshold); err != nil { - return nil, err - } else if crashing { - return cc.badRollback(curUID, fmt.Sprintf(status.CurFailCrashLoopReasonFmt, curUID), "") - } - } else { - // when the trial period is over, the current config becomes the last-known-good + } else if !trial { if err := cc.graduateCurrentToLastKnownGood(); err != nil { return nil, err } @@ -286,14 +251,6 @@ func (cc *Controller) initialize() error { if err := cc.checkpointStore.Initialize(); err != nil { return err } - // initialize bad config tracker - if err := cc.badConfigTracker.Initialize(); err != nil { - return err - } - // initialize startup tracker - if err := cc.startupTracker.Initialize(); err != nil { - return err - } return nil } @@ -320,21 +277,6 @@ func (cc *Controller) inTrial(trialDur time.Duration) (bool, error) { return false, nil } -// crashLooping returns true if the number of startups since the last modification of the current config exceeds `threshold`, false otherwise -func (cc *Controller) crashLooping(threshold int32) (bool, error) { - // determine the last time the current config changed - modTime, err := cc.checkpointStore.CurrentModified() - if err != nil { - return false, err - } - // get the number of startups since that modification time - num, err := cc.startupTracker.StartupsSince(modTime) - if err != nil { - return false, err - } - return num > threshold, nil -} - // graduateCurrentToLastKnownGood sets the last-known-good UID on the checkpointStore // to the same value as the current UID maintained by the checkpointStore func (cc *Controller) graduateCurrentToLastKnownGood() error { diff --git a/pkg/kubelet/kubeletconfig/rollback.go b/pkg/kubelet/kubeletconfig/rollback.go index 13e5acea3dc46..a2789566345b9 100644 --- a/pkg/kubelet/kubeletconfig/rollback.go +++ b/pkg/kubelet/kubeletconfig/rollback.go @@ -26,19 +26,10 @@ import ( utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log" ) -// badRollback makes an entry in the bad-config-tracking file for `uid` with `reason`, and returns the result of rolling back to the last-known-good config -func (cc *Controller) badRollback(uid, reason, detail string) (*kubeletconfig.KubeletConfiguration, error) { - utillog.Errorf(fmt.Sprintf("%s, %s", reason, detail)) - if err := cc.badConfigTracker.MarkBad(uid, reason); err != nil { - return nil, err - } - return cc.lkgRollback(reason) -} - // lkgRollback returns a valid last-known-good configuration, and updates the `cc.configOK` condition // regarding the `reason` for the rollback, or returns an error if a valid last-known-good could not be produced -func (cc *Controller) lkgRollback(reason string) (*kubeletconfig.KubeletConfiguration, error) { - utillog.Infof("rolling back to last-known-good config") +func (cc *Controller) lkgRollback(reason, detail string) (*kubeletconfig.KubeletConfiguration, error) { + utillog.Errorf(fmt.Sprintf("%s, %s", reason, detail)) lkgUID := "" if lkgSource, err := cc.checkpointStore.LastKnownGood(); err != nil { diff --git a/pkg/kubelet/kubeletconfig/startups/BUILD b/pkg/kubelet/kubeletconfig/startups/BUILD deleted file mode 100644 index 0326cad6f911d..0000000000000 --- a/pkg/kubelet/kubeletconfig/startups/BUILD +++ /dev/null @@ -1,48 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "fstracker.go", - "startups.go", - ], - deps = [ - "//pkg/kubelet/apis/kubeletconfig/validation:go_default_library", - "//pkg/kubelet/kubeletconfig/util/files:go_default_library", - "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library", - "//pkg/kubelet/kubeletconfig/util/log:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = [ - "fstracker_test.go", - "startups_test.go", - ], - library = ":go_default_library", - deps = [ - "//pkg/kubelet/kubeletconfig/util/files:go_default_library", - "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library", - "//pkg/kubelet/kubeletconfig/util/test:go_default_library", - ], -) diff --git a/pkg/kubelet/kubeletconfig/startups/fstracker.go b/pkg/kubelet/kubeletconfig/startups/fstracker.go deleted file mode 100644 index 1c1e5ce8e35c6..0000000000000 --- a/pkg/kubelet/kubeletconfig/startups/fstracker.go +++ /dev/null @@ -1,127 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package startups - -import ( - "encoding/json" - "fmt" - "path/filepath" - "time" - - utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files" - utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem" - utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log" -) - -const ( - startupsFile = "startups.json" -) - -// fsTracker tracks startups in the local filesystem -type fsTracker struct { - // fs is the filesystem to use for storage operations; can be mocked for testing - fs utilfs.Filesystem - // trackingDir is the absolute path to the storage directory for fsTracker - trackingDir string -} - -// NewFsTracker returns a Tracker that will store information in the `trackingDir` -func NewFsTracker(fs utilfs.Filesystem, trackingDir string) Tracker { - return &fsTracker{ - fs: fs, - trackingDir: trackingDir, - } -} - -func (tracker *fsTracker) Initialize() error { - utillog.Infof("initializing startups tracking directory %q", tracker.trackingDir) - if err := utilfiles.EnsureDir(tracker.fs, tracker.trackingDir); err != nil { - return err - } - if err := utilfiles.EnsureFile(tracker.fs, filepath.Join(tracker.trackingDir, startupsFile)); err != nil { - return err - } - return nil -} - -func (tracker *fsTracker) RecordStartup() error { - // load the file - ls, err := tracker.load() - if err != nil { - return err - } - - ls = recordStartup(ls) - - // save the file - err = tracker.save(ls) - if err != nil { - return err - } - return nil -} - -func (tracker *fsTracker) StartupsSince(t time.Time) (int32, error) { - // load the startups-tracking file - ls, err := tracker.load() - if err != nil { - return 0, err - } - return startupsSince(ls, t) -} - -// TODO(mtaufen): refactor into encode/decode like in badconfig.go - -// load loads the startups-tracking file from disk -func (tracker *fsTracker) load() ([]string, error) { - path := filepath.Join(tracker.trackingDir, startupsFile) - - // load the file - b, err := tracker.fs.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to load startups-tracking file %q, error: %v", path, err) - } - - // parse json into the slice - ls := []string{} - - // if the file is empty, just return empty slice - if len(b) == 0 { - return ls, nil - } - - // otherwise unmarshal the json - if err := json.Unmarshal(b, &ls); err != nil { - return nil, fmt.Errorf("failed to unmarshal json from startups-tracking file %q, error: %v", path, err) - } - return ls, nil -} - -// save replaces the contents of the startups-tracking file with `ls` -func (tracker *fsTracker) save(ls []string) error { - // marshal the json - b, err := json.Marshal(ls) - if err != nil { - return err - } - // save the file - path := filepath.Join(tracker.trackingDir, startupsFile) - if err := utilfiles.ReplaceFile(tracker.fs, path, b); err != nil { - return err - } - return nil -} diff --git a/pkg/kubelet/kubeletconfig/startups/fstracker_test.go b/pkg/kubelet/kubeletconfig/startups/fstracker_test.go deleted file mode 100644 index 0895c945982b3..0000000000000 --- a/pkg/kubelet/kubeletconfig/startups/fstracker_test.go +++ /dev/null @@ -1,294 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package startups - -import ( - "fmt" - "path/filepath" - "reflect" - "testing" - "time" - - utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files" - utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem" - utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test" -) - -const testTrackingDir = "/test-tracking-dir" - -// TODO(mtaufen): this file reuses a lot of test code from startups_test.go, should consolidate - -func newInitializedFakeFsTracker() (*fsTracker, error) { - fs := utilfs.NewFakeFs() - tracker := NewFsTracker(fs, testTrackingDir) - if err := tracker.Initialize(); err != nil { - return nil, err - } - return tracker.(*fsTracker), nil -} - -func TestFsTrackerInitialize(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("tracker.Initialize() failed with error: %v", err) - } - - // check that testTrackingDir exists - _, err = tracker.fs.Stat(testTrackingDir) - if err != nil { - t.Fatalf("expect %q to exist, but stat failed with error: %v", testTrackingDir, err) - } - - // check that testTrackingDir contains the startupsFile - path := filepath.Join(testTrackingDir, startupsFile) - _, err = tracker.fs.Stat(path) - if err != nil { - t.Fatalf("expect %q to exist, but stat failed with error: %v", path, err) - } -} - -func TestFsTrackerRecordStartup(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - now := time.Now() - - fullList := func() []string { - ls := []string{} - for i := maxStartups; i > 0; i-- { - // subtract decreasing amounts so timestamps increase but remain in the past - ls = append(ls, now.Add(-time.Duration(i)*time.Second).Format(time.RFC3339)) - } - return ls - }() - cases := []struct { - desc string - ls []string - expectHead []string // what we expect the first length-1 elements to look like after recording a new timestamp - expectLen int // how long the list should be after recording - }{ - // start empty - { - "start empty", - []string{}, - []string{}, - 1, - }, - // start non-empty - { - "start non-empty", - // subtract 1 so stamps are in the past - []string{now.Add(-1 * time.Second).Format(time.RFC3339)}, - []string{now.Add(-1 * time.Second).Format(time.RFC3339)}, - 2, - }, - // rotate list - { - "rotate list", - // make a slice with len == maxStartups, containing monotonically-increasing timestamps - fullList, - fullList[1:], - maxStartups, - }, - } - - for _, c := range cases { - // save the starting point, record a "startup" time, then load list from fs - if err := tracker.save(c.ls); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := tracker.RecordStartup(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - ls, err := tracker.load() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if c.expectLen != len(ls) { - t.Errorf("case %q, expected list %q to have length %d", c.desc, ls, c.expectLen) - } - if !reflect.DeepEqual(c.expectHead, ls[:len(ls)-1]) { - t.Errorf("case %q, expected elements 0 through n-1 of list %q to equal %q", c.desc, ls, c.expectHead) - } - // timestamps should be monotonically increasing (assuming system clock isn't jumping around at least) - if sorted, err := timestampsSorted(ls); err != nil { - t.Fatalf("unexpected error: %v", err) - } else if !sorted { - t.Errorf("case %q, expected monotonically increasing timestamps, but got %q", c.desc, ls) - } - } -} - -func TestFsTrackerStartupsSince(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - now, err := time.Parse(time.RFC3339, "2017-01-02T15:04:05Z") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - cases := []struct { - desc string - ls []string - expect int32 - err string - }{ - // empty list - {"empty list", []string{}, 0, ""}, - // no startups since - { - "no startups since", - []string{"2014-01-02T15:04:05Z", "2015-01-02T15:04:05Z", "2016-01-02T15:04:05Z"}, - 0, - "", - }, - // 2 startups since - { - "some startups since", - []string{"2016-01-02T15:04:05Z", "2018-01-02T15:04:05Z", "2019-01-02T15:04:05Z"}, - 2, - "", - }, - // all startups since - { - "all startups since", - []string{"2018-01-02T15:04:05Z", "2019-01-02T15:04:05Z", "2020-01-02T15:04:05Z"}, - 3, - "", - }, - // invalid timestamp - {"invalid timestamp", []string{"2018-01-02T15:04:05Z08:00"}, 0, "failed to parse"}, - } - - for _, c := range cases { - if err := tracker.save(c.ls); err != nil { - t.Fatalf("unexected error: %v", err) - } - num, err := tracker.StartupsSince(now) - if utiltest.SkipRest(t, c.desc, err, c.err) { - continue - } - if num != c.expect { - t.Errorf("case %q, expect %d startups but got %d", c.desc, c.expect, num) - } - } -} - -func TestFsTrackerLoad(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - nowstamp := time.Now().Format(time.RFC3339) - cases := []struct { - desc string - data []byte - expect []string - err string - }{ - // empty file - {"empty file", []byte(""), []string{}, ""}, - // empty list - {"empty list", []byte("[]"), []string{}, ""}, - // valid json - {"valid json", []byte(fmt.Sprintf(`["%s"]`, nowstamp)), []string{nowstamp}, ""}, - // invalid json - {"invalid json", []byte(`*`), []string{}, "failed to unmarshal"}, - } - - for _, c := range cases { - // save a file containing the correct serialization - utilfiles.ReplaceFile(tracker.fs, filepath.Join(testTrackingDir, startupsFile), c.data) - - // loading valid json should result in an object with the correct serialization - ls, err := tracker.load() - if utiltest.SkipRest(t, c.desc, err, c.err) { - continue - } - if !reflect.DeepEqual(c.expect, ls) { - // ls should equal expected decoded object - t.Errorf("case %q, expect %#v but got %#v", c.desc, c.expect, ls) - } - } - -} - -func TestFsTrackerSave(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - nowstamp := time.Now().Format(time.RFC3339) - cases := []struct { - desc string - ls []string - expect string - err string - }{ - // empty list - {"empty list", []string{}, "[]", ""}, - // 1-entry list - {"valid json", []string{nowstamp}, fmt.Sprintf(`["%s"]`, nowstamp), ""}, - } - - for _, c := range cases { - if err := tracker.save(c.ls); utiltest.SkipRest(t, c.desc, err, c.err) { - continue - } - - data, err := tracker.fs.ReadFile(filepath.Join(testTrackingDir, startupsFile)) - if err != nil { - t.Fatalf("failed to read startups file, error: %v", err) - } - json := string(data) - - if json != c.expect { - t.Errorf("case %q, expect %q but got %q", c.desc, c.expect, json) - } - } -} - -func TestFsTrackerRoundTrip(t *testing.T) { - tracker, err := newInitializedFakeFsTracker() - if err != nil { - t.Fatalf("failed to construct a tracker, error: %v", err) - } - - nowstamp := time.Now().Format(time.RFC3339) - expect := []string{nowstamp} - - // test that saving and loading an object results in the same value - err = tracker.save(expect) - if err != nil { - t.Fatalf("failed to save startups data, error: %v", err) - } - after, err := tracker.load() - if err != nil { - t.Fatalf("failed to load startups data, error: %v", err) - } - if !reflect.DeepEqual(expect, after) { - t.Errorf("expect round-tripping %#v to result in the same value, but got %#v", expect, after) - } -} diff --git a/pkg/kubelet/kubeletconfig/startups/startups.go b/pkg/kubelet/kubeletconfig/startups/startups.go deleted file mode 100644 index 6b384485d6210..0000000000000 --- a/pkg/kubelet/kubeletconfig/startups/startups.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package startups - -import ( - "fmt" - "time" - - "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/validation" -) - -const ( - // we allow one extra startup to account for the startup necessary to update configuration - maxStartups = validation.MaxCrashLoopThreshold + 1 -) - -// Tracker tracks Kubelet startups in a storage layer -type Tracker interface { - // Initialize sets up the storage layer - Initialize() error - // RecordStartup records the current time as a Kubelet startup - RecordStartup() error - // StartupsSince returns the number of Kubelet startus recorded since `t` - StartupsSince(t time.Time) (int32, error) -} - -func startupsSince(ls []string, start time.Time) (int32, error) { - // since the list is append-only we only need to count the number of timestamps since `t` - startups := int32(0) - for _, stamp := range ls { - t, err := time.Parse(time.RFC3339, stamp) - if err != nil { - return 0, fmt.Errorf("failed to parse timestamp while counting startups, error: %v", err) - } - if t.After(start) { - startups++ - } - } - return startups, nil -} - -func recordStartup(ls []string) []string { - // record current time - now := time.Now() - stamp := now.Format(time.RFC3339) // use RFC3339 time format - ls = append(ls, stamp) - - // rotate the slice if necessary - if len(ls) > maxStartups { - ls = ls[1:] - } - - // return the new slice - return ls -} diff --git a/pkg/kubelet/kubeletconfig/startups/startups_test.go b/pkg/kubelet/kubeletconfig/startups/startups_test.go deleted file mode 100644 index a0eafa68e3a62..0000000000000 --- a/pkg/kubelet/kubeletconfig/startups/startups_test.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package startups - -import ( - "reflect" - "testing" - "time" - - utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test" -) - -func TestRecordStartup(t *testing.T) { - now := time.Now() - - fullList := func() []string { - ls := []string{} - for i := maxStartups; i > 0; i-- { - // subtract decreasing amounts so timestamps increase but remain in the past - ls = append(ls, now.Add(-time.Duration(i)*time.Second).Format(time.RFC3339)) - } - return ls - }() - cases := []struct { - desc string - ls []string - expectHead []string // what we expect the first length-1 elements to look like after recording a new timestamp - expectLen int // how long the list should be after recording - }{ - // start empty - { - "start empty", - []string{}, - []string{}, - 1, - }, - // start non-empty - { - "start non-empty", - // subtract 1 so stamps are in the past - []string{now.Add(-1 * time.Second).Format(time.RFC3339)}, - []string{now.Add(-1 * time.Second).Format(time.RFC3339)}, - 2, - }, - // rotate list - { - "rotate list", - // make a slice with len == maxStartups, containing monotonically-increasing timestamps - fullList, - fullList[1:], - maxStartups, - }, - } - - for _, c := range cases { - ls := recordStartup(c.ls) - if c.expectLen != len(ls) { - t.Errorf("case %q, expected list %q to have length %d", c.desc, ls, c.expectLen) - } - if !reflect.DeepEqual(c.expectHead, ls[:len(ls)-1]) { - t.Errorf("case %q, expected elements 0 through n-1 of list %q to equal %q", c.desc, ls, c.expectHead) - } - // timestamps should be monotonically increasing (assuming system clock isn't jumping around at least) - if sorted, err := timestampsSorted(ls); err != nil { - t.Fatalf("unexpected error: %v", err) - } else if !sorted { - t.Errorf("case %q, expected monotonically increasing timestamps, but got %q", c.desc, ls) - } - } -} - -func TestStartupsSince(t *testing.T) { - now, err := time.Parse(time.RFC3339, "2017-01-02T15:04:05Z") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - cases := []struct { - desc string - ls []string - expect int32 - err string - }{ - // empty list - {"empty list", []string{}, 0, ""}, - // no startups since - { - "no startups since", - []string{"2014-01-02T15:04:05Z", "2015-01-02T15:04:05Z", "2016-01-02T15:04:05Z"}, - 0, - "", - }, - // 2 startups since - { - "some startups since", - []string{"2016-01-02T15:04:05Z", "2018-01-02T15:04:05Z", "2019-01-02T15:04:05Z"}, - 2, - "", - }, - // all startups since - { - "all startups since", - []string{"2018-01-02T15:04:05Z", "2019-01-02T15:04:05Z", "2020-01-02T15:04:05Z"}, - 3, - "", - }, - // invalid timestamp - {"invalid timestamp", []string{"2018-01-02T15:04:05Z08:00"}, 0, "failed to parse"}, - } - - for _, c := range cases { - num, err := startupsSince(c.ls, now) - if utiltest.SkipRest(t, c.desc, err, c.err) { - continue - } - if num != c.expect { - t.Errorf("case %q, expect %d startups but got %d", c.desc, c.expect, num) - } - } - -} - -// returns true if the timestamps are monotically increasing, false otherwise -func timestampsSorted(ls []string) (bool, error) { - if len(ls) < 2 { - return true, nil - } - prev, err := time.Parse(time.RFC3339, ls[0]) - if err != nil { - return false, err - } - for _, stamp := range ls[1:] { - cur, err := time.Parse(time.RFC3339, stamp) - if err != nil { - return false, err - } - if !cur.After(prev) { - return false, nil - } - prev = cur - } - return true, nil -} diff --git a/pkg/kubelet/kubeletconfig/status/status.go b/pkg/kubelet/kubeletconfig/status/status.go index f1c351b3c58d0..ea68e992a6bec 100644 --- a/pkg/kubelet/kubeletconfig/status/status.go +++ b/pkg/kubelet/kubeletconfig/status/status.go @@ -18,7 +18,6 @@ package status import ( "fmt" - "strings" "sync" "time" @@ -83,8 +82,8 @@ type ConfigOKCondition interface { Set(message, reason string, status apiv1.ConditionStatus) // SetFailedSyncCondition sets the condition for when syncing Kubelet config fails SetFailedSyncCondition(reason string) - // ClearFailedSyncCondition resets ConfigOKCondition to the correct condition for successfully syncing the kubelet config - ClearFailedSyncCondition(current string, lastKnownGood string, currentBadReason string, initConfig bool) + // ClearFailedSyncCondition clears the overlay from SetFailedSyncCondition + ClearFailedSyncCondition() // Sync patches the current condition into the Node identified by `nodeName` Sync(client clientset.Interface, nodeName string) } @@ -95,6 +94,8 @@ type configOKCondition struct { conditionMux sync.Mutex // condition is the current ConfigOK node condition, which will be reported in the Node.status.conditions condition *apiv1.NodeCondition + // failedSyncReason is sent in place of the usual reason when the Kubelet is failing to sync the remote config + failedSyncReason string // pendingCondition; write to this channel to indicate that ConfigOK needs to be synced to the API server pendingCondition chan bool } @@ -142,44 +143,20 @@ func (c *configOKCondition) Set(message, reason string, status apiv1.ConditionSt // SetFailedSyncCondition updates the ConfigOK status to reflect that we failed to sync to the latest config because we couldn't figure out what // config to use (e.g. due to a malformed reference, a download failure, etc) func (c *configOKCondition) SetFailedSyncCondition(reason string) { - c.Set(c.condition.Message, fmt.Sprintf("failed to sync, desired config unclear, reason: %s", reason), apiv1.ConditionUnknown) + c.conditionMux.Lock() + defer c.conditionMux.Unlock() + // set the reason overlay and poke the sync worker to send the update + c.failedSyncReason = fmt.Sprintf("failed to sync, desired config unclear, reason: %s", reason) + c.pokeSyncWorker() } -// ClearFailedSyncCondition resets ConfigOK to the correct condition for the config UIDs -// `current` and `lastKnownGood`, depending on whether current is bad (non-empty `currentBadReason`) -// and whether an init config exists (`initConfig` is true). -func (c *configOKCondition) ClearFailedSyncCondition(current string, - lastKnownGood string, - currentBadReason string, - initConfig bool) { - // since our reason-check relies on c.condition we must manually take the lock and use c.unsafeSet instead of c.Set +// ClearFailedSyncCondition removes the "failed to sync" reason overlay +func (c *configOKCondition) ClearFailedSyncCondition() { c.conditionMux.Lock() defer c.conditionMux.Unlock() - if strings.Contains(c.condition.Reason, "failed to sync, desired config unclear") { - // if we should report a "current is bad, rolled back" state - if len(currentBadReason) > 0 { - if len(current) == 0 { - if initConfig { - c.unsafeSet(LkgInitMessage, currentBadReason, apiv1.ConditionFalse) - return - } - c.unsafeSet(LkgDefaultMessage, currentBadReason, apiv1.ConditionFalse) - return - } - c.unsafeSet(fmt.Sprintf(LkgRemoteMessageFmt, lastKnownGood), currentBadReason, apiv1.ConditionFalse) - return - } - // if we should report a "current is ok" state - if len(current) == 0 { - if initConfig { - c.unsafeSet(CurInitMessage, CurInitOKReason, apiv1.ConditionTrue) - return - } - c.unsafeSet(CurDefaultMessage, CurDefaultOKReason, apiv1.ConditionTrue) - return - } - c.unsafeSet(fmt.Sprintf(CurRemoteMessageFmt, current), CurRemoteOKReason, apiv1.ConditionTrue) - } + // clear the reason overlay and poke the sync worker to send the update + c.failedSyncReason = "" + c.pokeSyncWorker() } // pokeSyncWorker notes that the ConfigOK condition needs to be synced to the API server @@ -239,6 +216,16 @@ func (c *configOKCondition) Sync(client clientset.Interface, nodeName string) { c.condition.LastTransitionTime = remote.LastTransitionTime } + // overlay the failedSyncReason if necessary + var condition *apiv1.NodeCondition + if len(c.failedSyncReason) > 0 { + // get a copy of the condition before we edit it + condition = c.condition.DeepCopy() + condition.Reason = c.failedSyncReason + } else { + condition = c.condition + } + // generate the patch mediaType := "application/json" info, ok := kuberuntime.SerializerInfoForMediaType(api.Codecs.SupportedMediaTypes(), mediaType) diff --git a/pkg/kubelet/kubeletconfig/util/codec/BUILD b/pkg/kubelet/kubeletconfig/util/codec/BUILD index 9a55c12caf4a2..df4916b26c879 100644 --- a/pkg/kubelet/kubeletconfig/util/codec/BUILD +++ b/pkg/kubelet/kubeletconfig/util/codec/BUILD @@ -12,9 +12,8 @@ go_library( "//pkg/api:go_default_library", "//pkg/api/install:go_default_library", "//pkg/kubelet/apis/kubeletconfig:go_default_library", - "//pkg/kubelet/apis/kubeletconfig/install:go_default_library", - "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", ], ) diff --git a/pkg/kubelet/kubeletconfig/util/codec/codec.go b/pkg/kubelet/kubeletconfig/util/codec/codec.go index 14432dae6e319..148967caba83a 100644 --- a/pkg/kubelet/kubeletconfig/util/codec/codec.go +++ b/pkg/kubelet/kubeletconfig/util/codec/codec.go @@ -21,13 +21,11 @@ import ( // ensure the core apis are installed _ "k8s.io/kubernetes/pkg/api/install" - // ensure the kubeletconfig apis are installed - _ "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/install" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" - kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" ) // TODO(mtaufen): allow an encoder to be injected into checkpoint objects at creation time? (then we could ultimately instantiate only one encoder) @@ -50,28 +48,18 @@ func NewJSONEncoder(groupName string) (runtime.Encoder, error) { return api.Codecs.EncoderForVersion(info.Serializer, versions[0]), nil } -// DecodeKubeletConfiguration decodes an encoded (v1alpha1) KubeletConfiguration object to the internal type -func DecodeKubeletConfiguration(data []byte) (*kubeletconfig.KubeletConfiguration, error) { - // decode the object, note we use the external version scheme to decode, because users provide the external version - obj, err := runtime.Decode(api.Codecs.UniversalDecoder(kubeletconfigv1alpha1.SchemeGroupVersion), data) +// DecodeKubeletConfiguration decodes a serialized KubeletConfiguration to the internal type +func DecodeKubeletConfiguration(kubeletCodecs *serializer.CodecFactory, data []byte) (*kubeletconfig.KubeletConfiguration, error) { + // the UniversalDecoder runs defaulting and returns the internal type by default + obj, gvk, err := kubeletCodecs.UniversalDecoder().Decode(data, nil, nil) if err != nil { return nil, fmt.Errorf("failed to decode, error: %v", err) } - externalKC, ok := obj.(*kubeletconfigv1alpha1.KubeletConfiguration) + internalKC, ok := obj.(*kubeletconfig.KubeletConfiguration) if !ok { - return nil, fmt.Errorf("failed to cast object to KubeletConfiguration, object: %#v", obj) + return nil, fmt.Errorf("failed to cast object to KubeletConfiguration, unexpected type: %v", gvk) } - // TODO(mtaufen): confirm whether api.Codecs.UniversalDecoder runs the defaulting, which would make this redundant - // run the defaulter on the decoded configuration before converting to internal type - api.Scheme.Default(externalKC) - - // convert to internal type - internalKC := &kubeletconfig.KubeletConfiguration{} - err = api.Scheme.Convert(externalKC, internalKC, nil) - if err != nil { - return nil, err - } return internalKC, nil } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index c2ba155e143d8..d57614aeb6902 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -790,7 +790,7 @@ func (m *kubeGenericRuntimeManager) GetAttach(id kubecontainer.ContainerID, stdi // RunInContainer synchronously executes the command in the container, and returns the output. func (m *kubeGenericRuntimeManager) RunInContainer(id kubecontainer.ContainerID, cmd []string, timeout time.Duration) ([]byte, error) { - stdout, stderr, err := m.runtimeService.ExecSync(id.ID, cmd, 0) + stdout, stderr, err := m.runtimeService.ExecSync(id.ID, cmd, timeout) // NOTE(tallclair): This does not correctly interleave stdout & stderr, but should be sufficient // for logging purposes. A combined output option will need to be added to the ExecSyncRequest // if more precise output ordering is ever required. diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index f89ca89a8f000..0be5b25d38d51 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -607,6 +607,11 @@ func (m *kubeGenericRuntimeManager) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStat if err != nil { createSandboxResult.Fail(kubecontainer.ErrCreatePodSandbox, msg) glog.Errorf("createPodSandbox for pod %q failed: %v", format.Pod(pod), err) + ref, err := ref.GetReference(api.Scheme, pod) + if err != nil { + glog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), err) + } + m.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedCreatePodSandBox, "Failed create pod sandbox.") return } glog.V(4).Infof("Created PodSandbox %q for pod %q", podSandboxID, format.Pod(pod)) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index fd5141dceae34..b8157b8c9406e 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -30,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" kubetypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/flowcontrol" "k8s.io/kubernetes/pkg/credentialprovider" apitest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing" @@ -216,15 +217,12 @@ func verifyPods(a, b []*kubecontainer.Pod) bool { return reflect.DeepEqual(a, b) } -func verifyFakeContainerList(fakeRuntime *apitest.FakeRuntimeService, expected []string) ([]string, bool) { - actual := []string{} +func verifyFakeContainerList(fakeRuntime *apitest.FakeRuntimeService, expected sets.String) (sets.String, bool) { + actual := sets.NewString() for _, c := range fakeRuntime.Containers { - actual = append(actual, c.Id) + actual.Insert(c.Id) } - sort.Sort(sort.StringSlice(actual)) - sort.Sort(sort.StringSlice(expected)) - - return actual, reflect.DeepEqual(expected, actual) + return actual, actual.Equal(expected) } type containerRecord struct { @@ -618,9 +616,9 @@ func TestPruneInitContainers(t *testing.T) { assert.NoError(t, err) m.pruneInitContainersBeforeStart(pod, podStatus) - expectedContainers := []string{fakes[0].Id, fakes[2].Id} + expectedContainers := sets.NewString(fakes[0].Id, fakes[2].Id) if actual, ok := verifyFakeContainerList(fakeRuntime, expectedContainers); !ok { - t.Errorf("expected %q, got %q", expectedContainers, actual) + t.Errorf("expected %v, got %v", expectedContainers, actual) } } diff --git a/pkg/kubelet/lifecycle/handlers.go b/pkg/kubelet/lifecycle/handlers.go index 3ee925cf25a5b..450c985e2a490 100644 --- a/pkg/kubelet/lifecycle/handlers.go +++ b/pkg/kubelet/lifecycle/handlers.go @@ -187,40 +187,34 @@ func (a *noNewPrivsAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult return PodAdmitResult{Admit: true} } - // Make sure it is either docker or rkt runtimes. - if a.Runtime.Type() != kubetypes.DockerContainerRuntime && a.Runtime.Type() != kubetypes.RktContainerRuntime { + // Always admit runtimes except docker. + if a.Runtime.Type() != kubetypes.DockerContainerRuntime { + return PodAdmitResult{Admit: true} + } + + // Make sure docker api version is valid. + rversion, err := a.Runtime.APIVersion() + if err != nil { return PodAdmitResult{ Admit: false, Reason: "NoNewPrivs", - Message: fmt.Sprintf("Cannot enforce NoNewPrivs: %s runtime not supported", a.Runtime.Type()), + Message: fmt.Sprintf("Cannot enforce NoNewPrivs: %v", err), } } - - if a.Runtime.Type() != kubetypes.DockerContainerRuntime { - // Make sure docker api version is valid. - rversion, err := a.Runtime.APIVersion() - if err != nil { - return PodAdmitResult{ - Admit: false, - Reason: "NoNewPrivs", - Message: fmt.Sprintf("Cannot enforce NoNewPrivs: %v", err), - } - } - v, err := rversion.Compare("1.23") - if err != nil { - return PodAdmitResult{ - Admit: false, - Reason: "NoNewPrivs", - Message: fmt.Sprintf("Cannot enforce NoNewPrivs: %v", err), - } + v, err := rversion.Compare("1.23.0") + if err != nil { + return PodAdmitResult{ + Admit: false, + Reason: "NoNewPrivs", + Message: fmt.Sprintf("Cannot enforce NoNewPrivs: %v", err), } - // If the version is less than 1.23 it will return -1 above. - if v == -1 { - return PodAdmitResult{ - Admit: false, - Reason: "NoNewPrivs", - Message: fmt.Sprintf("Cannot enforce NoNewPrivs: docker runtime API version %q must be greater than or equal to 1.23", rversion.String()), - } + } + // If the version is less than 1.23 it will return -1 above. + if v == -1 { + return PodAdmitResult{ + Admit: false, + Reason: "NoNewPrivs", + Message: fmt.Sprintf("Cannot enforce NoNewPrivs: docker runtime API version %q must be greater than or equal to 1.23", rversion.String()), } } diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index d2b1ed873c75a..0ae1edc801d80 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -119,6 +119,13 @@ func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNet glog.Warningf("Error loading CNI config file %s: %v", confFile, err) continue } + // Ensure the config has a "type" so we know what plugin to run. + // Also catches the case where somebody put a conflist into a conf file. + if conf.Network.Type == "" { + glog.Warningf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile) + continue + } + confList, err = libcni.ConfListFromConf(conf) if err != nil { glog.Warningf("Error converting CNI config file %s to list: %v", confFile, err) diff --git a/pkg/kubelet/pod/testing/BUILD b/pkg/kubelet/pod/testing/BUILD index 0120951709e38..0dd6bf9819d2d 100644 --- a/pkg/kubelet/pod/testing/BUILD +++ b/pkg/kubelet/pod/testing/BUILD @@ -7,10 +7,16 @@ load( go_library( name = "go_default_library", - srcs = ["fake_mirror_client.go"], + srcs = [ + "fake_mirror_client.go", + "mock_manager.go", + ], deps = [ "//pkg/kubelet/container:go_default_library", + "//pkg/kubelet/types:go_default_library", + "//vendor/github.com/stretchr/testify/mock:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], ) diff --git a/pkg/kubelet/pod/testing/mock_manager.go b/pkg/kubelet/pod/testing/mock_manager.go new file mode 100644 index 0000000000000..f15845b79ffd7 --- /dev/null +++ b/pkg/kubelet/pod/testing/mock_manager.go @@ -0,0 +1,291 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by mockery v1.0.0 +package testing + +import kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" +import mock "github.com/stretchr/testify/mock" + +import types "k8s.io/apimachinery/pkg/types" +import v1 "k8s.io/api/core/v1" + +// MockManager is an autogenerated mock type for the Manager type +type MockManager struct { + mock.Mock +} + +// AddPod provides a mock function with given fields: _a0 +func (_m *MockManager) AddPod(_a0 *v1.Pod) { + _m.Called(_a0) +} + +// CreateMirrorPod provides a mock function with given fields: _a0 +func (_m *MockManager) CreateMirrorPod(_a0 *v1.Pod) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(*v1.Pod) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeleteMirrorPod provides a mock function with given fields: podFullName +func (_m *MockManager) DeleteMirrorPod(podFullName string) error { + ret := _m.Called(podFullName) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(podFullName) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeleteOrphanedMirrorPods provides a mock function with given fields: +func (_m *MockManager) DeleteOrphanedMirrorPods() { + _m.Called() +} + +// DeletePod provides a mock function with given fields: _a0 +func (_m *MockManager) DeletePod(_a0 *v1.Pod) { + _m.Called(_a0) +} + +// GetMirrorPodByPod provides a mock function with given fields: _a0 +func (_m *MockManager) GetMirrorPodByPod(_a0 *v1.Pod) (*v1.Pod, bool) { + ret := _m.Called(_a0) + + var r0 *v1.Pod + if rf, ok := ret.Get(0).(func(*v1.Pod) *v1.Pod); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.Pod) + } + } + + var r1 bool + if rf, ok := ret.Get(1).(func(*v1.Pod) bool); ok { + r1 = rf(_a0) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// GetPodByFullName provides a mock function with given fields: podFullName +func (_m *MockManager) GetPodByFullName(podFullName string) (*v1.Pod, bool) { + ret := _m.Called(podFullName) + + var r0 *v1.Pod + if rf, ok := ret.Get(0).(func(string) *v1.Pod); ok { + r0 = rf(podFullName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.Pod) + } + } + + var r1 bool + if rf, ok := ret.Get(1).(func(string) bool); ok { + r1 = rf(podFullName) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// GetPodByMirrorPod provides a mock function with given fields: _a0 +func (_m *MockManager) GetPodByMirrorPod(_a0 *v1.Pod) (*v1.Pod, bool) { + ret := _m.Called(_a0) + + var r0 *v1.Pod + if rf, ok := ret.Get(0).(func(*v1.Pod) *v1.Pod); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.Pod) + } + } + + var r1 bool + if rf, ok := ret.Get(1).(func(*v1.Pod) bool); ok { + r1 = rf(_a0) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// GetPodByName provides a mock function with given fields: namespace, name +func (_m *MockManager) GetPodByName(namespace string, name string) (*v1.Pod, bool) { + ret := _m.Called(namespace, name) + + var r0 *v1.Pod + if rf, ok := ret.Get(0).(func(string, string) *v1.Pod); ok { + r0 = rf(namespace, name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.Pod) + } + } + + var r1 bool + if rf, ok := ret.Get(1).(func(string, string) bool); ok { + r1 = rf(namespace, name) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// GetPodByUID provides a mock function with given fields: _a0 +func (_m *MockManager) GetPodByUID(_a0 types.UID) (*v1.Pod, bool) { + ret := _m.Called(_a0) + + var r0 *v1.Pod + if rf, ok := ret.Get(0).(func(types.UID) *v1.Pod); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.Pod) + } + } + + var r1 bool + if rf, ok := ret.Get(1).(func(types.UID) bool); ok { + r1 = rf(_a0) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// GetPods provides a mock function with given fields: +func (_m *MockManager) GetPods() []*v1.Pod { + ret := _m.Called() + + var r0 []*v1.Pod + if rf, ok := ret.Get(0).(func() []*v1.Pod); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*v1.Pod) + } + } + + return r0 +} + +// GetPodsAndMirrorPods provides a mock function with given fields: +func (_m *MockManager) GetPodsAndMirrorPods() ([]*v1.Pod, []*v1.Pod) { + ret := _m.Called() + + var r0 []*v1.Pod + if rf, ok := ret.Get(0).(func() []*v1.Pod); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*v1.Pod) + } + } + + var r1 []*v1.Pod + if rf, ok := ret.Get(1).(func() []*v1.Pod); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]*v1.Pod) + } + } + + return r0, r1 +} + +// GetUIDTranslations provides a mock function with given fields: +func (_m *MockManager) GetUIDTranslations() (map[kubelettypes.ResolvedPodUID]kubelettypes.MirrorPodUID, map[kubelettypes.MirrorPodUID]kubelettypes.ResolvedPodUID) { + ret := _m.Called() + + var r0 map[kubelettypes.ResolvedPodUID]kubelettypes.MirrorPodUID + if rf, ok := ret.Get(0).(func() map[kubelettypes.ResolvedPodUID]kubelettypes.MirrorPodUID); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[kubelettypes.ResolvedPodUID]kubelettypes.MirrorPodUID) + } + } + + var r1 map[kubelettypes.MirrorPodUID]kubelettypes.ResolvedPodUID + if rf, ok := ret.Get(1).(func() map[kubelettypes.MirrorPodUID]kubelettypes.ResolvedPodUID); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(map[kubelettypes.MirrorPodUID]kubelettypes.ResolvedPodUID) + } + } + + return r0, r1 +} + +// IsMirrorPodOf provides a mock function with given fields: mirrorPod, _a1 +func (_m *MockManager) IsMirrorPodOf(mirrorPod *v1.Pod, _a1 *v1.Pod) bool { + ret := _m.Called(mirrorPod, _a1) + + var r0 bool + if rf, ok := ret.Get(0).(func(*v1.Pod, *v1.Pod) bool); ok { + r0 = rf(mirrorPod, _a1) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// SetPods provides a mock function with given fields: pods +func (_m *MockManager) SetPods(pods []*v1.Pod) { + _m.Called(pods) +} + +// TranslatePodUID provides a mock function with given fields: uid +func (_m *MockManager) TranslatePodUID(uid types.UID) kubelettypes.ResolvedPodUID { + ret := _m.Called(uid) + + var r0 kubelettypes.ResolvedPodUID + if rf, ok := ret.Get(0).(func(types.UID) kubelettypes.ResolvedPodUID); ok { + r0 = rf(uid) + } else { + r0 = ret.Get(0).(kubelettypes.ResolvedPodUID) + } + + return r0 +} + +// UpdatePod provides a mock function with given fields: _a0 +func (_m *MockManager) UpdatePod(_a0 *v1.Pod) { + _m.Called(_a0) +} diff --git a/pkg/kubelet/preemption/preemption_test.go b/pkg/kubelet/preemption/preemption_test.go index fcd1950c360a0..331b4f72a5775 100644 --- a/pkg/kubelet/preemption/preemption_test.go +++ b/pkg/kubelet/preemption/preemption_test.go @@ -338,47 +338,47 @@ func getTestPods() map[string]*v1.Pod { allPods := map[string]*v1.Pod{ tinyBurstable: getPodWithResources(tinyBurstable, v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("1m"), - "memory": resource.MustParse("1Mi"), + v1.ResourceCPU: resource.MustParse("1m"), + v1.ResourceMemory: resource.MustParse("1Mi"), }, }), bestEffort: getPodWithResources(bestEffort, v1.ResourceRequirements{}), critical: getPodWithResources(critical, v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("100m"), - "memory": resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), }, }), burstable: getPodWithResources(burstable, v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("100m"), - "memory": resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), }, }), guaranteed: getPodWithResources(guaranteed, v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("100m"), - "memory": resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), }, Limits: v1.ResourceList{ - "cpu": resource.MustParse("100m"), - "memory": resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), }, }), highRequestBurstable: getPodWithResources(highRequestBurstable, v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("300m"), - "memory": resource.MustParse("300Mi"), + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("300Mi"), }, }), highRequestGuaranteed: getPodWithResources(highRequestGuaranteed, v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("300m"), - "memory": resource.MustParse("300Mi"), + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("300Mi"), }, Limits: v1.ResourceList{ - "cpu": resource.MustParse("300m"), - "memory": resource.MustParse("300Mi"), + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("300Mi"), }, }), } diff --git a/pkg/kubelet/remote/remote_runtime.go b/pkg/kubelet/remote/remote_runtime.go index c8bfd6d1ef065..974c0931ff356 100644 --- a/pkg/kubelet/remote/remote_runtime.go +++ b/pkg/kubelet/remote/remote_runtime.go @@ -23,6 +23,7 @@ import ( "time" "github.com/golang/glog" + "golang.org/x/net/context" "google.golang.org/grpc" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" @@ -308,9 +309,14 @@ func (r *RemoteRuntimeService) UpdateContainerResources(containerID string, reso // ExecSync executes a command in the container, and returns the stdout output. // If command exits with a non-zero exit code, an error is returned. func (r *RemoteRuntimeService) ExecSync(containerID string, cmd []string, timeout time.Duration) (stdout []byte, stderr []byte, err error) { - ctx, cancel := getContextWithTimeout(timeout) - if timeout == 0 { - // Do not set timeout when timeout is 0. + // Do not set timeout when timeout is 0. + var ctx context.Context + var cancel context.CancelFunc + if timeout != 0 { + // Use timeout + default timeout (2 minutes) as timeout to leave some time for + // the runtime to do cleanup. + ctx, cancel = getContextWithTimeout(r.timeout + timeout) + } else { ctx, cancel = getContextWithCancel() } defer cancel() diff --git a/pkg/kubelet/rkt/rkt_test.go b/pkg/kubelet/rkt/rkt_test.go index 9a1a7d75b3951..7dda72763bf0b 100644 --- a/pkg/kubelet/rkt/rkt_test.go +++ b/pkg/kubelet/rkt/rkt_test.go @@ -1073,8 +1073,8 @@ func TestSetApp(t *testing.T) { Command: []string{"/bin/bar", "$(env-bar)"}, WorkingDir: tmpDir, Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{"cpu": resource.MustParse("50m"), "memory": resource.MustParse("50M")}, - Requests: v1.ResourceList{"cpu": resource.MustParse("5m"), "memory": resource.MustParse("5M")}, + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("50m"), v1.ResourceMemory: resource.MustParse("50M")}, + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m"), v1.ResourceMemory: resource.MustParse("5M")}, }, }, mountPoints: []appctypes.MountPoint{ @@ -1137,8 +1137,8 @@ func TestSetApp(t *testing.T) { Args: []string{"hello", "world", "$(env-bar)"}, WorkingDir: tmpDir, Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{"cpu": resource.MustParse("50m")}, - Requests: v1.ResourceList{"memory": resource.MustParse("5M")}, + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("50m")}, + Requests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("5M")}, }, }, mountPoints: []appctypes.MountPoint{ diff --git a/pkg/kubelet/runonce_test.go b/pkg/kubelet/runonce_test.go index 7449c271cc3dc..3dcc0301a75b4 100644 --- a/pkg/kubelet/runonce_test.go +++ b/pkg/kubelet/runonce_test.go @@ -92,7 +92,7 @@ func TestRunOnce(t *testing.T) { plug := &volumetest.FakeVolumePlugin{PluginName: "fake", Host: nil} kb.volumePluginMgr, err = - NewInitializedVolumePluginMgr(kb, fakeSecretManager, fakeConfigMapManager, []volume.VolumePlugin{plug}) + NewInitializedVolumePluginMgr(kb, fakeSecretManager, fakeConfigMapManager, []volume.VolumePlugin{plug}, nil /* prober */) if err != nil { t.Fatalf("failed to initialize VolumePluginMgr: %v", err) } @@ -113,7 +113,7 @@ func TestRunOnce(t *testing.T) { kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), kubeletconfig.HairpinNone, "", network.UseDefaultMTU) // TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency volumeStatsAggPeriod := time.Second * 10 - kb.resourceAnalyzer = stats.NewResourceAnalyzer(kb, volumeStatsAggPeriod, kb.containerRuntime) + kb.resourceAnalyzer = stats.NewResourceAnalyzer(kb, volumeStatsAggPeriod) nodeRef := &v1.ObjectReference{ Kind: "Node", Name: string(kb.nodeName), diff --git a/pkg/kubelet/server/BUILD b/pkg/kubelet/server/BUILD index cfc4800fa0452..61c57935567e6 100644 --- a/pkg/kubelet/server/BUILD +++ b/pkg/kubelet/server/BUILD @@ -16,7 +16,6 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/api/v1/validation:go_default_library", - "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/server/portforward:go_default_library", "//pkg/kubelet/server/remotecommand:go_default_library", @@ -25,11 +24,9 @@ go_library( "//pkg/kubelet/types:go_default_library", "//pkg/util/configz:go_default_library", "//pkg/util/limitwriter:go_default_library", - "//pkg/volume:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", - "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/github.com/google/cadvisor/metrics:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library", @@ -61,6 +58,7 @@ go_test( deps = [ "//pkg/api:go_default_library", "//pkg/api/install:go_default_library", + "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/container/testing:go_default_library", @@ -69,7 +67,6 @@ go_test( "//pkg/kubelet/server/stats:go_default_library", "//pkg/volume:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", - "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/golang.org/x/net/websocket:go_default_library", diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index 74a5036828a59..285a0d376cefb 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -33,7 +33,6 @@ import ( restful "github.com/emicklei/go-restful" "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/google/cadvisor/metrics" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -53,7 +52,6 @@ import ( "k8s.io/client-go/tools/remotecommand" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1/validation" - "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/server/portforward" remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" @@ -62,7 +60,6 @@ import ( kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/configz" "k8s.io/kubernetes/pkg/util/limitwriter" - "k8s.io/kubernetes/pkg/volume" ) const ( @@ -169,14 +166,10 @@ type AuthInterface interface { // HostInterface contains all the kubelet methods required by the server. // For testability. type HostInterface interface { - GetContainerInfo(podFullName string, uid types.UID, containerName string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) - GetContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) - GetRawContainerInfo(containerName string, req *cadvisorapi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorapi.ContainerInfo, error) + stats.StatsProvider GetVersionInfo() (*cadvisorapi.VersionInfo, error) GetCachedMachineInfo() (*cadvisorapi.MachineInfo, error) - GetPods() []*v1.Pod GetRunningPods() ([]*v1.Pod, error) - GetPodByName(namespace, name string) (*v1.Pod, bool) RunInContainer(name string, uid types.UID, container string, cmd []string) ([]byte, error) ExecInContainer(name string, uid types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error AttachContainer(name string, uid types.UID, container string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error @@ -186,12 +179,7 @@ type HostInterface interface { StreamingConnectionIdleTimeout() time.Duration ResyncInterval() time.Duration GetHostname() string - GetNode() (*v1.Node, error) - GetNodeConfig() cm.NodeConfig LatestLoopEntryTime() time.Time - ImagesFsInfo() (cadvisorapiv2.FsInfo, error) - RootFsInfo() (cadvisorapiv2.FsInfo, error) - ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) GetExec(podFullName string, podUID types.UID, containerName string, cmd []string, streamOpts remotecommandserver.Options) (*url.URL, error) GetAttach(podFullName string, podUID types.UID, containerName string, streamOpts remotecommandserver.Options) (*url.URL, error) GetPortForward(podName, podNamespace string, podUID types.UID, portForwardOpts portforward.V4Options) (*url.URL, error) @@ -813,21 +801,29 @@ func (a prometheusHostAdapter) GetMachineInfo() (*cadvisorapi.MachineInfo, error // containerPrometheusLabels maps cAdvisor labels to prometheus labels. func containerPrometheusLabels(c *cadvisorapi.ContainerInfo) map[string]string { - set := map[string]string{metrics.LabelID: c.Name} + // Prometheus requires that all metrics in the same family have the same labels, + // so we arrange to supply blank strings for missing labels + var name, image, podName, namespace, containerName string if len(c.Aliases) > 0 { - set[metrics.LabelName] = c.Aliases[0] - } - if image := c.Spec.Image; len(image) > 0 { - set[metrics.LabelImage] = image + name = c.Aliases[0] } + image = c.Spec.Image if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNameLabel]; ok { - set["pod_name"] = v + podName = v } if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNamespaceLabel]; ok { - set["namespace"] = v + namespace = v } if v, ok := c.Spec.Labels[kubelettypes.KubernetesContainerNameLabel]; ok { - set["container_name"] = v + containerName = v + } + set := map[string]string{ + metrics.LabelID: c.Name, + metrics.LabelName: name, + metrics.LabelImage: image, + "pod_name": podName, + "namespace": namespace, + "container_name": containerName, } return set } diff --git a/pkg/kubelet/server/server_test.go b/pkg/kubelet/server/server_test.go index b3b5ce9b46e44..85e62b2aae1d1 100644 --- a/pkg/kubelet/server/server_test.go +++ b/pkg/kubelet/server/server_test.go @@ -35,7 +35,6 @@ import ( "time" cadvisorapi "github.com/google/cadvisor/info/v1" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/api/core/v1" @@ -49,6 +48,7 @@ import ( "k8s.io/client-go/tools/remotecommand" utiltesting "k8s.io/client-go/util/testing" "k8s.io/kubernetes/pkg/api" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" // Do some initialization to decode the query parameters correctly. _ "k8s.io/kubernetes/pkg/api/install" "k8s.io/kubernetes/pkg/kubelet/cm" @@ -166,18 +166,6 @@ func (fk *fakeKubelet) StreamingConnectionIdleTimeout() time.Duration { } // Unused functions -func (_ *fakeKubelet) GetContainerInfoV2(_ string, _ cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) { - return nil, nil -} - -func (_ *fakeKubelet) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) { - return cadvisorapiv2.FsInfo{}, fmt.Errorf("Unsupported Operation ImagesFsInfo") -} - -func (_ *fakeKubelet) RootFsInfo() (cadvisorapiv2.FsInfo, error) { - return cadvisorapiv2.FsInfo{}, fmt.Errorf("Unsupport Operation RootFsInfo") -} - func (_ *fakeKubelet) GetNode() (*v1.Node, error) { return nil, nil } func (_ *fakeKubelet) GetNodeConfig() cm.NodeConfig { return cm.NodeConfig{} } @@ -185,6 +173,13 @@ func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Vo return map[string]volume.Volume{}, true } +func (_ *fakeKubelet) RootFsStats() (*statsapi.FsStats, error) { return nil, nil } +func (_ *fakeKubelet) ListPodStats() ([]statsapi.PodStats, error) { return nil, nil } +func (_ *fakeKubelet) ImageFsStats() (*statsapi.FsStats, error) { return nil, nil } +func (_ *fakeKubelet) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) { + return nil, nil, nil +} + type fakeAuth struct { authenticateFunc func(*http.Request) (user.Info, bool, error) attributesFunc func(user.Info, *http.Request) authorizer.Attributes @@ -242,7 +237,7 @@ func newServerTest() *serverTestFramework { } server := NewServer( fw.fakeKubelet, - stats.NewResourceAnalyzer(fw.fakeKubelet, time.Minute, &kubecontainertesting.FakeRuntime{}), + stats.NewResourceAnalyzer(fw.fakeKubelet, time.Minute), fw.fakeAuth, true, false, diff --git a/pkg/kubelet/server/stats/BUILD b/pkg/kubelet/server/stats/BUILD index 8c026960d4967..a1e299d9eb0f9 100644 --- a/pkg/kubelet/server/stats/BUILD +++ b/pkg/kubelet/server/stats/BUILD @@ -1,10 +1,4 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -16,19 +10,16 @@ go_library( "summary.go", "volume_stat_calculator.go", ], + visibility = ["//visibility:public"], deps = [ "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container:go_default_library", - "//pkg/kubelet/leaky:go_default_library", - "//pkg/kubelet/network:go_default_library", - "//pkg/kubelet/types:go_default_library", "//pkg/kubelet/util/format:go_default_library", "//pkg/volume:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", - "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", @@ -38,25 +29,16 @@ go_library( go_test( name = "go_default_test", - srcs = [ - "mocks_test.go", - "summary_test.go", - ], + srcs = ["summary_test.go"], library = ":go_default_library", deps = [ "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cm:go_default_library", - "//pkg/kubelet/container:go_default_library", - "//pkg/kubelet/leaky:go_default_library", - "//pkg/volume:go_default_library", - "//vendor/github.com/google/cadvisor/info/v1:go_default_library", - "//vendor/github.com/google/cadvisor/info/v2:go_default_library", + "//pkg/kubelet/server/stats/testing:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", - "//vendor/github.com/stretchr/testify/mock:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", ], ) @@ -69,6 +51,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//pkg/kubelet/server/stats/testing:all-srcs", + ], tags = ["automanaged"], + visibility = ["//visibility:public"], ) diff --git a/pkg/kubelet/server/stats/handler.go b/pkg/kubelet/server/stats/handler.go index f64095b20fd9f..969b25f3de127 100644 --- a/pkg/kubelet/server/stats/handler.go +++ b/pkg/kubelet/server/stats/handler.go @@ -24,14 +24,14 @@ import ( "path" "time" + restful "github.com/emicklei/go-restful" "github.com/golang/glog" cadvisorapi "github.com/google/cadvisor/info/v1" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" - "github.com/emicklei/go-restful" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/volume" @@ -39,15 +39,44 @@ import ( // Host methods required by stats handlers. type StatsProvider interface { + // The following stats are provided by either CRI or cAdvisor. + // + // ListPodStats returns the stats of all the containers managed by pods. + ListPodStats() ([]statsapi.PodStats, error) + // ImageFsStats returns the stats of the image filesystem. + ImageFsStats() (*statsapi.FsStats, error) + + // The following stats are provided by cAdvisor. + // + // GetCgroupStats returns the stats and the networking usage of the cgroup + // with the specified cgroupName. + GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) + // RootFsStats returns the stats of the node root filesystem. + RootFsStats() (*statsapi.FsStats, error) + + // The following stats are provided by cAdvisor for legacy usage. + // + // GetContainerInfo returns the information of the container with the + // containerName managed by the pod with the uid. GetContainerInfo(podFullName string, uid types.UID, containerName string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) - GetContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) + // GetRawContainerInfo returns the information of the container with the + // containerName. If subcontainers is true, this function will return the + // information of all the sub-containers as well. GetRawContainerInfo(containerName string, req *cadvisorapi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorapi.ContainerInfo, error) + + // The following information is provided by Kubelet. + // + // GetPodByName returns the spec of the pod with the name in the specified + // namespace. GetPodByName(namespace, name string) (*v1.Pod, bool) + // GetNode returns the spec of the local node. GetNode() (*v1.Node, error) + // GetNodeConfig returns the configuration of the local node. GetNodeConfig() cm.NodeConfig - ImagesFsInfo() (cadvisorapiv2.FsInfo, error) - RootFsInfo() (cadvisorapiv2.FsInfo, error) + // ListVolumesForPod returns the stats of the volume used by the pod with + // the podUID. ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) + // GetPods returns the specs of all the pods running on this node. GetPods() []*v1.Pod } diff --git a/pkg/kubelet/server/stats/mocks_test.go b/pkg/kubelet/server/stats/mocks_test.go deleted file mode 100644 index 5b044e9195e68..0000000000000 --- a/pkg/kubelet/server/stats/mocks_test.go +++ /dev/null @@ -1,244 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package stats - -import ( - cadvisorapi "github.com/google/cadvisor/info/v1" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" - "github.com/stretchr/testify/mock" - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/volume" -) - -// DO NOT EDIT -// GENERATED BY mockery - -type MockStatsProvider struct { - mock.Mock -} - -// GetContainerInfo provides a mock function with given fields: podFullName, uid, containerName, req -func (_m *MockStatsProvider) GetContainerInfo(podFullName string, uid types.UID, containerName string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) { - ret := _m.Called(podFullName, uid, containerName, req) - - var r0 *cadvisorapi.ContainerInfo - if rf, ok := ret.Get(0).(func(string, types.UID, string, *cadvisorapi.ContainerInfoRequest) *cadvisorapi.ContainerInfo); ok { - r0 = rf(podFullName, uid, containerName, req) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*cadvisorapi.ContainerInfo) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, types.UID, string, *cadvisorapi.ContainerInfoRequest) error); ok { - r1 = rf(podFullName, uid, containerName, req) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetContainerInfoV2 provides a mock function with given fields: name, options -func (_m *MockStatsProvider) GetContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) { - ret := _m.Called(name, options) - - var r0 map[string]cadvisorapiv2.ContainerInfo - if rf, ok := ret.Get(0).(func(string, cadvisorapiv2.RequestOptions) map[string]cadvisorapiv2.ContainerInfo); ok { - r0 = rf(name, options) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]cadvisorapiv2.ContainerInfo) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, cadvisorapiv2.RequestOptions) error); ok { - r1 = rf(name, options) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetRawContainerInfo provides a mock function with given fields: containerName, req, subcontainers -func (_m *MockStatsProvider) GetRawContainerInfo(containerName string, req *cadvisorapi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorapi.ContainerInfo, error) { - ret := _m.Called(containerName, req, subcontainers) - - var r0 map[string]*cadvisorapi.ContainerInfo - if rf, ok := ret.Get(0).(func(string, *cadvisorapi.ContainerInfoRequest, bool) map[string]*cadvisorapi.ContainerInfo); ok { - r0 = rf(containerName, req, subcontainers) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]*cadvisorapi.ContainerInfo) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, *cadvisorapi.ContainerInfoRequest, bool) error); ok { - r1 = rf(containerName, req, subcontainers) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetPodByName provides a mock function with given fields: namespace, name -func (_m *MockStatsProvider) GetPodByName(namespace string, name string) (*v1.Pod, bool) { - ret := _m.Called(namespace, name) - - var r0 *v1.Pod - if rf, ok := ret.Get(0).(func(string, string) *v1.Pod); ok { - r0 = rf(namespace, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*v1.Pod) - } - } - - var r1 bool - if rf, ok := ret.Get(1).(func(string, string) bool); ok { - r1 = rf(namespace, name) - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 -} - -// GetNode provides a mock function with given fields: -func (_m *MockStatsProvider) GetNode() (*v1.Node, error) { - ret := _m.Called() - - var r0 *v1.Node - if rf, ok := ret.Get(0).(func() *v1.Node); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*v1.Node) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetNodeConfig provides a mock function with given fields: -func (_m *MockStatsProvider) GetNodeConfig() cm.NodeConfig { - ret := _m.Called() - - var r0 cm.NodeConfig - if rf, ok := ret.Get(0).(func() cm.NodeConfig); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(cm.NodeConfig) - } - - return r0 -} - -// ImagesFsInfo provides a mock function with given fields: -func (_m *MockStatsProvider) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) { - ret := _m.Called() - - var r0 cadvisorapiv2.FsInfo - if rf, ok := ret.Get(0).(func() cadvisorapiv2.FsInfo); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(cadvisorapiv2.FsInfo) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// RootFsInfo provides a mock function with given fields: -func (_m *MockStatsProvider) RootFsInfo() (cadvisorapiv2.FsInfo, error) { - ret := _m.Called() - - var r0 cadvisorapiv2.FsInfo - if rf, ok := ret.Get(0).(func() cadvisorapiv2.FsInfo); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(cadvisorapiv2.FsInfo) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListVolumesForPod provides a mock function with given fields: podUID -func (_m *MockStatsProvider) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) { - ret := _m.Called(podUID) - - var r0 map[string]volume.Volume - if rf, ok := ret.Get(0).(func(types.UID) map[string]volume.Volume); ok { - r0 = rf(podUID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]volume.Volume) - } - } - - var r1 bool - if rf, ok := ret.Get(1).(func(types.UID) bool); ok { - r1 = rf(podUID) - } else { - r1 = ret.Get(1).(bool) - } - - return r0, r1 -} - -// GetPods provides a mock function with given fields: -func (_m *MockStatsProvider) GetPods() []*v1.Pod { - ret := _m.Called() - - var r0 []*v1.Pod - if rf, ok := ret.Get(0).(func() []*v1.Pod); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*v1.Pod) - } - } - - return r0 -} diff --git a/pkg/kubelet/server/stats/resource_analyzer.go b/pkg/kubelet/server/stats/resource_analyzer.go index 670e7a751aca4..5def6ba714d88 100644 --- a/pkg/kubelet/server/stats/resource_analyzer.go +++ b/pkg/kubelet/server/stats/resource_analyzer.go @@ -18,8 +18,6 @@ package stats import ( "time" - - "k8s.io/kubernetes/pkg/kubelet/container" ) // ResourceAnalyzer provides statistics on node resource consumption @@ -39,9 +37,9 @@ type resourceAnalyzer struct { var _ ResourceAnalyzer = &resourceAnalyzer{} // NewResourceAnalyzer returns a new ResourceAnalyzer -func NewResourceAnalyzer(statsProvider StatsProvider, calVolumeFrequency time.Duration, runtime container.Runtime) ResourceAnalyzer { +func NewResourceAnalyzer(statsProvider StatsProvider, calVolumeFrequency time.Duration) ResourceAnalyzer { fsAnalyzer := newFsResourceAnalyzer(statsProvider, calVolumeFrequency) - summaryProvider := NewSummaryProvider(statsProvider, fsAnalyzer, runtime) + summaryProvider := NewSummaryProvider(statsProvider) return &resourceAnalyzer{fsAnalyzer, summaryProvider} } diff --git a/pkg/kubelet/server/stats/summary.go b/pkg/kubelet/server/stats/summary.go index c45e16054a95e..313d0d57ea562 100644 --- a/pkg/kubelet/server/stats/summary.go +++ b/pkg/kubelet/server/stats/summary.go @@ -18,513 +18,85 @@ package stats import ( "fmt" - "sort" - "strings" - "time" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubetypes "k8s.io/apimachinery/pkg/types" - stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" - "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/kubelet/leaky" - "k8s.io/kubernetes/pkg/kubelet/network" - "k8s.io/kubernetes/pkg/kubelet/types" "github.com/golang/glog" - cadvisorapiv1 "github.com/google/cadvisor/info/v1" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" ) type SummaryProvider interface { - // Get provides a new Summary using the latest results from cadvisor - Get() (*stats.Summary, error) + Get() (*statsapi.Summary, error) } +// summaryProviderImpl implements the SummaryProvider interface. type summaryProviderImpl struct { - provider StatsProvider - fsResourceAnalyzer fsResourceAnalyzerInterface - runtime container.Runtime + provider StatsProvider } var _ SummaryProvider = &summaryProviderImpl{} -// NewSummaryProvider returns a new SummaryProvider -func NewSummaryProvider(statsProvider StatsProvider, fsResourceAnalyzer fsResourceAnalyzerInterface, cruntime container.Runtime) SummaryProvider { - return &summaryProviderImpl{statsProvider, fsResourceAnalyzer, cruntime} +// NewSummaryProvider returns a SummaryProvider using the stats provided by the +// specified statsProvider. +func NewSummaryProvider(statsProvider StatsProvider) SummaryProvider { + return &summaryProviderImpl{statsProvider} } -// Get implements the SummaryProvider interface -// Query cadvisor for the latest resource metrics and build into a summary -func (sp *summaryProviderImpl) Get() (*stats.Summary, error) { - options := cadvisorapiv2.RequestOptions{ - IdType: cadvisorapiv2.TypeName, - Count: 2, // 2 samples are needed to compute "instantaneous" CPU - Recursive: true, - } - infos, err := sp.provider.GetContainerInfoV2("/", options) - if err != nil { - if _, ok := infos["/"]; ok { - // If the failure is partial, log it and return a best-effort response. - glog.Errorf("Partial failure issuing GetContainerInfoV2: %v", err) - } else { - return nil, fmt.Errorf("failed GetContainerInfoV2: %v", err) - } - } - - // TODO(tallclair): Consider returning a best-effort response if any of the following errors - // occur. +// Get provides a new Summary with the stats from Kubelet. +func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) { + // TODO(timstclair): Consider returning a best-effort response if any of + // the following errors occur. node, err := sp.provider.GetNode() if err != nil { - return nil, fmt.Errorf("failed GetNode: %v", err) + return nil, fmt.Errorf("failed to get node info: %v", err) } - nodeConfig := sp.provider.GetNodeConfig() - rootFsInfo, err := sp.provider.RootFsInfo() + rootStats, networkStats, err := sp.provider.GetCgroupStats("/") if err != nil { - return nil, fmt.Errorf("failed RootFsInfo: %v", err) + return nil, fmt.Errorf("failed to get root cgroup stats: %v", err) } - imageFsInfo, err := sp.provider.ImagesFsInfo() + rootFsStats, err := sp.provider.RootFsStats() if err != nil { - return nil, fmt.Errorf("failed DockerImagesFsInfo: %v", err) + return nil, fmt.Errorf("failed to get rootFs stats: %v", err) } - imageStats, err := sp.runtime.ImageStats() - if err != nil || imageStats == nil { - return nil, fmt.Errorf("failed ImageStats: %v", err) - } - sb := &summaryBuilder{sp.fsResourceAnalyzer, node, nodeConfig, rootFsInfo, imageFsInfo, *imageStats, infos} - return sb.build() -} - -// summaryBuilder aggregates the datastructures provided by cadvisor into a Summary result -type summaryBuilder struct { - fsResourceAnalyzer fsResourceAnalyzerInterface - node *v1.Node - nodeConfig cm.NodeConfig - rootFsInfo cadvisorapiv2.FsInfo - imageFsInfo cadvisorapiv2.FsInfo - imageStats container.ImageStats - infos map[string]cadvisorapiv2.ContainerInfo -} - -// build returns a Summary from aggregating the input data -func (sb *summaryBuilder) build() (*stats.Summary, error) { - rootInfo, found := sb.infos["/"] - if !found { - return nil, fmt.Errorf("Missing stats for root container") - } - - var nodeFsInodesUsed *uint64 - if sb.rootFsInfo.Inodes != nil && sb.rootFsInfo.InodesFree != nil { - nodeFsIU := *sb.rootFsInfo.Inodes - *sb.rootFsInfo.InodesFree - nodeFsInodesUsed = &nodeFsIU + imageFsStats, err := sp.provider.ImageFsStats() + if err != nil { + return nil, fmt.Errorf("failed to get imageFs stats: %v", err) } - - var imageFsInodesUsed *uint64 - if sb.imageFsInfo.Inodes != nil && sb.imageFsInfo.InodesFree != nil { - imageFsIU := *sb.imageFsInfo.Inodes - *sb.imageFsInfo.InodesFree - imageFsInodesUsed = &imageFsIU + podStats, err := sp.provider.ListPodStats() + if err != nil { + return nil, fmt.Errorf("failed to list pod stats: %v", err) } - rootStats := sb.containerInfoV2ToStats("", &rootInfo) - cStats, _ := latestContainerStats(&rootInfo) - nodeStats := stats.NodeStats{ - NodeName: sb.node.Name, - CPU: rootStats.CPU, - Memory: rootStats.Memory, - Network: sb.containerInfoV2ToNetworkStats("node:"+sb.node.Name, &rootInfo), - Fs: &stats.FsStats{ - Time: metav1.NewTime(cStats.Timestamp), - AvailableBytes: &sb.rootFsInfo.Available, - CapacityBytes: &sb.rootFsInfo.Capacity, - UsedBytes: &sb.rootFsInfo.Usage, - InodesFree: sb.rootFsInfo.InodesFree, - Inodes: sb.rootFsInfo.Inodes, - InodesUsed: nodeFsInodesUsed, - }, + nodeStats := statsapi.NodeStats{ + NodeName: node.Name, + CPU: rootStats.CPU, + Memory: rootStats.Memory, + Network: networkStats, StartTime: rootStats.StartTime, - Runtime: &stats.RuntimeStats{ - ImageFs: &stats.FsStats{ - Time: metav1.NewTime(cStats.Timestamp), - AvailableBytes: &sb.imageFsInfo.Available, - CapacityBytes: &sb.imageFsInfo.Capacity, - UsedBytes: &sb.imageStats.TotalStorageBytes, - InodesFree: sb.imageFsInfo.InodesFree, - Inodes: sb.imageFsInfo.Inodes, - InodesUsed: imageFsInodesUsed, - }, - }, + Fs: rootFsStats, + Runtime: &statsapi.RuntimeStats{ImageFs: imageFsStats}, } systemContainers := map[string]string{ - stats.SystemContainerKubelet: sb.nodeConfig.KubeletCgroupsName, - stats.SystemContainerRuntime: sb.nodeConfig.RuntimeCgroupsName, - stats.SystemContainerMisc: sb.nodeConfig.SystemCgroupsName, + statsapi.SystemContainerKubelet: nodeConfig.KubeletCgroupsName, + statsapi.SystemContainerRuntime: nodeConfig.RuntimeCgroupsName, + statsapi.SystemContainerMisc: nodeConfig.SystemCgroupsName, } for sys, name := range systemContainers { - if info, ok := sb.infos[name]; ok { - sysCont := sb.containerInfoV2ToStats(sys, &info) - // System containers don't have a filesystem associated with them. - sysCont.Rootfs = nil - sysCont.Logs = nil - nodeStats.SystemContainers = append(nodeStats.SystemContainers, sysCont) + s, _, err := sp.provider.GetCgroupStats(name) + if err != nil { + glog.Errorf("Failed to get system container stats for %q: %v", name, err) + continue } + // System containers don't have a filesystem associated with them. + s.Logs, s.Rootfs = nil, nil + s.Name = sys + nodeStats.SystemContainers = append(nodeStats.SystemContainers, *s) } - summary := stats.Summary{ + summary := statsapi.Summary{ Node: nodeStats, - Pods: sb.buildSummaryPods(), + Pods: podStats, } return &summary, nil } - -// containerInfoV2FsStats populates the container fs stats -func (sb *summaryBuilder) containerInfoV2FsStats( - info *cadvisorapiv2.ContainerInfo, - cs *stats.ContainerStats) { - - lcs, found := latestContainerStats(info) - if !found { - return - } - - // The container logs live on the node rootfs device - cs.Logs = &stats.FsStats{ - Time: metav1.NewTime(lcs.Timestamp), - AvailableBytes: &sb.rootFsInfo.Available, - CapacityBytes: &sb.rootFsInfo.Capacity, - InodesFree: sb.rootFsInfo.InodesFree, - Inodes: sb.rootFsInfo.Inodes, - } - - if sb.rootFsInfo.Inodes != nil && sb.rootFsInfo.InodesFree != nil { - logsInodesUsed := *sb.rootFsInfo.Inodes - *sb.rootFsInfo.InodesFree - cs.Logs.InodesUsed = &logsInodesUsed - } - - // The container rootFs lives on the imageFs devices (which may not be the node root fs) - cs.Rootfs = &stats.FsStats{ - Time: metav1.NewTime(lcs.Timestamp), - AvailableBytes: &sb.imageFsInfo.Available, - CapacityBytes: &sb.imageFsInfo.Capacity, - InodesFree: sb.imageFsInfo.InodesFree, - Inodes: sb.imageFsInfo.Inodes, - } - cfs := lcs.Filesystem - - if cfs != nil { - if cfs.BaseUsageBytes != nil { - rootfsUsage := *cfs.BaseUsageBytes - cs.Rootfs.UsedBytes = &rootfsUsage - if cfs.TotalUsageBytes != nil { - logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes - cs.Logs.UsedBytes = &logsUsage - } - } - if cfs.InodeUsage != nil { - rootInodes := *cfs.InodeUsage - cs.Rootfs.InodesUsed = &rootInodes - } - } -} - -// latestContainerStats returns the latest container stats from cadvisor, or nil if none exist -func latestContainerStats(info *cadvisorapiv2.ContainerInfo) (*cadvisorapiv2.ContainerStats, bool) { - stats := info.Stats - if len(stats) < 1 { - return nil, false - } - latest := stats[len(stats)-1] - if latest == nil { - return nil, false - } - return latest, true -} - -// hasMemoryAndCPUInstUsage returns true if the specified container info has -// both non-zero CPU instantaneous usage and non-zero memory RSS usage, and -// false otherwise. -func hasMemoryAndCPUInstUsage(info *cadvisorapiv2.ContainerInfo) bool { - if !info.Spec.HasCpu || !info.Spec.HasMemory { - return false - } - cstat, found := latestContainerStats(info) - if !found { - return false - } - if cstat.CpuInst == nil { - return false - } - return cstat.CpuInst.Usage.Total != 0 && cstat.Memory.RSS != 0 -} - -// ByCreationTime implements sort.Interface for []containerInfoWithCgroup based -// on the cinfo.Spec.CreationTime field. -type ByCreationTime []containerInfoWithCgroup - -func (a ByCreationTime) Len() int { return len(a) } -func (a ByCreationTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByCreationTime) Less(i, j int) bool { - if a[i].cinfo.Spec.CreationTime.Equal(a[j].cinfo.Spec.CreationTime) { - // There shouldn't be two containers with the same name and/or the same - // creation time. However, to make the logic here robust, we break the - // tie by moving the one without CPU instantaneous or memory RSS usage - // to the beginning. - return hasMemoryAndCPUInstUsage(&a[j].cinfo) - } - return a[i].cinfo.Spec.CreationTime.Before(a[j].cinfo.Spec.CreationTime) -} - -// containerID is the identity of a container in a pod. -type containerID struct { - podRef stats.PodReference - containerName string -} - -// containerInfoWithCgroup contains the ContainerInfo and its cgroup name. -type containerInfoWithCgroup struct { - cinfo cadvisorapiv2.ContainerInfo - cgroup string -} - -// removeTerminatedContainerInfo returns the specified containerInfo but with -// the stats of the terminated containers removed. -// -// A ContainerInfo is considered to be of a terminated container if it has an -// older CreationTime and zero CPU instantaneous and memory RSS usage. -func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo { - cinfoMap := make(map[containerID][]containerInfoWithCgroup) - for key, cinfo := range containerInfo { - if !isPodManagedContainer(&cinfo) { - continue - } - cinfoID := containerID{ - podRef: buildPodRef(&cinfo), - containerName: types.GetContainerName(cinfo.Spec.Labels), - } - cinfoMap[cinfoID] = append(cinfoMap[cinfoID], containerInfoWithCgroup{ - cinfo: cinfo, - cgroup: key, - }) - } - result := make(map[string]cadvisorapiv2.ContainerInfo) - for _, refs := range cinfoMap { - if len(refs) == 1 { - result[refs[0].cgroup] = refs[0].cinfo - continue - } - sort.Sort(ByCreationTime(refs)) - i := 0 - for ; i < len(refs); i++ { - if hasMemoryAndCPUInstUsage(&refs[i].cinfo) { - // Stops removing when we first see an info with non-zero - // CPU/Memory usage. - break - } - } - for ; i < len(refs); i++ { - result[refs[i].cgroup] = refs[i].cinfo - } - } - return result -} - -// buildSummaryPods aggregates and returns the container stats in cinfos by the Pod managing the container. -// Containers not managed by a Pod are omitted. -func (sb *summaryBuilder) buildSummaryPods() []stats.PodStats { - // Map each container to a pod and update the PodStats with container data - podToStats := map[stats.PodReference]*stats.PodStats{} - infos := removeTerminatedContainerInfo(sb.infos) - for key, cinfo := range infos { - // on systemd using devicemapper each mount into the container has an associated cgroup. - // we ignore them to ensure we do not get duplicate entries in our summary. - // for details on .mount units: http://man7.org/linux/man-pages/man5/systemd.mount.5.html - if strings.HasSuffix(key, ".mount") { - continue - } - // Build the Pod key if this container is managed by a Pod - if !isPodManagedContainer(&cinfo) { - continue - } - ref := buildPodRef(&cinfo) - - // Lookup the PodStats for the pod using the PodRef. If none exists, initialize a new entry. - podStats, found := podToStats[ref] - if !found { - podStats = &stats.PodStats{PodRef: ref} - podToStats[ref] = podStats - } - - // Update the PodStats entry with the stats from the container by adding it to stats.Containers - containerName := types.GetContainerName(cinfo.Spec.Labels) - if containerName == leaky.PodInfraContainerName { - // Special case for infrastructure container which is hidden from the user and has network stats - podStats.Network = sb.containerInfoV2ToNetworkStats("pod:"+ref.Namespace+"_"+ref.Name, &cinfo) - podStats.StartTime = metav1.NewTime(cinfo.Spec.CreationTime) - } else { - podStats.Containers = append(podStats.Containers, sb.containerInfoV2ToStats(containerName, &cinfo)) - } - } - - // Add each PodStats to the result - result := make([]stats.PodStats, 0, len(podToStats)) - for _, podStats := range podToStats { - // Lookup the volume stats for each pod - podUID := kubetypes.UID(podStats.PodRef.UID) - if vstats, found := sb.fsResourceAnalyzer.GetPodVolumeStats(podUID); found { - podStats.VolumeStats = vstats.Volumes - } - result = append(result, *podStats) - } - return result -} - -// buildPodRef returns a PodReference that identifies the Pod managing cinfo -func buildPodRef(cinfo *cadvisorapiv2.ContainerInfo) stats.PodReference { - podName := types.GetPodName(cinfo.Spec.Labels) - podNamespace := types.GetPodNamespace(cinfo.Spec.Labels) - podUID := types.GetPodUID(cinfo.Spec.Labels) - return stats.PodReference{Name: podName, Namespace: podNamespace, UID: podUID} -} - -// isPodManagedContainer returns true if the cinfo container is managed by a Pod -func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool { - podName := types.GetPodName(cinfo.Spec.Labels) - podNamespace := types.GetPodNamespace(cinfo.Spec.Labels) - managed := podName != "" && podNamespace != "" - if !managed && podName != podNamespace { - glog.Warningf( - "Expect container to have either both podName (%s) and podNamespace (%s) labels, or neither.", - podName, podNamespace) - } - return managed -} - -func (sb *summaryBuilder) containerInfoV2ToStats( - name string, - info *cadvisorapiv2.ContainerInfo) stats.ContainerStats { - cStats := stats.ContainerStats{ - StartTime: metav1.NewTime(info.Spec.CreationTime), - Name: name, - } - cstat, found := latestContainerStats(info) - if !found { - return cStats - } - if info.Spec.HasCpu { - cpuStats := stats.CPUStats{ - Time: metav1.NewTime(cstat.Timestamp), - } - if cstat.CpuInst != nil { - cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total - } - if cstat.Cpu != nil { - cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total - } - cStats.CPU = &cpuStats - } - if info.Spec.HasMemory { - pageFaults := cstat.Memory.ContainerData.Pgfault - majorPageFaults := cstat.Memory.ContainerData.Pgmajfault - cStats.Memory = &stats.MemoryStats{ - Time: metav1.NewTime(cstat.Timestamp), - UsageBytes: &cstat.Memory.Usage, - WorkingSetBytes: &cstat.Memory.WorkingSet, - RSSBytes: &cstat.Memory.RSS, - PageFaults: &pageFaults, - MajorPageFaults: &majorPageFaults, - } - // availableBytes = memory limit (if known) - workingset - if !isMemoryUnlimited(info.Spec.Memory.Limit) { - availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet - cStats.Memory.AvailableBytes = &availableBytes - } - } - - sb.containerInfoV2FsStats(info, &cStats) - cStats.UserDefinedMetrics = sb.containerInfoV2ToUserDefinedMetrics(info) - return cStats -} - -// Size after which we consider memory to be "unlimited". This is not -// MaxInt64 due to rounding by the kernel. -// TODO: cadvisor should export this https://github.com/google/cadvisor/blob/master/metrics/prometheus.go#L596 -const maxMemorySize = uint64(1 << 62) - -func isMemoryUnlimited(v uint64) bool { - return v > maxMemorySize -} - -func (sb *summaryBuilder) containerInfoV2ToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *stats.NetworkStats { - if !info.Spec.HasNetwork { - return nil - } - cstat, found := latestContainerStats(info) - if !found { - return nil - } - for _, inter := range cstat.Network.Interfaces { - if inter.Name == network.DefaultInterfaceName { - return &stats.NetworkStats{ - Time: metav1.NewTime(cstat.Timestamp), - RxBytes: &inter.RxBytes, - RxErrors: &inter.RxErrors, - TxBytes: &inter.TxBytes, - TxErrors: &inter.TxErrors, - } - } - } - glog.V(4).Infof("Missing default interface %q for %s", network.DefaultInterfaceName, name) - return nil -} - -func (sb *summaryBuilder) containerInfoV2ToUserDefinedMetrics(info *cadvisorapiv2.ContainerInfo) []stats.UserDefinedMetric { - type specVal struct { - ref stats.UserDefinedMetricDescriptor - valType cadvisorapiv1.DataType - time time.Time - value float64 - } - udmMap := map[string]*specVal{} - for _, spec := range info.Spec.CustomMetrics { - udmMap[spec.Name] = &specVal{ - ref: stats.UserDefinedMetricDescriptor{ - Name: spec.Name, - Type: stats.UserDefinedMetricType(spec.Type), - Units: spec.Units, - }, - valType: spec.Format, - } - } - for _, stat := range info.Stats { - for name, values := range stat.CustomMetrics { - specVal, ok := udmMap[name] - if !ok { - glog.Warningf("spec for custom metric %q is missing from cAdvisor output. Spec: %+v, Metrics: %+v", name, info.Spec, stat.CustomMetrics) - continue - } - for _, value := range values { - // Pick the most recent value - if value.Timestamp.Before(specVal.time) { - continue - } - specVal.time = value.Timestamp - specVal.value = value.FloatValue - if specVal.valType == cadvisorapiv1.IntType { - specVal.value = float64(value.IntValue) - } - } - } - } - var udm []stats.UserDefinedMetric - for _, specVal := range udmMap { - udm = append(udm, stats.UserDefinedMetric{ - UserDefinedMetricDescriptor: specVal.ref, - Time: metav1.NewTime(specVal.time), - Value: specVal.value, - }) - } - return udm -} diff --git a/pkg/kubelet/server/stats/summary_test.go b/pkg/kubelet/server/stats/summary_test.go index 2253a10b876a7..22e38f7f67914 100644 --- a/pkg/kubelet/server/stats/summary_test.go +++ b/pkg/kubelet/server/stats/summary_test.go @@ -20,480 +20,123 @@ import ( "testing" "time" - "github.com/google/cadvisor/info/v1" - "github.com/google/cadvisor/info/v2" fuzz "github.com/google/gofuzz" "github.com/stretchr/testify/assert" - k8sv1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubestats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/kubelet/leaky" + statstest "k8s.io/kubernetes/pkg/kubelet/server/stats/testing" ) -const ( - // Offsets from seed value in generated container stats. - offsetCPUUsageCores = iota - offsetCPUUsageCoreSeconds - offsetMemPageFaults - offsetMemMajorPageFaults - offsetMemUsageBytes - offsetMemRSSBytes - offsetMemWorkingSetBytes - offsetNetRxBytes - offsetNetRxErrors - offsetNetTxBytes - offsetNetTxErrors -) - -var ( - timestamp = time.Now() - creationTime = timestamp.Add(-5 * time.Minute) -) - -func TestRemoveTerminatedContainerInfo(t *testing.T) { - const ( - seedPastPod0Infra = 1000 - seedPastPod0Container0 = 2000 - seedPod0Infra = 3000 - seedPod0Container0 = 4000 - ) - const ( - namespace = "test" - pName0 = "pod0" - cName00 = "c0" - ) - infos := map[string]v2.ContainerInfo{ - // ContainerInfo with past creation time and no CPU/memory usage for - // simulating uncleaned cgroups of already terminated containers, which - // should not be shown in the results. - "/pod0-i-terminated-1": summaryTerminatedContainerInfo(seedPastPod0Infra, pName0, namespace, leaky.PodInfraContainerName), - "/pod0-c0-terminated-1": summaryTerminatedContainerInfo(seedPastPod0Container0, pName0, namespace, cName00), - - // Same as above but uses the same creation time as the latest - // containers. They are terminated containers, so they should not be in - // the results. - "/pod0-i-terminated-2": summaryTerminatedContainerInfo(seedPod0Infra, pName0, namespace, leaky.PodInfraContainerName), - "/pod0-c0-terminated-2": summaryTerminatedContainerInfo(seedPod0Container0, pName0, namespace, cName00), - - // The latest containers, which should be in the results. - "/pod0-i": summaryTestContainerInfo(seedPod0Infra, pName0, namespace, leaky.PodInfraContainerName), - "/pod0-c0": summaryTestContainerInfo(seedPod0Container0, pName0, namespace, cName00), - - // Duplicated containers with non-zero CPU and memory usage. This case - // shouldn't happen unless something goes wrong, but we want to test - // that the metrics reporting logic works in this scenario. - "/pod0-i-duplicated": summaryTestContainerInfo(seedPod0Infra, pName0, namespace, leaky.PodInfraContainerName), - "/pod0-c0-duplicated": summaryTestContainerInfo(seedPod0Container0, pName0, namespace, cName00), - } - output := removeTerminatedContainerInfo(infos) - assert.Len(t, output, 4) - for _, c := range []string{"/pod0-i", "/pod0-c0", "/pod0-i-duplicated", "/pod0-c0-duplicated"} { - if _, found := output[c]; !found { - t.Errorf("%q is expected to be in the output\n", c) +func TestSummaryProvider(t *testing.T) { + var ( + podStats = []statsapi.PodStats{ + { + PodRef: statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"}, + StartTime: metav1.NewTime(time.Now()), + Containers: []statsapi.ContainerStats{*getContainerStats()}, + Network: getNetworkStats(), + VolumeStats: []statsapi.VolumeStats{*getVolumeStats()}, + }, } - } -} - -func TestBuildSummary(t *testing.T) { - node := k8sv1.Node{} - node.Name = "FooNode" - nodeConfig := cm.NodeConfig{ - RuntimeCgroupsName: "/docker-daemon", - SystemCgroupsName: "/system", - KubeletCgroupsName: "/kubelet", - } - const ( - namespace0 = "test0" - namespace2 = "test2" - ) - const ( - seedRoot = 0 - seedRuntime = 100 - seedKubelet = 200 - seedMisc = 300 - seedPod0Infra = 1000 - seedPod0Container0 = 2000 - seedPod0Container1 = 2001 - seedPod1Infra = 3000 - seedPod1Container = 4000 - seedPod2Infra = 5000 - seedPod2Container = 6000 - ) - const ( - pName0 = "pod0" - pName1 = "pod1" - pName2 = "pod0" // ensure pName2 conflicts with pName0, but is in a different namespace - ) - const ( - cName00 = "c0" - cName01 = "c1" - cName10 = "c0" // ensure cName10 conflicts with cName02, but is in a different pod - cName20 = "c1" // ensure cName20 conflicts with cName01, but is in a different pod + namespace - ) - const ( - rootfsCapacity = uint64(10000000) - rootfsAvailable = uint64(5000000) - rootfsInodesFree = uint64(1000) - rootfsInodes = uint64(2000) - imagefsCapacity = uint64(20000000) - imagefsAvailable = uint64(8000000) - imagefsInodesFree = uint64(2000) - imagefsInodes = uint64(4000) - ) - - prf0 := kubestats.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0} - prf1 := kubestats.PodReference{Name: pName1, Namespace: namespace0, UID: "UID" + pName1} - prf2 := kubestats.PodReference{Name: pName2, Namespace: namespace2, UID: "UID" + pName2} - infos := map[string]v2.ContainerInfo{ - "/": summaryTestContainerInfo(seedRoot, "", "", ""), - "/docker-daemon": summaryTestContainerInfo(seedRuntime, "", "", ""), - "/kubelet": summaryTestContainerInfo(seedKubelet, "", "", ""), - "/system": summaryTestContainerInfo(seedMisc, "", "", ""), - // Pod0 - Namespace0 - "/pod0-i": summaryTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName), - "/pod0-c0": summaryTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00), - "/pod0-c1": summaryTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01), - // Pod1 - Namespace0 - "/pod1-i": summaryTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName), - "/pod1-c0": summaryTestContainerInfo(seedPod1Container, pName1, namespace0, cName10), - // Pod2 - Namespace2 - "/pod2-i": summaryTestContainerInfo(seedPod2Infra, pName2, namespace2, leaky.PodInfraContainerName), - "/pod2-c0": summaryTestContainerInfo(seedPod2Container, pName2, namespace2, cName20), - } - - freeRootfsInodes := rootfsInodesFree - totalRootfsInodes := rootfsInodes - rootfs := v2.FsInfo{ - Capacity: rootfsCapacity, - Available: rootfsAvailable, - InodesFree: &freeRootfsInodes, - Inodes: &totalRootfsInodes, - } - freeImagefsInodes := imagefsInodesFree - totalImagefsInodes := imagefsInodes - imagefs := v2.FsInfo{ - Capacity: imagefsCapacity, - Available: imagefsAvailable, - InodesFree: &freeImagefsInodes, - Inodes: &totalImagefsInodes, - } - - // memory limit overrides for each container (used to test available bytes if a memory limit is known) - memoryLimitOverrides := map[string]uint64{ - "/": uint64(1 << 30), - "/pod2-c0": uint64(1 << 15), - } - for name, memoryLimitOverride := range memoryLimitOverrides { - info, found := infos[name] - if !found { - t.Errorf("No container defined with name %v", name) + imageFsStats = getFsStats() + rootFsStats = getFsStats() + node = &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "test-node"}} + nodeConfig = cm.NodeConfig{ + RuntimeCgroupsName: "/runtime", + SystemCgroupsName: "/system", + KubeletCgroupsName: "/kubelet", } - info.Spec.Memory.Limit = memoryLimitOverride - infos[name] = info - } - - sb := &summaryBuilder{ - newFsResourceAnalyzer(&MockStatsProvider{}, time.Minute*5), &node, nodeConfig, rootfs, imagefs, container.ImageStats{}, infos} - summary, err := sb.build() - - assert.NoError(t, err) - nodeStats := summary.Node - assert.Equal(t, "FooNode", nodeStats.NodeName) - assert.EqualValues(t, testTime(creationTime, seedRoot).Unix(), nodeStats.StartTime.Time.Unix()) - checkCPUStats(t, "Node", seedRoot, nodeStats.CPU) - checkMemoryStats(t, "Node", seedRoot, infos["/"], nodeStats.Memory) - checkNetworkStats(t, "Node", seedRoot, nodeStats.Network) - - systemSeeds := map[string]int{ - kubestats.SystemContainerRuntime: seedRuntime, - kubestats.SystemContainerKubelet: seedKubelet, - kubestats.SystemContainerMisc: seedMisc, - } - systemContainerToNodeCgroup := map[string]string{ - kubestats.SystemContainerRuntime: nodeConfig.RuntimeCgroupsName, - kubestats.SystemContainerKubelet: nodeConfig.KubeletCgroupsName, - kubestats.SystemContainerMisc: nodeConfig.SystemCgroupsName, - } - for _, sys := range nodeStats.SystemContainers { - name := sys.Name - info := infos[systemContainerToNodeCgroup[name]] - seed, found := systemSeeds[name] - if !found { - t.Errorf("Unknown SystemContainer: %q", name) + cgroupStatsMap = map[string]struct { + cs *statsapi.ContainerStats + ns *statsapi.NetworkStats + }{ + "/": {cs: getContainerStats(), ns: getNetworkStats()}, + "/runtime": {cs: getContainerStats(), ns: getNetworkStats()}, + "/system": {cs: getContainerStats(), ns: getNetworkStats()}, + "/kubelet": {cs: getContainerStats(), ns: getNetworkStats()}, } - assert.EqualValues(t, testTime(creationTime, seed).Unix(), sys.StartTime.Time.Unix(), name+".StartTime") - checkCPUStats(t, name, seed, sys.CPU) - checkMemoryStats(t, name, seed, info, sys.Memory) - assert.Nil(t, sys.Logs, name+".Logs") - assert.Nil(t, sys.Rootfs, name+".Rootfs") - } - - assert.Equal(t, 3, len(summary.Pods)) - indexPods := make(map[kubestats.PodReference]kubestats.PodStats, len(summary.Pods)) - for _, pod := range summary.Pods { - indexPods[pod.PodRef] = pod - } - - // Validate Pod0 Results - ps, found := indexPods[prf0] - assert.True(t, found) - assert.Len(t, ps.Containers, 2) - indexCon := make(map[string]kubestats.ContainerStats, len(ps.Containers)) - for _, con := range ps.Containers { - indexCon[con.Name] = con - } - con := indexCon[cName00] - assert.EqualValues(t, testTime(creationTime, seedPod0Container0).Unix(), con.StartTime.Time.Unix()) - checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU) - checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory) - - con = indexCon[cName01] - assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix()) - checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU) - checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory) - - assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix()) - checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network) - - // Validate Pod1 Results - ps, found = indexPods[prf1] - assert.True(t, found) - assert.Len(t, ps.Containers, 1) - con = ps.Containers[0] - assert.Equal(t, cName10, con.Name) - checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU) - checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory) - checkNetworkStats(t, "Pod1", seedPod1Infra, ps.Network) - - // Validate Pod2 Results - ps, found = indexPods[prf2] - assert.True(t, found) - assert.Len(t, ps.Containers, 1) - con = ps.Containers[0] - assert.Equal(t, cName20, con.Name) - checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU) - checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory) - checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network) -} - -func generateCustomMetricSpec() []v1.MetricSpec { - f := fuzz.New().NilChance(0).Funcs( - func(e *v1.MetricSpec, c fuzz.Continue) { - c.Fuzz(&e.Name) - switch c.Intn(3) { - case 0: - e.Type = v1.MetricGauge - case 1: - e.Type = v1.MetricCumulative - case 2: - e.Type = v1.MetricDelta - } - switch c.Intn(2) { - case 0: - e.Format = v1.IntType - case 1: - e.Format = v1.FloatType - } - c.Fuzz(&e.Units) - }) - var ret []v1.MetricSpec - f.Fuzz(&ret) - return ret -} - -func generateCustomMetrics(spec []v1.MetricSpec) map[string][]v1.MetricVal { - ret := map[string][]v1.MetricVal{} - for _, metricSpec := range spec { - f := fuzz.New().NilChance(0).Funcs( - func(e *v1.MetricVal, c fuzz.Continue) { - switch metricSpec.Format { - case v1.IntType: - c.Fuzz(&e.IntValue) - case v1.FloatType: - c.Fuzz(&e.FloatValue) - } - }) - - var metrics []v1.MetricVal - f.Fuzz(&metrics) - ret[metricSpec.Name] = metrics - } - return ret -} - -func summaryTerminatedContainerInfo(seed int, podName string, podNamespace string, containerName string) v2.ContainerInfo { - cinfo := summaryTestContainerInfo(seed, podName, podNamespace, containerName) - cinfo.Stats[0].Memory.RSS = 0 - cinfo.Stats[0].CpuInst.Usage.Total = 0 - return cinfo -} + ) -func summaryTestContainerInfo(seed int, podName string, podNamespace string, containerName string) v2.ContainerInfo { - labels := map[string]string{} - if podName != "" { - labels = map[string]string{ - "io.kubernetes.pod.name": podName, - "io.kubernetes.pod.uid": "UID" + podName, - "io.kubernetes.pod.namespace": podNamespace, - "io.kubernetes.container.name": containerName, - } - } - // by default, kernel will set memory.limit_in_bytes to 1 << 63 if not bounded - unlimitedMemory := uint64(1 << 63) - spec := v2.ContainerSpec{ - CreationTime: testTime(creationTime, seed), - HasCpu: true, - HasMemory: true, - HasNetwork: true, - Labels: labels, - Memory: v2.MemorySpec{ - Limit: unlimitedMemory, + assert := assert.New(t) + + mockStatsProvider := new(statstest.StatsProvider) + mockStatsProvider. + On("GetNode").Return(node, nil). + On("GetNodeConfig").Return(nodeConfig). + On("ListPodStats").Return(podStats, nil). + On("ImageFsStats").Return(imageFsStats, nil). + On("RootFsStats").Return(rootFsStats, nil). + On("GetCgroupStats", "/").Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil). + On("GetCgroupStats", "/runtime").Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil). + On("GetCgroupStats", "/system").Return(cgroupStatsMap["/system"].cs, cgroupStatsMap["/system"].ns, nil). + On("GetCgroupStats", "/kubelet").Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil) + + provider := NewSummaryProvider(mockStatsProvider) + summary, err := provider.Get() + assert.NoError(err) + + assert.Equal(summary.Node.NodeName, "test-node") + assert.Equal(summary.Node.StartTime, cgroupStatsMap["/"].cs.StartTime) + assert.Equal(summary.Node.CPU, cgroupStatsMap["/"].cs.CPU) + assert.Equal(summary.Node.Memory, cgroupStatsMap["/"].cs.Memory) + assert.Equal(summary.Node.Network, cgroupStatsMap["/"].ns) + assert.Equal(summary.Node.Fs, rootFsStats) + assert.Equal(summary.Node.Runtime, &statsapi.RuntimeStats{ImageFs: imageFsStats}) + + assert.Equal(len(summary.Node.SystemContainers), 3) + assert.Contains(summary.Node.SystemContainers, + statsapi.ContainerStats{ + Name: "kubelet", + StartTime: cgroupStatsMap["/kubelet"].cs.StartTime, + CPU: cgroupStatsMap["/kubelet"].cs.CPU, + Memory: cgroupStatsMap["/kubelet"].cs.Memory, + UserDefinedMetrics: cgroupStatsMap["/kubelet"].cs.UserDefinedMetrics, }, - CustomMetrics: generateCustomMetricSpec(), - } - - stats := v2.ContainerStats{ - Timestamp: testTime(timestamp, seed), - Cpu: &v1.CpuStats{}, - CpuInst: &v2.CpuInstStats{}, - Memory: &v1.MemoryStats{ - Usage: uint64(seed + offsetMemUsageBytes), - WorkingSet: uint64(seed + offsetMemWorkingSetBytes), - RSS: uint64(seed + offsetMemRSSBytes), - ContainerData: v1.MemoryStatsMemoryData{ - Pgfault: uint64(seed + offsetMemPageFaults), - Pgmajfault: uint64(seed + offsetMemMajorPageFaults), - }, + statsapi.ContainerStats{ + Name: "system", + StartTime: cgroupStatsMap["/system"].cs.StartTime, + CPU: cgroupStatsMap["/system"].cs.CPU, + Memory: cgroupStatsMap["/system"].cs.Memory, + UserDefinedMetrics: cgroupStatsMap["/system"].cs.UserDefinedMetrics, }, - Network: &v2.NetworkStats{ - Interfaces: []v1.InterfaceStats{{ - Name: "eth0", - RxBytes: uint64(seed + offsetNetRxBytes), - RxErrors: uint64(seed + offsetNetRxErrors), - TxBytes: uint64(seed + offsetNetTxBytes), - TxErrors: uint64(seed + offsetNetTxErrors), - }, { - Name: "cbr0", - RxBytes: 100, - RxErrors: 100, - TxBytes: 100, - TxErrors: 100, - }}, + statsapi.ContainerStats{ + Name: "runtime", + StartTime: cgroupStatsMap["/runtime"].cs.StartTime, + CPU: cgroupStatsMap["/runtime"].cs.CPU, + Memory: cgroupStatsMap["/runtime"].cs.Memory, + UserDefinedMetrics: cgroupStatsMap["/runtime"].cs.UserDefinedMetrics, }, - CustomMetrics: generateCustomMetrics(spec.CustomMetrics), - } - stats.Cpu.Usage.Total = uint64(seed + offsetCPUUsageCoreSeconds) - stats.CpuInst.Usage.Total = uint64(seed + offsetCPUUsageCores) - return v2.ContainerInfo{ - Spec: spec, - Stats: []*v2.ContainerStats{&stats}, - } -} - -func testTime(base time.Time, seed int) time.Time { - return base.Add(time.Duration(seed) * time.Second) + ) + assert.Equal(summary.Pods, podStats) } -func checkNetworkStats(t *testing.T, label string, seed int, stats *kubestats.NetworkStats) { - assert.NotNil(t, stats) - assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Net.Time") - assert.EqualValues(t, seed+offsetNetRxBytes, *stats.RxBytes, label+".Net.RxBytes") - assert.EqualValues(t, seed+offsetNetRxErrors, *stats.RxErrors, label+".Net.RxErrors") - assert.EqualValues(t, seed+offsetNetTxBytes, *stats.TxBytes, label+".Net.TxBytes") - assert.EqualValues(t, seed+offsetNetTxErrors, *stats.TxErrors, label+".Net.TxErrors") +func getFsStats() *statsapi.FsStats { + f := fuzz.New().NilChance(0) + v := &statsapi.FsStats{} + f.Fuzz(v) + return v } -func checkCPUStats(t *testing.T, label string, seed int, stats *kubestats.CPUStats) { - assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".CPU.Time") - assert.EqualValues(t, seed+offsetCPUUsageCores, *stats.UsageNanoCores, label+".CPU.UsageCores") - assert.EqualValues(t, seed+offsetCPUUsageCoreSeconds, *stats.UsageCoreNanoSeconds, label+".CPU.UsageCoreSeconds") +func getContainerStats() *statsapi.ContainerStats { + f := fuzz.New().NilChance(0) + v := &statsapi.ContainerStats{} + f.Fuzz(v) + return v } -func checkMemoryStats(t *testing.T, label string, seed int, info v2.ContainerInfo, stats *kubestats.MemoryStats) { - assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Mem.Time") - assert.EqualValues(t, seed+offsetMemUsageBytes, *stats.UsageBytes, label+".Mem.UsageBytes") - assert.EqualValues(t, seed+offsetMemWorkingSetBytes, *stats.WorkingSetBytes, label+".Mem.WorkingSetBytes") - assert.EqualValues(t, seed+offsetMemRSSBytes, *stats.RSSBytes, label+".Mem.RSSBytes") - assert.EqualValues(t, seed+offsetMemPageFaults, *stats.PageFaults, label+".Mem.PageFaults") - assert.EqualValues(t, seed+offsetMemMajorPageFaults, *stats.MajorPageFaults, label+".Mem.MajorPageFaults") - if !info.Spec.HasMemory || isMemoryUnlimited(info.Spec.Memory.Limit) { - assert.Nil(t, stats.AvailableBytes, label+".Mem.AvailableBytes") - } else { - expected := info.Spec.Memory.Limit - *stats.WorkingSetBytes - assert.EqualValues(t, expected, *stats.AvailableBytes, label+".Mem.AvailableBytes") - } +func getVolumeStats() *statsapi.VolumeStats { + f := fuzz.New().NilChance(0) + v := &statsapi.VolumeStats{} + f.Fuzz(v) + return v } -func TestCustomMetrics(t *testing.T) { - spec := []v1.MetricSpec{ - { - Name: "qos", - Type: v1.MetricGauge, - Format: v1.IntType, - Units: "per second", - }, - { - Name: "cpuLoad", - Type: v1.MetricCumulative, - Format: v1.FloatType, - Units: "count", - }, - } - timestamp1 := time.Now() - timestamp2 := time.Now().Add(time.Minute) - metrics := map[string][]v1.MetricVal{ - "qos": { - { - Timestamp: timestamp1, - IntValue: 10, - }, - { - Timestamp: timestamp2, - IntValue: 100, - }, - }, - "cpuLoad": { - { - Timestamp: timestamp1, - FloatValue: 1.2, - }, - { - Timestamp: timestamp2, - FloatValue: 2.1, - }, - }, - } - cInfo := v2.ContainerInfo{ - Spec: v2.ContainerSpec{ - CustomMetrics: spec, - }, - Stats: []*v2.ContainerStats{ - { - CustomMetrics: metrics, - }, - }, - } - sb := &summaryBuilder{} - assert.Contains(t, sb.containerInfoV2ToUserDefinedMetrics(&cInfo), - kubestats.UserDefinedMetric{ - UserDefinedMetricDescriptor: kubestats.UserDefinedMetricDescriptor{ - Name: "qos", - Type: kubestats.MetricGauge, - Units: "per second", - }, - Time: metav1.NewTime(timestamp2), - Value: 100, - }, - kubestats.UserDefinedMetric{ - UserDefinedMetricDescriptor: kubestats.UserDefinedMetricDescriptor{ - Name: "cpuLoad", - Type: kubestats.MetricCumulative, - Units: "count", - }, - Time: metav1.NewTime(timestamp2), - Value: 2.1, - }) +func getNetworkStats() *statsapi.NetworkStats { + f := fuzz.New().NilChance(0) + v := &statsapi.NetworkStats{} + f.Fuzz(v) + return v } diff --git a/pkg/kubelet/server/stats/testing/BUILD b/pkg/kubelet/server/stats/testing/BUILD new file mode 100644 index 0000000000000..23c1db4915243 --- /dev/null +++ b/pkg/kubelet/server/stats/testing/BUILD @@ -0,0 +1,30 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["mock_stats_provider.go"], + visibility = ["//visibility:public"], + deps = [ + "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", + "//pkg/kubelet/cm:go_default_library", + "//pkg/volume:go_default_library", + "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/github.com/stretchr/testify/mock:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubelet/server/stats/testing/mock_stats_provider.go b/pkg/kubelet/server/stats/testing/mock_stats_provider.go new file mode 100644 index 0000000000000..befa19b0bf756 --- /dev/null +++ b/pkg/kubelet/server/stats/testing/mock_stats_provider.go @@ -0,0 +1,280 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import cm "k8s.io/kubernetes/pkg/kubelet/cm" +import corev1 "k8s.io/api/core/v1" +import mock "github.com/stretchr/testify/mock" + +import types "k8s.io/apimachinery/pkg/types" +import v1 "github.com/google/cadvisor/info/v1" +import v1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" +import volume "k8s.io/kubernetes/pkg/volume" + +// DO NOT EDIT +// GENERATED BY mockery + +// StatsProvider is an autogenerated mock type for the StatsProvider type +type StatsProvider struct { + mock.Mock +} + +// GetCgroupStats provides a mock function with given fields: cgroupName +func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerStats, *v1alpha1.NetworkStats, error) { + ret := _m.Called(cgroupName) + + var r0 *v1alpha1.ContainerStats + if rf, ok := ret.Get(0).(func(string) *v1alpha1.ContainerStats); ok { + r0 = rf(cgroupName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1alpha1.ContainerStats) + } + } + + var r1 *v1alpha1.NetworkStats + if rf, ok := ret.Get(1).(func(string) *v1alpha1.NetworkStats); ok { + r1 = rf(cgroupName) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*v1alpha1.NetworkStats) + } + } + + var r2 error + if rf, ok := ret.Get(2).(func(string) error); ok { + r2 = rf(cgroupName) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetContainerInfo provides a mock function with given fields: podFullName, uid, containerName, req +func (_m *StatsProvider) GetContainerInfo(podFullName string, uid types.UID, containerName string, req *v1.ContainerInfoRequest) (*v1.ContainerInfo, error) { + ret := _m.Called(podFullName, uid, containerName, req) + + var r0 *v1.ContainerInfo + if rf, ok := ret.Get(0).(func(string, types.UID, string, *v1.ContainerInfoRequest) *v1.ContainerInfo); ok { + r0 = rf(podFullName, uid, containerName, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.ContainerInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, types.UID, string, *v1.ContainerInfoRequest) error); ok { + r1 = rf(podFullName, uid, containerName, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetNode provides a mock function with given fields: +func (_m *StatsProvider) GetNode() (*corev1.Node, error) { + ret := _m.Called() + + var r0 *corev1.Node + if rf, ok := ret.Get(0).(func() *corev1.Node); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*corev1.Node) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetNodeConfig provides a mock function with given fields: +func (_m *StatsProvider) GetNodeConfig() cm.NodeConfig { + ret := _m.Called() + + var r0 cm.NodeConfig + if rf, ok := ret.Get(0).(func() cm.NodeConfig); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(cm.NodeConfig) + } + + return r0 +} + +// GetPodByName provides a mock function with given fields: namespace, name +func (_m *StatsProvider) GetPodByName(namespace string, name string) (*corev1.Pod, bool) { + ret := _m.Called(namespace, name) + + var r0 *corev1.Pod + if rf, ok := ret.Get(0).(func(string, string) *corev1.Pod); ok { + r0 = rf(namespace, name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*corev1.Pod) + } + } + + var r1 bool + if rf, ok := ret.Get(1).(func(string, string) bool); ok { + r1 = rf(namespace, name) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// GetPods provides a mock function with given fields: +func (_m *StatsProvider) GetPods() []*corev1.Pod { + ret := _m.Called() + + var r0 []*corev1.Pod + if rf, ok := ret.Get(0).(func() []*corev1.Pod); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*corev1.Pod) + } + } + + return r0 +} + +// GetRawContainerInfo provides a mock function with given fields: containerName, req, subcontainers +func (_m *StatsProvider) GetRawContainerInfo(containerName string, req *v1.ContainerInfoRequest, subcontainers bool) (map[string]*v1.ContainerInfo, error) { + ret := _m.Called(containerName, req, subcontainers) + + var r0 map[string]*v1.ContainerInfo + if rf, ok := ret.Get(0).(func(string, *v1.ContainerInfoRequest, bool) map[string]*v1.ContainerInfo); ok { + r0 = rf(containerName, req, subcontainers) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]*v1.ContainerInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, *v1.ContainerInfoRequest, bool) error); ok { + r1 = rf(containerName, req, subcontainers) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ImageFsStats provides a mock function with given fields: +func (_m *StatsProvider) ImageFsStats() (*v1alpha1.FsStats, error) { + ret := _m.Called() + + var r0 *v1alpha1.FsStats + if rf, ok := ret.Get(0).(func() *v1alpha1.FsStats); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1alpha1.FsStats) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListPodStats provides a mock function with given fields: +func (_m *StatsProvider) ListPodStats() ([]v1alpha1.PodStats, error) { + ret := _m.Called() + + var r0 []v1alpha1.PodStats + if rf, ok := ret.Get(0).(func() []v1alpha1.PodStats); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]v1alpha1.PodStats) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListVolumesForPod provides a mock function with given fields: podUID +func (_m *StatsProvider) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) { + ret := _m.Called(podUID) + + var r0 map[string]volume.Volume + if rf, ok := ret.Get(0).(func(types.UID) map[string]volume.Volume); ok { + r0 = rf(podUID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]volume.Volume) + } + } + + var r1 bool + if rf, ok := ret.Get(1).(func(types.UID) bool); ok { + r1 = rf(podUID) + } else { + r1 = ret.Get(1).(bool) + } + + return r0, r1 +} + +// RootFsStats provides a mock function with given fields: +func (_m *StatsProvider) RootFsStats() (*v1alpha1.FsStats, error) { + ret := _m.Called() + + var r0 *v1alpha1.FsStats + if rf, ok := ret.Get(0).(func() *v1alpha1.FsStats); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1alpha1.FsStats) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/pkg/kubelet/stats/BUILD b/pkg/kubelet/stats/BUILD new file mode 100644 index 0000000000000..e96170a6bd969 --- /dev/null +++ b/pkg/kubelet/stats/BUILD @@ -0,0 +1,69 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "cadvisor_stats_provider.go", + "cri_stats_provider.go", + "helper.go", + "stats_provider.go", + ], + visibility = ["//visibility:public"], + deps = [ + "//pkg/kubelet/apis/cri:go_default_library", + "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", + "//pkg/kubelet/cadvisor:go_default_library", + "//pkg/kubelet/container:go_default_library", + "//pkg/kubelet/leaky:go_default_library", + "//pkg/kubelet/network:go_default_library", + "//pkg/kubelet/pod:go_default_library", + "//pkg/kubelet/server/stats:go_default_library", + "//pkg/kubelet/types:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/github.com/google/cadvisor/info/v2:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = [ + "cadvisor_stats_provider_test.go", + "helper_test.go", + "stats_provider_test.go", + ], + library = ":go_default_library", + deps = [ + "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", + "//pkg/kubelet/cadvisor/testing:go_default_library", + "//pkg/kubelet/container:go_default_library", + "//pkg/kubelet/container/testing:go_default_library", + "//pkg/kubelet/leaky:go_default_library", + "//pkg/kubelet/pod/testing:go_default_library", + "//pkg/kubelet/server/stats:go_default_library", + "//pkg/kubelet/types:go_default_library", + "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + "//vendor/github.com/google/cadvisor/info/v2:go_default_library", + "//vendor/github.com/google/gofuzz:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + "//vendor/github.com/stretchr/testify/require:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + ], +) diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go new file mode 100644 index 0000000000000..39a6b8aac695b --- /dev/null +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -0,0 +1,288 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "fmt" + "sort" + "strings" + + "github.com/golang/glog" + cadvisorapiv2 "github.com/google/cadvisor/info/v2" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/kubelet/cadvisor" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/kubelet/leaky" + "k8s.io/kubernetes/pkg/kubelet/server/stats" + kubetypes "k8s.io/kubernetes/pkg/kubelet/types" +) + +// cadvisorStatsProvider implements the containerStatsProvider interface by +// getting the container stats from cAdvisor. This is needed by docker and rkt +// integrations since they do not provide stats from CRI. +type cadvisorStatsProvider struct { + // cadvisor is used to get the stats of the cgroup for the containers that + // are managed by pods. + cadvisor cadvisor.Interface + // resourceAnalyzer is used to get the volume stats of the pods. + resourceAnalyzer stats.ResourceAnalyzer + // imageService is used to get the stats of the image filesystem. + imageService kubecontainer.ImageService +} + +// newCadvisorStatsProvider returns a containerStatsProvider that provides +// container stats from cAdvisor. +func newCadvisorStatsProvider( + cadvisor cadvisor.Interface, + resourceAnalyzer stats.ResourceAnalyzer, + imageService kubecontainer.ImageService, +) containerStatsProvider { + return &cadvisorStatsProvider{ + cadvisor: cadvisor, + resourceAnalyzer: resourceAnalyzer, + imageService: imageService, + } +} + +// ListPodStats returns the stats of all the pod-managed containers. +func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { + // Gets node root filesystem information and image filesystem stats, which + // will be used to populate the available and capacity bytes/inodes in + // container stats. + rootFsInfo, err := p.cadvisor.RootFsInfo() + if err != nil { + return nil, fmt.Errorf("failed to get rootFs info: %v", err) + } + imageFsInfo, err := p.cadvisor.ImagesFsInfo() + if err != nil { + return nil, fmt.Errorf("failed to get imageFs info: %v", err) + } + + infos, err := p.cadvisor.ContainerInfoV2("/", cadvisorapiv2.RequestOptions{ + IdType: cadvisorapiv2.TypeName, + Count: 2, // 2 samples are needed to compute "instantaneous" CPU + Recursive: true, + }) + if err != nil { + if _, ok := infos["/"]; ok { + // If the failure is partial, log it and return a best-effort + // response. + glog.Errorf("Partial failure issuing cadvisor.ContainerInfoV2: %v", err) + } else { + return nil, fmt.Errorf("failed to get root cgroup stats: %v", err) + } + } + + infos = removeTerminatedContainerInfo(infos) + + // Map each container to a pod and update the PodStats with container data. + podToStats := map[statsapi.PodReference]*statsapi.PodStats{} + for key, cinfo := range infos { + // On systemd using devicemapper each mount into the container has an + // associated cgroup. We ignore them to ensure we do not get duplicate + // entries in our summary. For details on .mount units: + // http://man7.org/linux/man-pages/man5/systemd.mount.5.html + if strings.HasSuffix(key, ".mount") { + continue + } + // Build the Pod key if this container is managed by a Pod + if !isPodManagedContainer(&cinfo) { + continue + } + ref := buildPodRef(&cinfo) + + // Lookup the PodStats for the pod using the PodRef. If none exists, + // initialize a new entry. + podStats, found := podToStats[ref] + if !found { + podStats = &statsapi.PodStats{PodRef: ref} + podToStats[ref] = podStats + } + + // Update the PodStats entry with the stats from the container by + // adding it to podStats.Containers. + containerName := kubetypes.GetContainerName(cinfo.Spec.Labels) + if containerName == leaky.PodInfraContainerName { + // Special case for infrastructure container which is hidden from + // the user and has network stats. + podStats.Network = cadvisorInfoToNetworkStats("pod:"+ref.Namespace+"_"+ref.Name, &cinfo) + podStats.StartTime = metav1.NewTime(cinfo.Spec.CreationTime) + } else { + podStats.Containers = append(podStats.Containers, *cadvisorInfoToContainerStats(containerName, &cinfo, &rootFsInfo, &imageFsInfo)) + } + } + + // Add each PodStats to the result. + result := make([]statsapi.PodStats, 0, len(podToStats)) + for _, podStats := range podToStats { + // Lookup the volume stats for each pod. + podUID := types.UID(podStats.PodRef.UID) + if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found { + podStats.VolumeStats = vstats.Volumes + } + result = append(result, *podStats) + } + + return result, nil +} + +// ImageFsStats returns the stats of the filesystem for storing images. +func (p *cadvisorStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { + imageFsInfo, err := p.cadvisor.ImagesFsInfo() + if err != nil { + return nil, fmt.Errorf("failed to get imageFs info: %v", err) + } + imageStats, err := p.imageService.ImageStats() + if err != nil || imageStats == nil { + return nil, fmt.Errorf("failed to get image stats: %v", err) + } + + var imageFsInodesUsed *uint64 + if imageFsInfo.Inodes != nil && imageFsInfo.InodesFree != nil { + imageFsIU := *imageFsInfo.Inodes - *imageFsInfo.InodesFree + imageFsInodesUsed = &imageFsIU + } + + // Get the root container stats's timestamp, which will be used as the + // imageFs stats timestamp. + rootStats, err := getCgroupStats(p.cadvisor, "/") + if err != nil { + return nil, fmt.Errorf("failed to get root container stats: %v", err) + } + + return &statsapi.FsStats{ + Time: metav1.NewTime(rootStats.Timestamp), + AvailableBytes: &imageFsInfo.Available, + CapacityBytes: &imageFsInfo.Capacity, + UsedBytes: &imageStats.TotalStorageBytes, + InodesFree: imageFsInfo.InodesFree, + Inodes: imageFsInfo.Inodes, + InodesUsed: imageFsInodesUsed, + }, nil +} + +// buildPodRef returns a PodReference that identifies the Pod managing cinfo +func buildPodRef(cinfo *cadvisorapiv2.ContainerInfo) statsapi.PodReference { + podName := kubetypes.GetPodName(cinfo.Spec.Labels) + podNamespace := kubetypes.GetPodNamespace(cinfo.Spec.Labels) + podUID := kubetypes.GetPodUID(cinfo.Spec.Labels) + return statsapi.PodReference{Name: podName, Namespace: podNamespace, UID: podUID} +} + +// isPodManagedContainer returns true if the cinfo container is managed by a Pod +func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool { + podName := kubetypes.GetPodName(cinfo.Spec.Labels) + podNamespace := kubetypes.GetPodNamespace(cinfo.Spec.Labels) + managed := podName != "" && podNamespace != "" + if !managed && podName != podNamespace { + glog.Warningf( + "Expect container to have either both podName (%s) and podNamespace (%s) labels, or neither.", + podName, podNamespace) + } + return managed +} + +// removeTerminatedContainerInfo returns the specified containerInfo but with +// the stats of the terminated containers removed. +// +// A ContainerInfo is considered to be of a terminated container if it has an +// older CreationTime and zero CPU instantaneous and memory RSS usage. +func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo { + cinfoMap := make(map[containerID][]containerInfoWithCgroup) + for key, cinfo := range containerInfo { + if !isPodManagedContainer(&cinfo) { + continue + } + cinfoID := containerID{ + podRef: buildPodRef(&cinfo), + containerName: kubetypes.GetContainerName(cinfo.Spec.Labels), + } + cinfoMap[cinfoID] = append(cinfoMap[cinfoID], containerInfoWithCgroup{ + cinfo: cinfo, + cgroup: key, + }) + } + result := make(map[string]cadvisorapiv2.ContainerInfo) + for _, refs := range cinfoMap { + if len(refs) == 1 { + result[refs[0].cgroup] = refs[0].cinfo + continue + } + sort.Sort(ByCreationTime(refs)) + i := 0 + for ; i < len(refs); i++ { + if hasMemoryAndCPUInstUsage(&refs[i].cinfo) { + // Stops removing when we first see an info with non-zero + // CPU/Memory usage. + break + } + } + for ; i < len(refs); i++ { + result[refs[i].cgroup] = refs[i].cinfo + } + } + return result +} + +// ByCreationTime implements sort.Interface for []containerInfoWithCgroup based +// on the cinfo.Spec.CreationTime field. +type ByCreationTime []containerInfoWithCgroup + +func (a ByCreationTime) Len() int { return len(a) } +func (a ByCreationTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByCreationTime) Less(i, j int) bool { + if a[i].cinfo.Spec.CreationTime.Equal(a[j].cinfo.Spec.CreationTime) { + // There shouldn't be two containers with the same name and/or the same + // creation time. However, to make the logic here robust, we break the + // tie by moving the one without CPU instantaneous or memory RSS usage + // to the beginning. + return hasMemoryAndCPUInstUsage(&a[j].cinfo) + } + return a[i].cinfo.Spec.CreationTime.Before(a[j].cinfo.Spec.CreationTime) +} + +// containerID is the identity of a container in a pod. +type containerID struct { + podRef statsapi.PodReference + containerName string +} + +// containerInfoWithCgroup contains the ContainerInfo and its cgroup name. +type containerInfoWithCgroup struct { + cinfo cadvisorapiv2.ContainerInfo + cgroup string +} + +// hasMemoryAndCPUInstUsage returns true if the specified container info has +// both non-zero CPU instantaneous usage and non-zero memory RSS usage, and +// false otherwise. +func hasMemoryAndCPUInstUsage(info *cadvisorapiv2.ContainerInfo) bool { + if !info.Spec.HasCpu || !info.Spec.HasMemory { + return false + } + cstat, found := latestContainerStats(info) + if !found { + return false + } + if cstat.CpuInst == nil { + return false + } + return cstat.CpuInst.Usage.Total != 0 && cstat.Memory.RSS != 0 +} diff --git a/pkg/kubelet/stats/cadvisor_stats_provider_test.go b/pkg/kubelet/stats/cadvisor_stats_provider_test.go new file mode 100644 index 0000000000000..06b60a64f5dfd --- /dev/null +++ b/pkg/kubelet/stats/cadvisor_stats_provider_test.go @@ -0,0 +1,271 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "testing" + + cadvisorapiv2 "github.com/google/cadvisor/info/v2" + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" + "k8s.io/kubernetes/pkg/kubelet/leaky" +) + +func TestRemoveTerminatedContainerInfo(t *testing.T) { + const ( + seedPastPod0Infra = 1000 + seedPastPod0Container0 = 2000 + seedPod0Infra = 3000 + seedPod0Container0 = 4000 + ) + const ( + namespace = "test" + pName0 = "pod0" + cName00 = "c0" + ) + infos := map[string]cadvisorapiv2.ContainerInfo{ + // ContainerInfo with past creation time and no CPU/memory usage for + // simulating uncleaned cgroups of already terminated containers, which + // should not be shown in the results. + "/pod0-i-terminated-1": getTerminatedContainerInfo(seedPastPod0Infra, pName0, namespace, leaky.PodInfraContainerName), + "/pod0-c0-terminated-1": getTerminatedContainerInfo(seedPastPod0Container0, pName0, namespace, cName00), + + // Same as above but uses the same creation time as the latest + // containers. They are terminated containers, so they should not be in + // the results. + "/pod0-i-terminated-2": getTerminatedContainerInfo(seedPod0Infra, pName0, namespace, leaky.PodInfraContainerName), + "/pod0-c0-terminated-2": getTerminatedContainerInfo(seedPod0Container0, pName0, namespace, cName00), + + // The latest containers, which should be in the results. + "/pod0-i": getTestContainerInfo(seedPod0Infra, pName0, namespace, leaky.PodInfraContainerName), + "/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace, cName00), + + // Duplicated containers with non-zero CPU and memory usage. This case + // shouldn't happen unless something goes wrong, but we want to test + // that the metrics reporting logic works in this scenario. + "/pod0-i-duplicated": getTestContainerInfo(seedPod0Infra, pName0, namespace, leaky.PodInfraContainerName), + "/pod0-c0-duplicated": getTestContainerInfo(seedPod0Container0, pName0, namespace, cName00), + } + output := removeTerminatedContainerInfo(infos) + assert.Len(t, output, 4) + for _, c := range []string{"/pod0-i", "/pod0-c0", "/pod0-i-duplicated", "/pod0-c0-duplicated"} { + if _, found := output[c]; !found { + t.Errorf("%q is expected to be in the output\n", c) + } + } +} + +func TestListPodStats(t *testing.T) { + const ( + namespace0 = "test0" + namespace2 = "test2" + ) + const ( + seedRoot = 0 + seedRuntime = 100 + seedKubelet = 200 + seedMisc = 300 + seedPod0Infra = 1000 + seedPod0Container0 = 2000 + seedPod0Container1 = 2001 + seedPod1Infra = 3000 + seedPod1Container = 4000 + seedPod2Infra = 5000 + seedPod2Container = 6000 + ) + const ( + pName0 = "pod0" + pName1 = "pod1" + pName2 = "pod0" // ensure pName2 conflicts with pName0, but is in a different namespace + ) + const ( + cName00 = "c0" + cName01 = "c1" + cName10 = "c0" // ensure cName10 conflicts with cName02, but is in a different pod + cName20 = "c1" // ensure cName20 conflicts with cName01, but is in a different pod + namespace + ) + const ( + rootfsCapacity = uint64(10000000) + rootfsAvailable = uint64(5000000) + rootfsInodesFree = uint64(1000) + rootfsInodes = uint64(2000) + imagefsCapacity = uint64(20000000) + imagefsAvailable = uint64(8000000) + imagefsInodesFree = uint64(2000) + imagefsInodes = uint64(4000) + ) + + prf0 := statsapi.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0} + prf1 := statsapi.PodReference{Name: pName1, Namespace: namespace0, UID: "UID" + pName1} + prf2 := statsapi.PodReference{Name: pName2, Namespace: namespace2, UID: "UID" + pName2} + infos := map[string]cadvisorapiv2.ContainerInfo{ + "/": getTestContainerInfo(seedRoot, "", "", ""), + "/docker-daemon": getTestContainerInfo(seedRuntime, "", "", ""), + "/kubelet": getTestContainerInfo(seedKubelet, "", "", ""), + "/system": getTestContainerInfo(seedMisc, "", "", ""), + // Pod0 - Namespace0 + "/pod0-i": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName), + "/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00), + "/pod0-c1": getTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01), + // Pod1 - Namespace0 + "/pod1-i": getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName), + "/pod1-c0": getTestContainerInfo(seedPod1Container, pName1, namespace0, cName10), + // Pod2 - Namespace2 + "/pod2-i": getTestContainerInfo(seedPod2Infra, pName2, namespace2, leaky.PodInfraContainerName), + "/pod2-c0": getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20), + } + + freeRootfsInodes := rootfsInodesFree + totalRootfsInodes := rootfsInodes + rootfs := cadvisorapiv2.FsInfo{ + Capacity: rootfsCapacity, + Available: rootfsAvailable, + InodesFree: &freeRootfsInodes, + Inodes: &totalRootfsInodes, + } + + freeImagefsInodes := imagefsInodesFree + totalImagefsInodes := imagefsInodes + imagefs := cadvisorapiv2.FsInfo{ + Capacity: imagefsCapacity, + Available: imagefsAvailable, + InodesFree: &freeImagefsInodes, + Inodes: &totalImagefsInodes, + } + + // memory limit overrides for each container (used to test available bytes if a memory limit is known) + memoryLimitOverrides := map[string]uint64{ + "/": uint64(1 << 30), + "/pod2-c0": uint64(1 << 15), + } + for name, memoryLimitOverride := range memoryLimitOverrides { + info, found := infos[name] + if !found { + t.Errorf("No container defined with name %v", name) + } + info.Spec.Memory.Limit = memoryLimitOverride + infos[name] = info + } + + options := cadvisorapiv2.RequestOptions{ + IdType: cadvisorapiv2.TypeName, + Count: 2, + Recursive: true, + } + + mockCadvisor := new(cadvisortest.Mock) + mockCadvisor. + On("ContainerInfoV2", "/", options).Return(infos, nil). + On("RootFsInfo").Return(rootfs, nil). + On("ImagesFsInfo").Return(imagefs, nil) + + mockRuntime := new(containertest.Mock) + mockRuntime. + On("ImageStats").Return(&kubecontainer.ImageStats{TotalStorageBytes: 123}, nil) + + resourceAnalyzer := &fakeResourceAnalyzer{} + + p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime) + pods, err := p.ListPodStats() + assert.NoError(t, err) + + assert.Equal(t, 3, len(pods)) + indexPods := make(map[statsapi.PodReference]statsapi.PodStats, len(pods)) + for _, pod := range pods { + indexPods[pod.PodRef] = pod + } + + // Validate Pod0 Results + ps, found := indexPods[prf0] + assert.True(t, found) + assert.Len(t, ps.Containers, 2) + indexCon := make(map[string]statsapi.ContainerStats, len(ps.Containers)) + for _, con := range ps.Containers { + indexCon[con.Name] = con + } + con := indexCon[cName00] + assert.EqualValues(t, testTime(creationTime, seedPod0Container0).Unix(), con.StartTime.Time.Unix()) + checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU) + checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory) + + con = indexCon[cName01] + assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix()) + checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU) + checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory) + + assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix()) + checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network) + + // Validate Pod1 Results + ps, found = indexPods[prf1] + assert.True(t, found) + assert.Len(t, ps.Containers, 1) + con = ps.Containers[0] + assert.Equal(t, cName10, con.Name) + checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU) + checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory) + checkNetworkStats(t, "Pod1", seedPod1Infra, ps.Network) + + // Validate Pod2 Results + ps, found = indexPods[prf2] + assert.True(t, found) + assert.Len(t, ps.Containers, 1) + con = ps.Containers[0] + assert.Equal(t, cName20, con.Name) + checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU) + checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory) + checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network) +} + +func TestImagesFsStats(t *testing.T) { + var ( + assert = assert.New(t) + mockCadvisor = new(cadvisortest.Mock) + mockRuntime = new(containertest.Mock) + + seed = 100 + options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false} + imageFsInfo = getTestFsInfo(100) + containerInfo = map[string]cadvisorapiv2.ContainerInfo{"/": getTestContainerInfo(seed, "test-pod", "test-ns", "test-container")} + imageStats = &kubecontainer.ImageStats{TotalStorageBytes: 100} + ) + + mockCadvisor. + On("ImagesFsInfo").Return(imageFsInfo, nil). + On("ContainerInfoV2", "/", options).Return(containerInfo, nil) + mockRuntime. + On("ImageStats").Return(imageStats, nil) + + provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime) + stats, err := provider.ImageFsStats() + assert.NoError(err) + + assert.Equal(stats.Time, metav1.NewTime(containerInfo["/"].Stats[0].Timestamp)) + assert.Equal(*stats.AvailableBytes, imageFsInfo.Available) + assert.Equal(*stats.CapacityBytes, imageFsInfo.Capacity) + assert.Equal(*stats.UsedBytes, imageStats.TotalStorageBytes) + assert.Equal(stats.InodesFree, imageFsInfo.InodesFree) + assert.Equal(stats.Inodes, imageFsInfo.Inodes) + assert.Equal(*stats.InodesUsed, *imageFsInfo.Inodes-*imageFsInfo.InodesFree) + + mockCadvisor.AssertExpectations(t) +} diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go new file mode 100644 index 0000000000000..105fa853f916f --- /dev/null +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -0,0 +1,66 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "fmt" + + internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/kubelet/server/stats" +) + +// criStatsProvider implements the containerStatsProvider interface by getting +// the container stats from CRI. +type criStatsProvider struct { + // cadvisor is used to get the node root filesystem's stats (such as the + // capacity/available bytes/inodes) that will be populated in per container + // filesystem stats. + cadvisor cadvisor.Interface + // resourceAnalyzer is used to get the volume stats of the pods. + resourceAnalyzer stats.ResourceAnalyzer + // runtimeService is used to get the status and stats of the pods and its + // managed containers. + runtimeService internalapi.RuntimeService + // imageService is used to get the stats of the image filesystem. + imageService internalapi.ImageManagerService +} + +// newCRIStatsProvider returns a containerStatsProvider implementation that +// provides container stats using CRI. +func newCRIStatsProvider( + cadvisor cadvisor.Interface, + resourceAnalyzer stats.ResourceAnalyzer, + runtimeService internalapi.RuntimeService, + imageService internalapi.ImageManagerService, +) containerStatsProvider { + return &criStatsProvider{ + cadvisor: cadvisor, + resourceAnalyzer: resourceAnalyzer, + runtimeService: runtimeService, + imageService: imageService, + } +} + +func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { + return nil, fmt.Errorf("not implemented") +} + +func (p *criStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { + return nil, fmt.Errorf("not implemented") +} diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go new file mode 100644 index 0000000000000..091a0d51da1dc --- /dev/null +++ b/pkg/kubelet/stats/helper.go @@ -0,0 +1,248 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "fmt" + "time" + + "github.com/golang/glog" + + cadvisorapiv1 "github.com/google/cadvisor/info/v1" + cadvisorapiv2 "github.com/google/cadvisor/info/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/kubelet/network" +) + +// cadvisorInfoToContainerStats returns the statsapi.ContainerStats converted +// from the container and filesystem info. +func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo, rootFs, imageFs *cadvisorapiv2.FsInfo) *statsapi.ContainerStats { + result := &statsapi.ContainerStats{ + StartTime: metav1.NewTime(info.Spec.CreationTime), + Name: name, + } + + cstat, found := latestContainerStats(info) + if !found { + return result + } + + if info.Spec.HasCpu { + cpuStats := statsapi.CPUStats{ + Time: metav1.NewTime(cstat.Timestamp), + } + if cstat.CpuInst != nil { + cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total + } + if cstat.Cpu != nil { + cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total + } + result.CPU = &cpuStats + } + + if info.Spec.HasMemory { + pageFaults := cstat.Memory.ContainerData.Pgfault + majorPageFaults := cstat.Memory.ContainerData.Pgmajfault + result.Memory = &statsapi.MemoryStats{ + Time: metav1.NewTime(cstat.Timestamp), + UsageBytes: &cstat.Memory.Usage, + WorkingSetBytes: &cstat.Memory.WorkingSet, + RSSBytes: &cstat.Memory.RSS, + PageFaults: &pageFaults, + MajorPageFaults: &majorPageFaults, + } + // availableBytes = memory limit (if known) - workingset + if !isMemoryUnlimited(info.Spec.Memory.Limit) { + availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet + result.Memory.AvailableBytes = &availableBytes + } + } + + // The container logs live on the node rootfs device + result.Logs = &statsapi.FsStats{ + Time: metav1.NewTime(cstat.Timestamp), + AvailableBytes: &rootFs.Available, + CapacityBytes: &rootFs.Capacity, + InodesFree: rootFs.InodesFree, + Inodes: rootFs.Inodes, + } + + if rootFs.Inodes != nil && rootFs.InodesFree != nil { + logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree + result.Logs.InodesUsed = &logsInodesUsed + } + + // The container rootFs lives on the imageFs devices (which may not be the node root fs) + result.Rootfs = &statsapi.FsStats{ + Time: metav1.NewTime(cstat.Timestamp), + AvailableBytes: &imageFs.Available, + CapacityBytes: &imageFs.Capacity, + InodesFree: imageFs.InodesFree, + Inodes: imageFs.Inodes, + } + + cfs := cstat.Filesystem + if cfs != nil { + if cfs.BaseUsageBytes != nil { + rootfsUsage := *cfs.BaseUsageBytes + result.Rootfs.UsedBytes = &rootfsUsage + if cfs.TotalUsageBytes != nil { + logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes + result.Logs.UsedBytes = &logsUsage + } + } + if cfs.InodeUsage != nil { + rootInodes := *cfs.InodeUsage + result.Rootfs.InodesUsed = &rootInodes + } + } + + result.UserDefinedMetrics = cadvisorInfoToUserDefinedMetrics(info) + return result +} + +// cadvisorInfoToNetworkStats returns the statsapi.NetworkStats converted from +// the container info from cadvisor. +func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.NetworkStats { + if !info.Spec.HasNetwork { + return nil + } + cstat, found := latestContainerStats(info) + if !found { + return nil + } + for _, inter := range cstat.Network.Interfaces { + if inter.Name == network.DefaultInterfaceName { + return &statsapi.NetworkStats{ + Time: metav1.NewTime(cstat.Timestamp), + RxBytes: &inter.RxBytes, + RxErrors: &inter.RxErrors, + TxBytes: &inter.TxBytes, + TxErrors: &inter.TxErrors, + } + } + } + glog.V(4).Infof("Missing default interface %q for %s", network.DefaultInterfaceName, name) + return nil +} + +// cadvisorInfoToUserDefinedMetrics returns the statsapi.UserDefinedMetric +// converted from the container info from cadvisor. +func cadvisorInfoToUserDefinedMetrics(info *cadvisorapiv2.ContainerInfo) []statsapi.UserDefinedMetric { + type specVal struct { + ref statsapi.UserDefinedMetricDescriptor + valType cadvisorapiv1.DataType + time time.Time + value float64 + } + udmMap := map[string]*specVal{} + for _, spec := range info.Spec.CustomMetrics { + udmMap[spec.Name] = &specVal{ + ref: statsapi.UserDefinedMetricDescriptor{ + Name: spec.Name, + Type: statsapi.UserDefinedMetricType(spec.Type), + Units: spec.Units, + }, + valType: spec.Format, + } + } + for _, stat := range info.Stats { + for name, values := range stat.CustomMetrics { + specVal, ok := udmMap[name] + if !ok { + glog.Warningf("spec for custom metric %q is missing from cAdvisor output. Spec: %+v, Metrics: %+v", name, info.Spec, stat.CustomMetrics) + continue + } + for _, value := range values { + // Pick the most recent value + if value.Timestamp.Before(specVal.time) { + continue + } + specVal.time = value.Timestamp + specVal.value = value.FloatValue + if specVal.valType == cadvisorapiv1.IntType { + specVal.value = float64(value.IntValue) + } + } + } + } + var udm []statsapi.UserDefinedMetric + for _, specVal := range udmMap { + udm = append(udm, statsapi.UserDefinedMetric{ + UserDefinedMetricDescriptor: specVal.ref, + Time: metav1.NewTime(specVal.time), + Value: specVal.value, + }) + } + return udm +} + +// latestContainerStats returns the latest container stats from cadvisor, or nil if none exist +func latestContainerStats(info *cadvisorapiv2.ContainerInfo) (*cadvisorapiv2.ContainerStats, bool) { + stats := info.Stats + if len(stats) < 1 { + return nil, false + } + latest := stats[len(stats)-1] + if latest == nil { + return nil, false + } + return latest, true +} + +func isMemoryUnlimited(v uint64) bool { + // Size after which we consider memory to be "unlimited". This is not + // MaxInt64 due to rounding by the kernel. + // TODO: cadvisor should export this https://github.com/google/cadvisor/blob/master/metrics/prometheus.go#L596 + const maxMemorySize = uint64(1 << 62) + + return v > maxMemorySize +} + +// getCgroupInfo returns the information of the container with the specified +// containerName from cadvisor. +func getCgroupInfo(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerInfo, error) { + infoMap, err := cadvisor.ContainerInfoV2(containerName, cadvisorapiv2.RequestOptions{ + IdType: cadvisorapiv2.TypeName, + Count: 2, // 2 samples are needed to compute "instantaneous" CPU + Recursive: false, + }) + if err != nil { + return nil, fmt.Errorf("failed to get container info for %q: %v", containerName, err) + } + if len(infoMap) != 1 { + return nil, fmt.Errorf("unexpected number of containers: %v", len(infoMap)) + } + info := infoMap[containerName] + return &info, nil +} + +// getCgroupStats returns the latest stats of the container having the +// specified containerName from cadvisor. +func getCgroupStats(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerStats, error) { + info, err := getCgroupInfo(cadvisor, containerName) + if err != nil { + return nil, err + } + stats, found := latestContainerStats(info) + if !found { + return nil, fmt.Errorf("failed to get latest stats from container info for %q", containerName) + } + return stats, nil +} diff --git a/pkg/kubelet/stats/helper_test.go b/pkg/kubelet/stats/helper_test.go new file mode 100644 index 0000000000000..c1b13423cbb75 --- /dev/null +++ b/pkg/kubelet/stats/helper_test.go @@ -0,0 +1,99 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "testing" + "time" + + cadvisorapiv1 "github.com/google/cadvisor/info/v1" + cadvisorapiv2 "github.com/google/cadvisor/info/v2" + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" +) + +func TestCustomMetrics(t *testing.T) { + spec := []cadvisorapiv1.MetricSpec{ + { + Name: "qos", + Type: cadvisorapiv1.MetricGauge, + Format: cadvisorapiv1.IntType, + Units: "per second", + }, + { + Name: "cpuLoad", + Type: cadvisorapiv1.MetricCumulative, + Format: cadvisorapiv1.FloatType, + Units: "count", + }, + } + timestamp1 := time.Now() + timestamp2 := time.Now().Add(time.Minute) + metrics := map[string][]cadvisorapiv1.MetricVal{ + "qos": { + { + Timestamp: timestamp1, + IntValue: 10, + }, + { + Timestamp: timestamp2, + IntValue: 100, + }, + }, + "cpuLoad": { + { + Timestamp: timestamp1, + FloatValue: 1.2, + }, + { + Timestamp: timestamp2, + FloatValue: 2.1, + }, + }, + } + cInfo := cadvisorapiv2.ContainerInfo{ + Spec: cadvisorapiv2.ContainerSpec{ + CustomMetrics: spec, + }, + Stats: []*cadvisorapiv2.ContainerStats{ + { + CustomMetrics: metrics, + }, + }, + } + assert.Contains(t, cadvisorInfoToUserDefinedMetrics(&cInfo), + statsapi.UserDefinedMetric{ + UserDefinedMetricDescriptor: statsapi.UserDefinedMetricDescriptor{ + Name: "qos", + Type: statsapi.MetricGauge, + Units: "per second", + }, + Time: metav1.NewTime(timestamp2), + Value: 100, + }, + statsapi.UserDefinedMetric{ + UserDefinedMetricDescriptor: statsapi.UserDefinedMetricDescriptor{ + Name: "cpuLoad", + Type: statsapi.MetricCumulative, + Units: "count", + }, + Time: metav1.NewTime(timestamp2), + Value: 2.1, + }) +} diff --git a/pkg/kubelet/stats/stats_provider.go b/pkg/kubelet/stats/stats_provider.go new file mode 100644 index 0000000000000..af7e9d418bfbd --- /dev/null +++ b/pkg/kubelet/stats/stats_provider.go @@ -0,0 +1,175 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "fmt" + + cadvisorapiv1 "github.com/google/cadvisor/info/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/kubelet/cadvisor" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + kubepod "k8s.io/kubernetes/pkg/kubelet/pod" + "k8s.io/kubernetes/pkg/kubelet/server/stats" +) + +// NewCRIStatsProvider returns a StatsProvider that provides the node stats +// from cAdvisor and the container stats from CRI. +func NewCRIStatsProvider( + cadvisor cadvisor.Interface, + resourceAnalyzer stats.ResourceAnalyzer, + podManager kubepod.Manager, + runtimeCache kubecontainer.RuntimeCache, + runtimeService internalapi.RuntimeService, + imageService internalapi.ImageManagerService, +) *StatsProvider { + return newStatsProvider(cadvisor, podManager, runtimeCache, newCRIStatsProvider(cadvisor, resourceAnalyzer, runtimeService, imageService)) +} + +// NewCadvisorStatsProvider returns a containerStatsProvider that provides both +// the node and the container stats from cAdvisor. +func NewCadvisorStatsProvider( + cadvisor cadvisor.Interface, + resourceAnalyzer stats.ResourceAnalyzer, + podManager kubepod.Manager, + runtimeCache kubecontainer.RuntimeCache, + imageService kubecontainer.ImageService, +) *StatsProvider { + return newStatsProvider(cadvisor, podManager, runtimeCache, newCadvisorStatsProvider(cadvisor, resourceAnalyzer, imageService)) +} + +// newStatsProvider returns a new StatsProvider that provides node stats from +// cAdvisor and the container stats using the containerStatsProvider. +func newStatsProvider( + cadvisor cadvisor.Interface, + podManager kubepod.Manager, + runtimeCache kubecontainer.RuntimeCache, + containerStatsProvider containerStatsProvider, +) *StatsProvider { + return &StatsProvider{ + cadvisor: cadvisor, + podManager: podManager, + runtimeCache: runtimeCache, + containerStatsProvider: containerStatsProvider, + } +} + +// StatsProvider provides the stats of the node and the pod-managed containers. +type StatsProvider struct { + cadvisor cadvisor.Interface + podManager kubepod.Manager + runtimeCache kubecontainer.RuntimeCache + containerStatsProvider +} + +// containerStatsProvider is an interface that provides the stats of the +// containers managed by pods. +type containerStatsProvider interface { + ListPodStats() ([]statsapi.PodStats, error) + ImageFsStats() (*statsapi.FsStats, error) +} + +// GetCgroupStats returns the stats of the cgroup with the cgroupName. +func (p *StatsProvider) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) { + info, err := getCgroupInfo(p.cadvisor, cgroupName) + if err != nil { + return nil, nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err) + } + rootFsInfo, err := p.cadvisor.RootFsInfo() + if err != nil { + return nil, nil, fmt.Errorf("failed to get rootFs info: %v", err) + } + imageFsInfo, err := p.cadvisor.ImagesFsInfo() + if err != nil { + return nil, nil, fmt.Errorf("failed to get imageFs info: %v", err) + } + s := cadvisorInfoToContainerStats(cgroupName, info, &rootFsInfo, &imageFsInfo) + n := cadvisorInfoToNetworkStats(cgroupName, info) + return s, n, nil +} + +// RootFsStats returns the stats of the node root filesystem. +func (p *StatsProvider) RootFsStats() (*statsapi.FsStats, error) { + rootFsInfo, err := p.cadvisor.RootFsInfo() + if err != nil { + return nil, fmt.Errorf("failed to get rootFs info: %v", err) + } + + var nodeFsInodesUsed *uint64 + if rootFsInfo.Inodes != nil && rootFsInfo.InodesFree != nil { + nodeFsIU := *rootFsInfo.Inodes - *rootFsInfo.InodesFree + nodeFsInodesUsed = &nodeFsIU + } + + // Get the root container stats's timestamp, which will be used as the + // imageFs stats timestamp. + rootStats, err := getCgroupStats(p.cadvisor, "/") + if err != nil { + return nil, fmt.Errorf("failed to get root container stats: %v", err) + } + + return &statsapi.FsStats{ + Time: metav1.NewTime(rootStats.Timestamp), + AvailableBytes: &rootFsInfo.Available, + CapacityBytes: &rootFsInfo.Capacity, + UsedBytes: &rootFsInfo.Usage, + InodesFree: rootFsInfo.InodesFree, + Inodes: rootFsInfo.Inodes, + InodesUsed: nodeFsInodesUsed, + }, nil +} + +// GetContainerInfo returns stats (from cAdvisor) for a container. +func (p *StatsProvider) GetContainerInfo(podFullName string, podUID types.UID, containerName string, req *cadvisorapiv1.ContainerInfoRequest) (*cadvisorapiv1.ContainerInfo, error) { + // Resolve and type convert back again. + // We need the static pod UID but the kubecontainer API works with types.UID. + podUID = types.UID(p.podManager.TranslatePodUID(podUID)) + + pods, err := p.runtimeCache.GetPods() + if err != nil { + return nil, err + } + pod := kubecontainer.Pods(pods).FindPod(podFullName, podUID) + container := pod.FindContainerByName(containerName) + if container == nil { + return nil, kubecontainer.ErrContainerNotFound + } + + ci, err := p.cadvisor.DockerContainer(container.ID.ID, req) + if err != nil { + return nil, err + } + return &ci, nil +} + +// GetRawContainerInfo returns the stats (from cadvisor) for a non-Kubernetes +// container. +func (p *StatsProvider) GetRawContainerInfo(containerName string, req *cadvisorapiv1.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorapiv1.ContainerInfo, error) { + if subcontainers { + return p.cadvisor.SubcontainerInfo(containerName, req) + } + containerInfo, err := p.cadvisor.ContainerInfo(containerName, req) + if err != nil { + return nil, err + } + return map[string]*cadvisorapiv1.ContainerInfo{ + containerInfo.Name: containerInfo, + }, nil +} diff --git a/pkg/kubelet/stats/stats_provider_test.go b/pkg/kubelet/stats/stats_provider_test.go new file mode 100644 index 0000000000000..532a43292e3bc --- /dev/null +++ b/pkg/kubelet/stats/stats_provider_test.go @@ -0,0 +1,579 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "fmt" + "testing" + "time" + + cadvisorapiv1 "github.com/google/cadvisor/info/v1" + cadvisorapiv2 "github.com/google/cadvisor/info/v2" + fuzz "github.com/google/gofuzz" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing" + kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing" + serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats" + kubetypes "k8s.io/kubernetes/pkg/kubelet/types" +) + +const ( + // Offsets from seed value in generated container stats. + offsetCPUUsageCores = iota + offsetCPUUsageCoreSeconds + offsetMemPageFaults + offsetMemMajorPageFaults + offsetMemUsageBytes + offsetMemRSSBytes + offsetMemWorkingSetBytes + offsetNetRxBytes + offsetNetRxErrors + offsetNetTxBytes + offsetNetTxErrors + offsetFsCapacity + offsetFsAvailable + offsetFsUsage + offsetFsInodes + offsetFsInodesFree + offsetFsTotalUsageBytes + offsetFsBaseUsageBytes + offsetFsInodeUsage +) + +var ( + timestamp = time.Now() + creationTime = timestamp.Add(-5 * time.Minute) +) + +func TestGetCgroupStats(t *testing.T) { + const ( + cgroupName = "test-cgroup-name" + rootFsInfoSeed = 1000 + imageFsInfoSeed = 2000 + containerInfoSeed = 3000 + ) + var ( + mockCadvisor = new(cadvisortest.Mock) + mockPodManager = new(kubepodtest.MockManager) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + + assert = assert.New(t) + options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false} + + rootFsInfo = getTestFsInfo(rootFsInfoSeed) + imageFsInfo = getTestFsInfo(imageFsInfoSeed) + containerInfo = getTestContainerInfo(containerInfoSeed, "test-pod", "test-ns", "test-container") + containerInfoMap = map[string]cadvisorapiv2.ContainerInfo{cgroupName: containerInfo} + ) + + mockCadvisor. + On("RootFsInfo").Return(rootFsInfo, nil). + On("ImagesFsInfo").Return(imageFsInfo, nil). + On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil) + + provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{}) + cs, ns, err := provider.GetCgroupStats(cgroupName) + assert.NoError(err) + + checkCPUStats(t, "", containerInfoSeed, cs.CPU) + checkMemoryStats(t, "", containerInfoSeed, containerInfo, cs.Memory) + checkFsStats(t, "", imageFsInfoSeed, cs.Rootfs) + checkFsStats(t, "", rootFsInfoSeed, cs.Logs) + checkNetworkStats(t, "", containerInfoSeed, ns) + + assert.Equal(cs.Name, cgroupName) + assert.Equal(cs.StartTime, metav1.NewTime(containerInfo.Spec.CreationTime)) + + assert.Equal(cs.Rootfs.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp)) + assert.Equal(*cs.Rootfs.UsedBytes, *containerInfo.Stats[0].Filesystem.BaseUsageBytes) + assert.Equal(*cs.Rootfs.InodesUsed, *containerInfo.Stats[0].Filesystem.InodeUsage) + + assert.Equal(cs.Logs.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp)) + assert.Equal(*cs.Logs.UsedBytes, *containerInfo.Stats[0].Filesystem.TotalUsageBytes-*containerInfo.Stats[0].Filesystem.BaseUsageBytes) + assert.Equal(*cs.Logs.InodesUsed, *rootFsInfo.Inodes-*rootFsInfo.InodesFree) + + mockCadvisor.AssertExpectations(t) +} + +func TestRootFsStats(t *testing.T) { + const ( + rootFsInfoSeed = 1000 + containerInfoSeed = 2000 + ) + var ( + mockCadvisor = new(cadvisortest.Mock) + mockPodManager = new(kubepodtest.MockManager) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + + assert = assert.New(t) + options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false} + + rootFsInfo = getTestFsInfo(rootFsInfoSeed) + containerInfo = getTestContainerInfo(containerInfoSeed, "test-pod", "test-ns", "test-container") + containerInfoMap = map[string]cadvisorapiv2.ContainerInfo{"/": containerInfo} + ) + + mockCadvisor. + On("RootFsInfo").Return(rootFsInfo, nil). + On("ContainerInfoV2", "/", options).Return(containerInfoMap, nil) + + provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{}) + stats, err := provider.RootFsStats() + assert.NoError(err) + + checkFsStats(t, "", rootFsInfoSeed, stats) + + assert.Equal(stats.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp)) + assert.Equal(*stats.UsedBytes, rootFsInfo.Usage) + assert.Equal(*stats.InodesUsed, *rootFsInfo.Inodes-*rootFsInfo.InodesFree) + + mockCadvisor.AssertExpectations(t) +} + +func TestGetContainerInfo(t *testing.T) { + cadvisorAPIFailure := fmt.Errorf("cAdvisor failure") + runtimeError := fmt.Errorf("List containers error") + tests := []struct { + name string + containerID string + containerPath string + cadvisorContainerInfo cadvisorapiv1.ContainerInfo + runtimeError error + podList []*kubecontainer.Pod + requestedPodFullName string + requestedPodUID types.UID + requestedContainerName string + expectDockerContainerCall bool + mockError error + expectedError error + expectStats bool + }{ + { + name: "get container info", + containerID: "ab2cdf", + containerPath: "/docker/ab2cdf", + cadvisorContainerInfo: cadvisorapiv1.ContainerInfo{ + ContainerReference: cadvisorapiv1.ContainerReference{ + Name: "/docker/ab2cdf", + }, + }, + runtimeError: nil, + podList: []*kubecontainer.Pod{ + { + ID: "12345678", + Name: "qux", + Namespace: "ns", + Containers: []*kubecontainer.Container{ + { + Name: "foo", + ID: kubecontainer.ContainerID{Type: "test", ID: "ab2cdf"}, + }, + }, + }, + }, + requestedPodFullName: "qux_ns", + requestedPodUID: "", + requestedContainerName: "foo", + expectDockerContainerCall: true, + mockError: nil, + expectedError: nil, + expectStats: true, + }, + { + name: "get container info when cadvisor failed", + containerID: "ab2cdf", + containerPath: "/docker/ab2cdf", + cadvisorContainerInfo: cadvisorapiv1.ContainerInfo{}, + runtimeError: nil, + podList: []*kubecontainer.Pod{ + { + ID: "uuid", + Name: "qux", + Namespace: "ns", + Containers: []*kubecontainer.Container{ + { + Name: "foo", + ID: kubecontainer.ContainerID{Type: "test", ID: "ab2cdf"}, + }, + }, + }, + }, + requestedPodFullName: "qux_ns", + requestedPodUID: "uuid", + requestedContainerName: "foo", + expectDockerContainerCall: true, + mockError: cadvisorAPIFailure, + expectedError: cadvisorAPIFailure, + expectStats: false, + }, + { + name: "get container info on non-existent container", + containerID: "", + containerPath: "", + cadvisorContainerInfo: cadvisorapiv1.ContainerInfo{}, + runtimeError: nil, + podList: []*kubecontainer.Pod{}, + requestedPodFullName: "qux", + requestedPodUID: "", + requestedContainerName: "foo", + expectDockerContainerCall: false, + mockError: nil, + expectedError: kubecontainer.ErrContainerNotFound, + expectStats: false, + }, + { + name: "get container info when container runtime failed", + containerID: "", + containerPath: "", + cadvisorContainerInfo: cadvisorapiv1.ContainerInfo{}, + runtimeError: runtimeError, + podList: []*kubecontainer.Pod{}, + requestedPodFullName: "qux", + requestedPodUID: "", + requestedContainerName: "foo", + mockError: nil, + expectedError: runtimeError, + expectStats: false, + }, + { + name: "get container info with no containers", + containerID: "", + containerPath: "", + cadvisorContainerInfo: cadvisorapiv1.ContainerInfo{}, + runtimeError: nil, + podList: []*kubecontainer.Pod{}, + requestedPodFullName: "qux_ns", + requestedPodUID: "", + requestedContainerName: "foo", + mockError: nil, + expectedError: kubecontainer.ErrContainerNotFound, + expectStats: false, + }, + { + name: "get container info with no matching containers", + containerID: "", + containerPath: "", + cadvisorContainerInfo: cadvisorapiv1.ContainerInfo{}, + runtimeError: nil, + podList: []*kubecontainer.Pod{ + { + ID: "12345678", + Name: "qux", + Namespace: "ns", + Containers: []*kubecontainer.Container{ + { + Name: "bar", + ID: kubecontainer.ContainerID{Type: "test", ID: "fakeID"}, + }, + }, + }, + }, + requestedPodFullName: "qux_ns", + requestedPodUID: "", + requestedContainerName: "foo", + mockError: nil, + expectedError: kubecontainer.ErrContainerNotFound, + expectStats: false, + }, + } + + for _, tc := range tests { + var ( + mockCadvisor = new(cadvisortest.Mock) + mockPodManager = new(kubepodtest.MockManager) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + + cadvisorReq = &cadvisorapiv1.ContainerInfoRequest{} + ) + + mockPodManager.On("TranslatePodUID", tc.requestedPodUID).Return(kubetypes.ResolvedPodUID(tc.requestedPodUID)) + mockRuntimeCache.On("GetPods").Return(tc.podList, tc.runtimeError) + if tc.expectDockerContainerCall { + mockCadvisor.On("DockerContainer", tc.containerID, cadvisorReq).Return(tc.cadvisorContainerInfo, tc.mockError) + } + + provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{}) + stats, err := provider.GetContainerInfo(tc.requestedPodFullName, tc.requestedPodUID, tc.requestedContainerName, cadvisorReq) + assert.Equal(t, tc.expectedError, err) + + if tc.expectStats { + require.NotNil(t, stats) + } + mockCadvisor.AssertExpectations(t) + } +} + +func TestGetRawContainerInfoRoot(t *testing.T) { + var ( + mockCadvisor = new(cadvisortest.Mock) + mockPodManager = new(kubepodtest.MockManager) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + + cadvisorReq = &cadvisorapiv1.ContainerInfoRequest{} + containerPath = "/" + containerInfo = &cadvisorapiv1.ContainerInfo{ + ContainerReference: cadvisorapiv1.ContainerReference{ + Name: containerPath, + }, + } + ) + + mockCadvisor.On("ContainerInfo", containerPath, cadvisorReq).Return(containerInfo, nil) + + provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{}) + _, err := provider.GetRawContainerInfo(containerPath, cadvisorReq, false) + assert.NoError(t, err) + mockCadvisor.AssertExpectations(t) +} + +func TestGetRawContainerInfoSubcontainers(t *testing.T) { + var ( + mockCadvisor = new(cadvisortest.Mock) + mockPodManager = new(kubepodtest.MockManager) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + + cadvisorReq = &cadvisorapiv1.ContainerInfoRequest{} + containerPath = "/kubelet" + containerInfo = map[string]*cadvisorapiv1.ContainerInfo{ + containerPath: { + ContainerReference: cadvisorapiv1.ContainerReference{ + Name: containerPath, + }, + }, + "/kubelet/sub": { + ContainerReference: cadvisorapiv1.ContainerReference{ + Name: "/kubelet/sub", + }, + }, + } + ) + + mockCadvisor.On("SubcontainerInfo", containerPath, cadvisorReq).Return(containerInfo, nil) + + provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{}) + result, err := provider.GetRawContainerInfo(containerPath, cadvisorReq, true) + assert.NoError(t, err) + assert.Len(t, result, 2) + mockCadvisor.AssertExpectations(t) +} + +func getTerminatedContainerInfo(seed int, podName string, podNamespace string, containerName string) cadvisorapiv2.ContainerInfo { + cinfo := getTestContainerInfo(seed, podName, podNamespace, containerName) + cinfo.Stats[0].Memory.RSS = 0 + cinfo.Stats[0].CpuInst.Usage.Total = 0 + return cinfo +} + +func getTestContainerInfo(seed int, podName string, podNamespace string, containerName string) cadvisorapiv2.ContainerInfo { + labels := map[string]string{} + if podName != "" { + labels = map[string]string{ + "io.kubernetes.pod.name": podName, + "io.kubernetes.pod.uid": "UID" + podName, + "io.kubernetes.pod.namespace": podNamespace, + "io.kubernetes.container.name": containerName, + } + } + // by default, kernel will set memory.limit_in_bytes to 1 << 63 if not bounded + unlimitedMemory := uint64(1 << 63) + spec := cadvisorapiv2.ContainerSpec{ + CreationTime: testTime(creationTime, seed), + HasCpu: true, + HasMemory: true, + HasNetwork: true, + Labels: labels, + Memory: cadvisorapiv2.MemorySpec{ + Limit: unlimitedMemory, + }, + CustomMetrics: generateCustomMetricSpec(), + } + + totalUsageBytes := uint64(seed + offsetFsTotalUsageBytes) + baseUsageBytes := uint64(seed + offsetFsBaseUsageBytes) + inodeUsage := uint64(seed + offsetFsInodeUsage) + + stats := cadvisorapiv2.ContainerStats{ + Timestamp: testTime(timestamp, seed), + Cpu: &cadvisorapiv1.CpuStats{}, + CpuInst: &cadvisorapiv2.CpuInstStats{}, + Memory: &cadvisorapiv1.MemoryStats{ + Usage: uint64(seed + offsetMemUsageBytes), + WorkingSet: uint64(seed + offsetMemWorkingSetBytes), + RSS: uint64(seed + offsetMemRSSBytes), + ContainerData: cadvisorapiv1.MemoryStatsMemoryData{ + Pgfault: uint64(seed + offsetMemPageFaults), + Pgmajfault: uint64(seed + offsetMemMajorPageFaults), + }, + }, + Network: &cadvisorapiv2.NetworkStats{ + Interfaces: []cadvisorapiv1.InterfaceStats{{ + Name: "eth0", + RxBytes: uint64(seed + offsetNetRxBytes), + RxErrors: uint64(seed + offsetNetRxErrors), + TxBytes: uint64(seed + offsetNetTxBytes), + TxErrors: uint64(seed + offsetNetTxErrors), + }, { + Name: "cbr0", + RxBytes: 100, + RxErrors: 100, + TxBytes: 100, + TxErrors: 100, + }}, + }, + CustomMetrics: generateCustomMetrics(spec.CustomMetrics), + Filesystem: &cadvisorapiv2.FilesystemStats{ + TotalUsageBytes: &totalUsageBytes, + BaseUsageBytes: &baseUsageBytes, + InodeUsage: &inodeUsage, + }, + } + stats.Cpu.Usage.Total = uint64(seed + offsetCPUUsageCoreSeconds) + stats.CpuInst.Usage.Total = uint64(seed + offsetCPUUsageCores) + return cadvisorapiv2.ContainerInfo{ + Spec: spec, + Stats: []*cadvisorapiv2.ContainerStats{&stats}, + } +} + +func getTestFsInfo(seed int) cadvisorapiv2.FsInfo { + var ( + inodes = uint64(seed + offsetFsInodes) + inodesFree = uint64(seed + offsetFsInodesFree) + ) + return cadvisorapiv2.FsInfo{ + Device: "test-device", + Mountpoint: "test-mount-point", + Capacity: uint64(seed + offsetFsCapacity), + Available: uint64(seed + offsetFsAvailable), + Usage: uint64(seed + offsetFsUsage), + Inodes: &inodes, + InodesFree: &inodesFree, + } +} + +func generateCustomMetricSpec() []cadvisorapiv1.MetricSpec { + f := fuzz.New().NilChance(0).Funcs( + func(e *cadvisorapiv1.MetricSpec, c fuzz.Continue) { + c.Fuzz(&e.Name) + switch c.Intn(3) { + case 0: + e.Type = cadvisorapiv1.MetricGauge + case 1: + e.Type = cadvisorapiv1.MetricCumulative + case 2: + e.Type = cadvisorapiv1.MetricDelta + } + switch c.Intn(2) { + case 0: + e.Format = cadvisorapiv1.IntType + case 1: + e.Format = cadvisorapiv1.FloatType + } + c.Fuzz(&e.Units) + }) + var ret []cadvisorapiv1.MetricSpec + f.Fuzz(&ret) + return ret +} + +func generateCustomMetrics(spec []cadvisorapiv1.MetricSpec) map[string][]cadvisorapiv1.MetricVal { + ret := map[string][]cadvisorapiv1.MetricVal{} + for _, metricSpec := range spec { + f := fuzz.New().NilChance(0).Funcs( + func(e *cadvisorapiv1.MetricVal, c fuzz.Continue) { + switch metricSpec.Format { + case cadvisorapiv1.IntType: + c.Fuzz(&e.IntValue) + case cadvisorapiv1.FloatType: + c.Fuzz(&e.FloatValue) + } + }) + + var metrics []cadvisorapiv1.MetricVal + f.Fuzz(&metrics) + ret[metricSpec.Name] = metrics + } + return ret +} + +func testTime(base time.Time, seed int) time.Time { + return base.Add(time.Duration(seed) * time.Second) +} + +func checkNetworkStats(t *testing.T, label string, seed int, stats *statsapi.NetworkStats) { + assert.NotNil(t, stats) + assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Net.Time") + assert.EqualValues(t, seed+offsetNetRxBytes, *stats.RxBytes, label+".Net.RxBytes") + assert.EqualValues(t, seed+offsetNetRxErrors, *stats.RxErrors, label+".Net.RxErrors") + assert.EqualValues(t, seed+offsetNetTxBytes, *stats.TxBytes, label+".Net.TxBytes") + assert.EqualValues(t, seed+offsetNetTxErrors, *stats.TxErrors, label+".Net.TxErrors") +} + +func checkCPUStats(t *testing.T, label string, seed int, stats *statsapi.CPUStats) { + assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".CPU.Time") + assert.EqualValues(t, seed+offsetCPUUsageCores, *stats.UsageNanoCores, label+".CPU.UsageCores") + assert.EqualValues(t, seed+offsetCPUUsageCoreSeconds, *stats.UsageCoreNanoSeconds, label+".CPU.UsageCoreSeconds") +} + +func checkMemoryStats(t *testing.T, label string, seed int, info cadvisorapiv2.ContainerInfo, stats *statsapi.MemoryStats) { + assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Mem.Time") + assert.EqualValues(t, seed+offsetMemUsageBytes, *stats.UsageBytes, label+".Mem.UsageBytes") + assert.EqualValues(t, seed+offsetMemWorkingSetBytes, *stats.WorkingSetBytes, label+".Mem.WorkingSetBytes") + assert.EqualValues(t, seed+offsetMemRSSBytes, *stats.RSSBytes, label+".Mem.RSSBytes") + assert.EqualValues(t, seed+offsetMemPageFaults, *stats.PageFaults, label+".Mem.PageFaults") + assert.EqualValues(t, seed+offsetMemMajorPageFaults, *stats.MajorPageFaults, label+".Mem.MajorPageFaults") + if !info.Spec.HasMemory || isMemoryUnlimited(info.Spec.Memory.Limit) { + assert.Nil(t, stats.AvailableBytes, label+".Mem.AvailableBytes") + } else { + expected := info.Spec.Memory.Limit - *stats.WorkingSetBytes + assert.EqualValues(t, expected, *stats.AvailableBytes, label+".Mem.AvailableBytes") + } +} + +func checkFsStats(t *testing.T, label string, seed int, stats *statsapi.FsStats) { + assert.EqualValues(t, seed+offsetFsCapacity, *stats.CapacityBytes, label+".CapacityBytes") + assert.EqualValues(t, seed+offsetFsAvailable, *stats.AvailableBytes, label+".AvailableBytes") + assert.EqualValues(t, seed+offsetFsInodes, *stats.Inodes, label+".Inodes") + assert.EqualValues(t, seed+offsetFsInodesFree, *stats.InodesFree, label+".InodesFree") +} + +type fakeResourceAnalyzer struct { + podVolumeStats serverstats.PodVolumeStats +} + +func (o *fakeResourceAnalyzer) Start() {} +func (o *fakeResourceAnalyzer) Get() (*statsapi.Summary, error) { return nil, nil } +func (o *fakeResourceAnalyzer) GetPodVolumeStats(uid types.UID) (serverstats.PodVolumeStats, bool) { + return o.podVolumeStats, true +} + +type fakeContainerStatsProvider struct { +} + +func (p fakeContainerStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { + return nil, fmt.Errorf("not implemented") +} +func (p fakeContainerStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { + return nil, fmt.Errorf("not implemented") +} diff --git a/pkg/kubelet/status/status_manager_test.go b/pkg/kubelet/status/status_manager_test.go index 23148065c4900..dff67680e03a6 100644 --- a/pkg/kubelet/status/status_manager_test.go +++ b/pkg/kubelet/status/status_manager_test.go @@ -214,7 +214,7 @@ func TestChangedStatusKeepsStartTime(t *testing.T) { t.Errorf("StartTime should not be zero") } expected := now.Rfc3339Copy() - if !finalStatus.StartTime.Equal(expected) { + if !finalStatus.StartTime.Equal(&expected) { t.Errorf("Expected %v, but got %v", expected, finalStatus.StartTime) } } @@ -244,7 +244,7 @@ func TestChangedStatusUpdatesLastTransitionTime(t *testing.T) { if newReadyCondition.LastTransitionTime.IsZero() { t.Errorf("Unexpected: last transition time not set") } - if newReadyCondition.LastTransitionTime.Before(oldReadyCondition.LastTransitionTime) { + if newReadyCondition.LastTransitionTime.Before(&oldReadyCondition.LastTransitionTime) { t.Errorf("Unexpected: new transition time %s, is before old transition time %s", newReadyCondition.LastTransitionTime, oldReadyCondition.LastTransitionTime) } } @@ -283,7 +283,7 @@ func TestUnchangedStatusPreservesLastTransitionTime(t *testing.T) { if newReadyCondition.LastTransitionTime.IsZero() { t.Errorf("Unexpected: last transition time not set") } - if !oldReadyCondition.LastTransitionTime.Equal(newReadyCondition.LastTransitionTime) { + if !oldReadyCondition.LastTransitionTime.Equal(&newReadyCondition.LastTransitionTime) { t.Errorf("Unexpected: new transition time %s, is not equal to old transition time %s", newReadyCondition.LastTransitionTime, oldReadyCondition.LastTransitionTime) } } diff --git a/pkg/kubelet/util/sliceutils/sliceutils.go b/pkg/kubelet/util/sliceutils/sliceutils.go index f1a56e9e48fad..ff9fb56031f51 100644 --- a/pkg/kubelet/util/sliceutils/sliceutils.go +++ b/pkg/kubelet/util/sliceutils/sliceutils.go @@ -44,7 +44,7 @@ func (s PodsByCreationTime) Swap(i, j int) { } func (s PodsByCreationTime) Less(i, j int) bool { - return s[i].CreationTimestamp.Before(s[j].CreationTimestamp) + return s[i].CreationTimestamp.Before(&s[j].CreationTimestamp) } // ByImageSize makes an array of images sortable by their size in descending diff --git a/pkg/kubelet/volume_host.go b/pkg/kubelet/volume_host.go index 51dd94f023f18..26323b7d84883 100644 --- a/pkg/kubelet/volume_host.go +++ b/pkg/kubelet/volume_host.go @@ -41,7 +41,8 @@ func NewInitializedVolumePluginMgr( kubelet *Kubelet, secretManager secret.Manager, configMapManager configmap.Manager, - plugins []volume.VolumePlugin) (*volume.VolumePluginMgr, error) { + plugins []volume.VolumePlugin, + prober volume.DynamicPluginProber) (*volume.VolumePluginMgr, error) { kvh := &kubeletVolumeHost{ kubelet: kubelet, volumePluginMgr: volume.VolumePluginMgr{}, @@ -49,7 +50,7 @@ func NewInitializedVolumePluginMgr( configMapManager: configMapManager, } - if err := kvh.volumePluginMgr.InitPlugins(plugins, kvh); err != nil { + if err := kvh.volumePluginMgr.InitPlugins(plugins, prober, kvh); err != nil { return nil, fmt.Errorf( "Could not initialize volume plugins for KubeletVolumePluginMgr: %v", err) @@ -117,7 +118,7 @@ func (kvh *kubeletVolumeHost) GetCloudProvider() cloudprovider.Interface { return kvh.kubelet.cloud } -func (kvh *kubeletVolumeHost) GetMounter() mount.Interface { +func (kvh *kubeletVolumeHost) GetMounter(pluginName string) mount.Interface { return kvh.kubelet.mounter } @@ -156,3 +157,7 @@ func (kvh *kubeletVolumeHost) GetNodeLabels() (map[string]string, error) { } return node.Labels, nil } + +func (kvh *kubeletVolumeHost) GetExec(pluginName string) mount.Exec { + return mount.NewOsExec() +} diff --git a/pkg/kubelet/volumemanager/volume_manager.go b/pkg/kubelet/volumemanager/volume_manager.go index a71c05412f553..d48266f52a351 100644 --- a/pkg/kubelet/volumemanager/volume_manager.go +++ b/pkg/kubelet/volumemanager/volume_manager.go @@ -18,6 +18,7 @@ package volumemanager import ( "fmt" + "sort" "strconv" "time" @@ -286,15 +287,8 @@ func (vm *volumeManager) GetVolumesInUse() []v1.UniqueVolumeName { // volume *should* be attached to this node until it is safely unmounted. desiredVolumes := vm.desiredStateOfWorld.GetVolumesToMount() mountedVolumes := vm.actualStateOfWorld.GetGloballyMountedVolumes() - volumesToReportInUse := - make( - []v1.UniqueVolumeName, - 0, /* len */ - len(desiredVolumes)+len(mountedVolumes) /* cap */) - desiredVolumesMap := - make( - map[v1.UniqueVolumeName]bool, - len(desiredVolumes)+len(mountedVolumes) /* cap */) + volumesToReportInUse := make([]v1.UniqueVolumeName, 0, len(desiredVolumes)+len(mountedVolumes)) + desiredVolumesMap := make(map[v1.UniqueVolumeName]bool, len(desiredVolumes)+len(mountedVolumes)) for _, volume := range desiredVolumes { if volume.PluginIsAttachable { @@ -313,6 +307,9 @@ func (vm *volumeManager) GetVolumesInUse() []v1.UniqueVolumeName { } } + sort.Slice(volumesToReportInUse, func(i, j int) bool { + return string(volumesToReportInUse[i]) < string(volumesToReportInUse[j]) + }) return volumesToReportInUse } diff --git a/pkg/kubelet/volumemanager/volume_manager_test.go b/pkg/kubelet/volumemanager/volume_manager_test.go index 2648e0f2b0d96..6eb81cb7f74c0 100644 --- a/pkg/kubelet/volumemanager/volume_manager_test.go +++ b/pkg/kubelet/volumemanager/volume_manager_test.go @@ -217,7 +217,8 @@ func newTestVolumeManager(tmpDir string, podManager pod.Manager, kubeClient clie plug := &volumetest.FakeVolumePlugin{PluginName: "fake", Host: nil} fakeRecorder := &record.FakeRecorder{} plugMgr := &volume.VolumePluginMgr{} - plugMgr.InitPlugins([]volume.VolumePlugin{plug}, volumetest.NewFakeVolumeHost(tmpDir, kubeClient, nil)) + // TODO (#51147) inject mock prober + plugMgr.InitPlugins([]volume.VolumePlugin{plug}, nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, kubeClient, nil)) statusManager := status.NewManager(kubeClient, podManager, &statustest.FakePodDeletionSafetyProvider{}) vm := NewVolumeManager( diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index b5fa78188c283..db52f2f08886f 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -22,10 +22,9 @@ import ( clientset "k8s.io/client-go/kubernetes" kubeletapp "k8s.io/kubernetes/cmd/kubelet/app" "k8s.io/kubernetes/cmd/kubelet/app/options" - "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/kubelet" "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig" - "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" + kubeletv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" @@ -112,17 +111,16 @@ func GetHollowKubeletConfig( f := &options.KubeletFlags{ RootDirectory: testRootDir, HostnameOverride: nodeName, + CloudProvider: kubeletv1alpha1.AutoDetectCloudProvider, // Use the default runtime options. ContainerRuntimeOptions: *options.NewContainerRuntimeOptions(), } // Config struct - // Do the external -> internal conversion to make sure that defaults - // are set for fields not overridden in NewHollowKubelet. - tmp := &v1alpha1.KubeletConfiguration{} - api.Scheme.Default(tmp) - c := &kubeletconfig.KubeletConfiguration{} - api.Scheme.Convert(tmp, c, nil) + c, err := options.NewKubeletConfiguration() + if err != nil { + panic(err) + } c.ManifestURL = "" c.Address = "0.0.0.0" /* bind address */ diff --git a/pkg/master/OWNERS b/pkg/master/OWNERS index 739cc9ee46aae..afb28125c4dae 100644 --- a/pkg/master/OWNERS +++ b/pkg/master/OWNERS @@ -38,3 +38,4 @@ reviewers: - madhusudancs - hongchaodeng - jszczepkowski +- enj diff --git a/pkg/master/controller.go b/pkg/master/controller.go index 0e922e17b8257..e38718c6167a4 100644 --- a/pkg/master/controller.go +++ b/pkg/master/controller.go @@ -250,6 +250,7 @@ func (c *Controller) CreateOrUpdateMasterServiceIfNeeded(serviceName string, ser } return nil } + timeoutSeconds := api.DefaultClientIPServiceAffinitySeconds svc := &api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, @@ -263,6 +264,11 @@ func (c *Controller) CreateOrUpdateMasterServiceIfNeeded(serviceName string, ser ClusterIP: serviceIP.String(), SessionAffinity: api.ServiceAffinityClientIP, Type: serviceType, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, }, } diff --git a/pkg/master/controller_test.go b/pkg/master/controller_test.go index 50dc1bab734fe..f0011223347db 100644 --- a/pkg/master/controller_test.go +++ b/pkg/master/controller_test.go @@ -546,6 +546,7 @@ func TestCreateOrUpdateMasterService(t *testing.T) { om := func(name string) metav1.ObjectMeta { return metav1.ObjectMeta{Namespace: ns, Name: name} } + timeoutSeconds := api.DefaultClientIPServiceAffinitySeconds create_tests := []struct { testName string @@ -570,7 +571,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, }, @@ -625,7 +631,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, expectUpdate: &api.Service{ @@ -637,7 +648,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, }, @@ -658,7 +674,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, expectUpdate: &api.Service{ @@ -671,7 +692,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, }, @@ -691,7 +717,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, expectUpdate: &api.Service{ @@ -703,7 +734,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, }, @@ -723,7 +759,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, expectUpdate: &api.Service{ @@ -735,7 +776,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, }, @@ -755,7 +801,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, expectUpdate: &api.Service{ @@ -767,7 +818,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, }, @@ -787,7 +843,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, expectUpdate: &api.Service{ @@ -799,7 +860,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, }, @@ -819,7 +885,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeNodePort, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeNodePort, }, }, expectUpdate: &api.Service{ @@ -831,7 +902,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, }, @@ -851,7 +927,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, expectUpdate: nil, @@ -910,7 +991,12 @@ func TestCreateOrUpdateMasterService(t *testing.T) { Selector: nil, ClusterIP: "1.2.3.4", SessionAffinity: api.ServiceAffinityClientIP, - Type: api.ServiceTypeClusterIP, + SessionAffinityConfig: &api.SessionAffinityConfig{ + ClientIP: &api.ClientIPConfig{ + TimeoutSeconds: &timeoutSeconds, + }, + }, + Type: api.ServiceTypeClusterIP, }, }, expectUpdate: nil, diff --git a/pkg/master/master.go b/pkg/master/master.go index 97c5c5357bf80..d6f7130a07685 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -278,9 +278,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) m.installTunneler(c.Tunneler, corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig).Nodes()) } - if err := m.GenericAPIServer.AddPostStartHook("ca-registration", c.ClientCARegistrationHook.PostStartHook); err != nil { - glog.Fatalf("Error registering PostStartHook %q: %v", "ca-registration", err) - } + m.GenericAPIServer.AddPostStartHookOrDie("ca-registration", c.ClientCARegistrationHook.PostStartHook) return m, nil } @@ -294,9 +292,7 @@ func (m *Master) InstallLegacyAPI(c *Config, restOptionsGetter generic.RESTOptio if c.EnableCoreControllers { coreClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) bootstrapController := c.NewBootstrapController(legacyRESTStorage, coreClient, coreClient) - if err := m.GenericAPIServer.AddPostStartHook("bootstrap-controller", bootstrapController.PostStartHook); err != nil { - glog.Fatalf("Error registering PostStartHook %q: %v", "bootstrap-controller", err) - } + m.GenericAPIServer.AddPostStartHookOrDie("bootstrap-controller", bootstrapController.PostStartHook) } if err := m.GenericAPIServer.InstallLegacyAPIGroup(genericapiserver.DefaultLegacyAPIPrefix, &apiGroupInfo); err != nil { @@ -341,9 +337,7 @@ func (m *Master) InstallAPIs(apiResourceConfigSource serverstorage.APIResourceCo if err != nil { glog.Fatalf("Error building PostStartHook: %v", err) } - if err := m.GenericAPIServer.AddPostStartHook(name, hook); err != nil { - glog.Fatalf("Error registering PostStartHook %q: %v", name, err) - } + m.GenericAPIServer.AddPostStartHookOrDie(name, hook) } apiGroupsInfo = append(apiGroupsInfo, apiGroupInfo) diff --git a/pkg/printers/internalversion/BUILD b/pkg/printers/internalversion/BUILD index 6d5281bd171d1..fa0ad5d66b021 100644 --- a/pkg/printers/internalversion/BUILD +++ b/pkg/printers/internalversion/BUILD @@ -55,7 +55,6 @@ go_library( "printers.go", ], deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//federation/apis/federation:go_default_library", "//federation/apis/federation/v1beta1:go_default_library", "//federation/client/clientset_generated/federation_clientset:go_default_library", diff --git a/pkg/printers/internalversion/describe.go b/pkg/printers/internalversion/describe.go index 00e732dd4a297..94a3f39187f97 100644 --- a/pkg/printers/internalversion/describe.go +++ b/pkg/printers/internalversion/describe.go @@ -846,7 +846,8 @@ func printISCSIVolumeSource(iscsi *api.ISCSIVolumeSource, w PrefixWriter) { " DiscoveryCHAPAuth:\t%v\n"+ " SessionCHAPAuth:\t%v\n"+ " SecretRef:\t%v\n", - iscsi.TargetPortal, iscsi.IQN, iscsi.Lun, iscsi.ISCSIInterface, iscsi.FSType, iscsi.ReadOnly, iscsi.Portals, iscsi.DiscoveryCHAPAuth, iscsi.SessionCHAPAuth, iscsi.SecretRef) + " InitiatorName:\t%v\n", + iscsi.TargetPortal, iscsi.IQN, iscsi.Lun, iscsi.ISCSIInterface, iscsi.FSType, iscsi.ReadOnly, iscsi.Portals, iscsi.DiscoveryCHAPAuth, iscsi.SessionCHAPAuth, iscsi.SecretRef, iscsi.InitiatorName) } func printGlusterfsVolumeSource(glusterfs *api.GlusterfsVolumeSource, w PrefixWriter) { @@ -953,6 +954,17 @@ func printCephFSVolumeSource(cephfs *api.CephFSVolumeSource, w PrefixWriter) { cephfs.Monitors, cephfs.Path, cephfs.User, cephfs.SecretFile, cephfs.SecretRef, cephfs.ReadOnly) } +func printCephFSPersistentVolumeSource(cephfs *api.CephFSPersistentVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tCephFS (a CephFS mount on the host that shares a pod's lifetime)\n"+ + " Monitors:\t%v\n"+ + " Path:\t%v\n"+ + " User:\t%v\n"+ + " SecretFile:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n", + cephfs.Monitors, cephfs.Path, cephfs.User, cephfs.SecretFile, cephfs.SecretRef, cephfs.ReadOnly) +} + func printStorageOSVolumeSource(storageos *api.StorageOSVolumeSource, w PrefixWriter) { w.Write(LEVEL_2, "Type:\tStorageOS (a StorageOS Persistent Disk resource)\n"+ " VolumeName:\t%v\n"+ @@ -992,6 +1004,19 @@ func printAzureFileVolumeSource(azureFile *api.AzureFileVolumeSource, w PrefixWr azureFile.SecretName, azureFile.ShareName, azureFile.ReadOnly) } +func printAzureFilePersistentVolumeSource(azureFile *api.AzureFilePersistentVolumeSource, w PrefixWriter) { + ns := "" + if azureFile.SecretNamespace != nil { + ns = *azureFile.SecretNamespace + } + w.Write(LEVEL_2, "Type:\tAzureFile (an Azure File Service mount on the host and bind mount to the pod)\n"+ + " SecretName:\t%v\n"+ + " SecretNamespace:\t%v\n"+ + " ShareName:\t%v\n"+ + " ReadOnly:\t%v\n", + azureFile.SecretName, ns, azureFile.ShareName, azureFile.ReadOnly) +} + func printFlexVolumeSource(flex *api.FlexVolumeSource, w PrefixWriter) { w.Write(LEVEL_2, "Type:\tFlexVolume (a generic volume resource that is provisioned/attached using an exec based plugin)\n"+ " Driver:\t%v\n"+ @@ -1081,13 +1106,13 @@ func describePersistentVolume(pv *api.PersistentVolume, events *api.EventList) ( case pv.Spec.Local != nil: printLocalVolumeSource(pv.Spec.Local, w) case pv.Spec.CephFS != nil: - printCephFSVolumeSource(pv.Spec.CephFS, w) + printCephFSPersistentVolumeSource(pv.Spec.CephFS, w) case pv.Spec.StorageOS != nil: printStorageOSPersistentVolumeSource(pv.Spec.StorageOS, w) case pv.Spec.FC != nil: printFCVolumeSource(pv.Spec.FC, w) case pv.Spec.AzureFile != nil: - printAzureFileVolumeSource(pv.Spec.AzureFile, w) + printAzureFilePersistentVolumeSource(pv.Spec.AzureFile, w) case pv.Spec.FlexVolume != nil: printFlexVolumeSource(pv.Spec.FlexVolume, w) case pv.Spec.Flocker != nil: @@ -2173,10 +2198,15 @@ func (d *ServiceAccountDescriber) Describe(namespace, name string, describerSett } } - return describeServiceAccount(serviceAccount, tokens, missingSecrets) + var events *api.EventList + if describerSettings.ShowEvents { + events, _ = d.Core().Events(namespace).Search(api.Scheme, serviceAccount) + } + + return describeServiceAccount(serviceAccount, tokens, missingSecrets, events) } -func describeServiceAccount(serviceAccount *api.ServiceAccount, tokens []api.Secret, missingSecrets sets.String) (string, error) { +func describeServiceAccount(serviceAccount *api.ServiceAccount, tokens []api.Secret, missingSecrets sets.String, events *api.EventList) (string, error) { return tabbedString(func(out io.Writer) error { w := NewPrefixWriter(out) w.Write(LEVEL_0, "Name:\t%s\n", serviceAccount.Name) @@ -2228,6 +2258,10 @@ func describeServiceAccount(serviceAccount *api.ServiceAccount, tokens []api.Sec w.WriteLine() } + if events != nil { + DescribeEvents(events, w) + } + return nil }) } @@ -2436,7 +2470,11 @@ func describeNode(node *api.Node, nodeNonTerminatedPodsList *api.PodList, events return tabbedString(func(out io.Writer) error { w := NewPrefixWriter(out) w.Write(LEVEL_0, "Name:\t%s\n", node.Name) - w.Write(LEVEL_0, "Role:\t%s\n", findNodeRole(node)) + if roles := findNodeRoles(node); len(roles) > 0 { + w.Write(LEVEL_0, "Roles:\t%s\n", strings.Join(roles, ",")) + } else { + w.Write(LEVEL_0, "Roles:\t%s\n", "") + } printLabelsMultiline(w, "Labels", node.Labels) printAnnotationsMultiline(w, "Annotations", node.Annotations) printNodeTaintsMultiline(w, "Taints", node.Spec.Taints) diff --git a/pkg/printers/internalversion/printers.go b/pkg/printers/internalversion/printers.go index 3da77c4583cf1..5283c82d80785 100644 --- a/pkg/printers/internalversion/printers.go +++ b/pkg/printers/internalversion/printers.go @@ -41,7 +41,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/federation/apis/federation" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/events" @@ -60,7 +59,16 @@ import ( "k8s.io/kubernetes/pkg/util/node" ) -const loadBalancerWidth = 16 +const ( + loadBalancerWidth = 16 + + // labelNodeRolePrefix is a label prefix for node roles + // It's copied over to here until it's merged in core: https://github.com/kubernetes/kubernetes/pull/39112 + labelNodeRolePrefix = "node-role.kubernetes.io/" + + // nodeLabelRole specifies the role of a node + nodeLabelRole = "kubernetes.io/role" +) // AddHandlers adds print handlers for default Kubernetes types dealing with internal versions. // TODO: handle errors from Handler @@ -209,6 +217,7 @@ func AddHandlers(h printers.PrintHandler) { nodeColumnDefinitions := []metav1alpha1.TableColumnDefinition{ {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, {Name: "Status", Type: "string", Description: "The status of the node"}, + {Name: "Roles", Type: "string", Description: "The roles of the node"}, {Name: "Age", Type: "string", Description: metav1.ObjectMeta{}.SwaggerDoc()["creationTimestamp"]}, {Name: "Version", Type: "string", Description: apiv1.NodeSystemInfo{}.SwaggerDoc()["kubeletVersion"]}, {Name: "External-IP", Type: "string", Priority: 1, Description: apiv1.NodeStatus{}.SwaggerDoc()["addresses"]}, @@ -277,6 +286,7 @@ func AddHandlers(h printers.PrintHandler) { persistentVolumeClaimColumnDefinitions := []metav1alpha1.TableColumnDefinition{ {Name: "Name", Type: "string", Format: "name", Description: metav1.ObjectMeta{}.SwaggerDoc()["name"]}, {Name: "Status", Type: "string", Description: apiv1.PersistentVolumeClaimStatus{}.SwaggerDoc()["phase"]}, + {Name: "Volume", Type: "string", Description: apiv1.PersistentVolumeSpec{}.SwaggerDoc()["volumeName"]}, {Name: "Capacity", Type: "string", Description: apiv1.PersistentVolumeClaimStatus{}.SwaggerDoc()["capacity"]}, {Name: "Access Modes", Type: "string", Description: apiv1.PersistentVolumeClaimStatus{}.SwaggerDoc()["accessModes"]}, {Name: "StorageClass", Type: "string", Description: "StorageClass of the pvc"}, @@ -1155,12 +1165,13 @@ func printNode(obj *api.Node, options printers.PrintOptions) ([]metav1alpha1.Tab if obj.Spec.Unschedulable { status = append(status, "SchedulingDisabled") } - role := findNodeRole(obj) - if role != "" { - status = append(status, role) + + roles := strings.Join(findNodeRoles(obj), ",") + if len(roles) == 0 { + roles = "" } - row.Cells = append(row.Cells, obj.Name, strings.Join(status, ","), translateTimestamp(obj.CreationTimestamp), obj.Status.NodeInfo.KubeletVersion) + row.Cells = append(row.Cells, obj.Name, strings.Join(status, ","), roles, translateTimestamp(obj.CreationTimestamp), obj.Status.NodeInfo.KubeletVersion) if options.Wide { osImage, kernelVersion, crVersion := obj.Status.NodeInfo.OSImage, obj.Status.NodeInfo.KernelVersion, obj.Status.NodeInfo.ContainerRuntimeVersion if osImage == "" { @@ -1189,17 +1200,24 @@ func getNodeExternalIP(node *api.Node) string { return "" } -// findNodeRole returns the role of a given node, or "" if none found. -// The role is determined by looking in order for: -// * a kubernetes.io/role label -// * a kubeadm.alpha.kubernetes.io/role label -// If no role is found, ("", nil) is returned -func findNodeRole(node *api.Node) string { - if role := node.Labels[kubeadm.NodeLabelKubeadmAlphaRole]; role != "" { - return role +// findNodeRoles returns the roles of a given node. +// The roles are determined by looking for: +// * a node-role.kubernetes.io/="" label +// * a kubernetes.io/role="" label +func findNodeRoles(node *api.Node) []string { + roles := sets.NewString() + for k, v := range node.Labels { + switch { + case strings.HasPrefix(k, labelNodeRolePrefix): + if role := strings.TrimPrefix(k, labelNodeRolePrefix); len(role) > 0 { + roles.Insert(role) + } + + case k == nodeLabelRole && v != "": + roles.Insert(v) + } } - // No role found - return "" + return roles.List() } func printNodeList(list *api.NodeList, options printers.PrintOptions) ([]metav1alpha1.TableRow, error) { diff --git a/pkg/printers/internalversion/printers_test.go b/pkg/printers/internalversion/printers_test.go index 701e9678f2a19..ababeaca3bf05 100644 --- a/pkg/printers/internalversion/printers_test.go +++ b/pkg/printers/internalversion/printers_test.go @@ -825,15 +825,50 @@ func TestPrintNodeStatus(t *testing.T) { }, status: "Unknown,SchedulingDisabled", }, + } + + for _, test := range table { + buffer := &bytes.Buffer{} + err := printer.PrintObj(&test.node, buffer) + if err != nil { + t.Fatalf("An error occurred printing Node: %#v", err) + } + if !contains(strings.Fields(buffer.String()), test.status) { + t.Fatalf("Expect printing node %s with status %#v, got: %#v", test.node.Name, test.status, buffer.String()) + } + } +} + +func TestPrintNodeRole(t *testing.T) { + printer := printers.NewHumanReadablePrinter(nil, nil, printers.PrintOptions{}) + AddHandlers(printer) + table := []struct { + node api.Node + expected string + }{ + { + node: api.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "foo9"}, + }, + expected: "", + }, { node: api.Node{ ObjectMeta: metav1.ObjectMeta{ - Name: "foo12", - Labels: map[string]string{"kubeadm.alpha.kubernetes.io/role": "node"}, + Name: "foo10", + Labels: map[string]string{"node-role.kubernetes.io/master": "", "node-role.kubernetes.io/proxy": "", "kubernetes.io/role": "node"}, }, - Status: api.NodeStatus{Conditions: []api.NodeCondition{{Type: api.NodeReady, Status: api.ConditionTrue}}}, }, - status: "Ready,node", + expected: "master,node,proxy", + }, + { + node: api.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo11", + Labels: map[string]string{"kubernetes.io/role": "node"}, + }, + }, + expected: "node", }, } @@ -843,8 +878,8 @@ func TestPrintNodeStatus(t *testing.T) { if err != nil { t.Fatalf("An error occurred printing Node: %#v", err) } - if !contains(strings.Fields(buffer.String()), test.status) { - t.Fatalf("Expect printing node %s with status %#v, got: %#v", test.node.Name, test.status, buffer.String()) + if !contains(strings.Fields(buffer.String()), test.expected) { + t.Fatalf("Expect printing node %s with role %#v, got: %#v", test.node.Name, test.expected, buffer.String()) } } } @@ -2882,3 +2917,103 @@ func TestPrintReplicaSet(t *testing.T) { buf.Reset() } } + +func TestPrintPersistentVolumeClaim(t *testing.T) { + myScn := "my-scn" + tests := []struct { + pvc api.PersistentVolumeClaim + expect string + }{ + { + // Test name, num of containers, restarts, container ready status + api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test1", + }, + Spec: api.PersistentVolumeClaimSpec{ + VolumeName: "my-volume", + }, + Status: api.PersistentVolumeClaimStatus{ + Phase: api.ClaimBound, + AccessModes: []api.PersistentVolumeAccessMode{api.ReadOnlyMany}, + Capacity: map[api.ResourceName]resource.Quantity{ + api.ResourceStorage: resource.MustParse("4Gi"), + }, + }, + }, + "test1\tBound\tmy-volume\t4Gi\tROX\t\t\n", + }, + { + // Test name, num of containers, restarts, container ready status + api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test2", + }, + Spec: api.PersistentVolumeClaimSpec{}, + Status: api.PersistentVolumeClaimStatus{ + Phase: api.ClaimLost, + AccessModes: []api.PersistentVolumeAccessMode{api.ReadOnlyMany}, + Capacity: map[api.ResourceName]resource.Quantity{ + api.ResourceStorage: resource.MustParse("4Gi"), + }, + }, + }, + "test2\tLost\t\t\t\t\t\n", + }, + { + // Test name, num of containers, restarts, container ready status + api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test3", + }, + Spec: api.PersistentVolumeClaimSpec{ + VolumeName: "my-volume", + }, + Status: api.PersistentVolumeClaimStatus{ + Phase: api.ClaimPending, + AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteMany}, + Capacity: map[api.ResourceName]resource.Quantity{ + api.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + "test3\tPending\tmy-volume\t10Gi\tRWX\t\t\n", + }, + { + // Test name, num of containers, restarts, container ready status + api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test4", + }, + Spec: api.PersistentVolumeClaimSpec{ + VolumeName: "my-volume", + StorageClassName: &myScn, + }, + Status: api.PersistentVolumeClaimStatus{ + Phase: api.ClaimPending, + AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, + Capacity: map[api.ResourceName]resource.Quantity{ + api.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + "test4\tPending\tmy-volume\t10Gi\tRWO\tmy-scn\t\n", + }, + } + buf := bytes.NewBuffer([]byte{}) + for _, test := range tests { + table, err := printers.NewTablePrinter().With(AddHandlers).PrintTable(&test.pvc, printers.PrintOptions{}) + if err != nil { + t.Fatal(err) + } + if err := printers.PrintTable(table, buf, printers.PrintOptions{NoHeaders: true}); err != nil { + t.Fatal(err) + } + if buf.String() != test.expect { + fmt.Println(buf.String()) + fmt.Println(test.expect) + t.Fatalf("Expected: %s, but got: %s", test.expect, buf.String()) + } + buf.Reset() + } +} diff --git a/pkg/proxy/iptables/BUILD b/pkg/proxy/iptables/BUILD index e6c45650b382e..ac49fb2093e47 100644 --- a/pkg/proxy/iptables/BUILD +++ b/pkg/proxy/iptables/BUILD @@ -43,6 +43,7 @@ go_test( deps = [ "//pkg/api:go_default_library", "//pkg/proxy:go_default_library", + "//pkg/proxy/util:go_default_library", "//pkg/util/async:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/iptables/testing:go_default_library", diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 1c497df967368..f745833268987 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -143,7 +143,7 @@ type serviceInfo struct { nodePort int loadBalancerStatus api.LoadBalancerStatus sessionAffinityType api.ServiceAffinity - stickyMaxAgeMinutes int + stickyMaxAgeSeconds int externalIPs []string loadBalancerSourceRanges []string onlyNodeLocalEndpoints bool @@ -194,6 +194,10 @@ func newServiceInfo(svcPortName proxy.ServicePortName, port *api.ServicePort, se apiservice.RequestsOnlyLocalTraffic(service) { onlyNodeLocalEndpoints = true } + var stickyMaxAgeSeconds int + if service.Spec.SessionAffinity == api.ServiceAffinityClientIP { + stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds) + } info := &serviceInfo{ clusterIP: net.ParseIP(service.Spec.ClusterIP), port: int(port.Port), @@ -202,11 +206,12 @@ func newServiceInfo(svcPortName proxy.ServicePortName, port *api.ServicePort, se // Deep-copy in case the service instance changes loadBalancerStatus: *helper.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer), sessionAffinityType: service.Spec.SessionAffinity, - stickyMaxAgeMinutes: 180, // TODO: paramaterize this in the API. + stickyMaxAgeSeconds: stickyMaxAgeSeconds, externalIPs: make([]string, len(service.Spec.ExternalIPs)), loadBalancerSourceRanges: make([]string, len(service.Spec.LoadBalancerSourceRanges)), onlyNodeLocalEndpoints: onlyNodeLocalEndpoints, } + copy(info.loadBalancerSourceRanges, service.Spec.LoadBalancerSourceRanges) copy(info.externalIPs, service.Spec.ExternalIPs) @@ -369,7 +374,7 @@ type Proxier struct { mu sync.Mutex // protects the following fields serviceMap proxyServiceMap endpointsMap proxyEndpointsMap - portsMap map[localPort]closeable + portsMap map[utilproxy.LocalPort]utilproxy.Closeable // endpointsSynced and servicesSynced are set to true when corresponding // objects are synced after startup. This is used to avoid updating iptables // with some partial data after kube-proxy restart. @@ -386,7 +391,7 @@ type Proxier struct { clusterCIDR string hostname string nodeIP net.IP - portMapper portOpener + portMapper utilproxy.PortOpener recorder record.EventRecorder healthChecker healthcheck.Server healthzServer healthcheck.HealthzUpdater @@ -405,32 +410,11 @@ type Proxier struct { natRules *bytes.Buffer } -type localPort struct { - desc string - ip string - port int - protocol string -} - -func (lp *localPort) String() string { - return fmt.Sprintf("%q (%s:%d/%s)", lp.desc, lp.ip, lp.port, lp.protocol) -} - -type closeable interface { - Close() error -} - -// portOpener is an interface around port opening/closing. -// Abstracted out for testing. -type portOpener interface { - OpenLocalPort(lp *localPort) (closeable, error) -} - // listenPortOpener opens ports by calling bind() and listen(). type listenPortOpener struct{} // OpenLocalPort holds the given local port open. -func (l *listenPortOpener) OpenLocalPort(lp *localPort) (closeable, error) { +func (l *listenPortOpener) OpenLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { return openLocalPort(lp) } @@ -491,7 +475,7 @@ func NewProxier(ipt utiliptables.Interface, healthChecker := healthcheck.NewServer(hostname, recorder, nil, nil) // use default implementations of deps proxier := &Proxier{ - portsMap: make(map[localPort]closeable), + portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable), serviceMap: make(proxyServiceMap), serviceChanges: newServiceChangeMap(), endpointsMap: make(proxyEndpointsMap), @@ -687,20 +671,6 @@ func (proxier *Proxier) OnServiceSynced() { proxier.syncProxyRules() } -func shouldSkipService(svcName types.NamespacedName, service *api.Service) bool { - // if ClusterIP is "None" or empty, skip proxying - if !helper.IsServiceIPSet(service) { - glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) - return true - } - // Even if ClusterIP is set, ServiceTypeExternalName services don't get proxied - if service.Spec.Type == api.ServiceTypeExternalName { - glog.V(3).Infof("Skipping service %s due to Type=ExternalName", svcName) - return true - } - return false -} - // is updated by this function (based on the given changes). // map is cleared after applying them. func updateServiceMap( @@ -894,7 +864,7 @@ func serviceToServiceMap(service *api.Service) proxyServiceMap { return nil } svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} - if shouldSkipService(svcName, service) { + if utilproxy.ShouldSkipService(svcName, service) { return nil } @@ -1140,7 +1110,7 @@ func (proxier *Proxier) syncProxyRules() { activeNATChains := map[utiliptables.Chain]bool{} // use a map as a set // Accumulate the set of local ports that we will be holding open once this update is complete - replacementPortsMap := map[localPort]closeable{} + replacementPortsMap := map[utilproxy.LocalPort]utilproxy.Closeable{} // We are creating those slices ones here to avoid memory reallocations // in every loop. Note that reuse the memory, instead of doing: @@ -1211,14 +1181,14 @@ func (proxier *Proxier) syncProxyRules() { // If the "external" IP happens to be an IP that is local to this // machine, hold the local port open so no other process can open it // (because the socket might open but it would never work). - if local, err := isLocalIP(externalIP); err != nil { + if local, err := utilproxy.IsLocalIP(externalIP); err != nil { glog.Errorf("can't determine if IP is local, assuming not: %v", err) } else if local { - lp := localPort{ - desc: "externalIP for " + svcNameString, - ip: externalIP, - port: svcInfo.port, - protocol: protocol, + lp := utilproxy.LocalPort{ + Description: "externalIP for " + svcNameString, + IP: externalIP, + Port: svcInfo.port, + Protocol: protocol, } if proxier.portsMap[lp] != nil { glog.V(4).Infof("Port %s was open before and is still needed", lp.String()) @@ -1351,11 +1321,11 @@ func (proxier *Proxier) syncProxyRules() { if svcInfo.nodePort != 0 { // Hold the local port open so no other process can open it // (because the socket might open but it would never work). - lp := localPort{ - desc: "nodePort for " + svcNameString, - ip: "", - port: svcInfo.nodePort, - protocol: protocol, + lp := utilproxy.LocalPort{ + Description: "nodePort for " + svcNameString, + IP: "", + Port: svcInfo.nodePort, + Protocol: protocol, } if proxier.portsMap[lp] != nil { glog.V(4).Infof("Port %s was open before and is still needed", lp.String()) @@ -1366,14 +1336,14 @@ func (proxier *Proxier) syncProxyRules() { glog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) continue } - if lp.protocol == "udp" { + if lp.Protocol == "udp" { // TODO: We might have multiple services using the same port, and this will clear conntrack for all of them. // This is very low impact. The NodePort range is intentionally obscure, and unlikely to actually collide with real Services. // This only affects UDP connections, which are not common. // See issue: https://github.com/kubernetes/kubernetes/issues/49881 - err := utilproxy.ClearUDPConntrackForPort(proxier.exec, lp.port) + err := utilproxy.ClearUDPConntrackForPort(proxier.exec, lp.Port) if err != nil { - glog.Errorf("Failed to clear udp conntrack for port %d, error: %v", lp.port, err) + glog.Errorf("Failed to clear udp conntrack for port %d, error: %v", lp.Port, err) } } replacementPortsMap[lp] = socket @@ -1454,7 +1424,7 @@ func (proxier *Proxier) syncProxyRules() { "-A", string(svcChain), "-m", "comment", "--comment", svcNameString, "-m", "recent", "--name", string(endpointChain), - "--rcheck", "--seconds", strconv.Itoa(svcInfo.stickyMaxAgeMinutes*60), "--reap", + "--rcheck", "--seconds", strconv.Itoa(svcInfo.stickyMaxAgeSeconds), "--reap", "-j", string(endpointChain)) } } @@ -1615,7 +1585,8 @@ func (proxier *Proxier) syncProxyRules() { } // Revert new local ports. - revertPorts(replacementPortsMap, proxier.portsMap) + glog.V(2).Infof("Closing local ports after iptables-restore failure") + utilproxy.RevertPorts(replacementPortsMap, proxier.portsMap) return } @@ -1665,24 +1636,7 @@ func writeLine(buf *bytes.Buffer, words ...string) { } } -func isLocalIP(ip string) (bool, error) { - addrs, err := net.InterfaceAddrs() - if err != nil { - return false, err - } - for i := range addrs { - intf, _, err := net.ParseCIDR(addrs[i].String()) - if err != nil { - return false, err - } - if net.ParseIP(ip).Equal(intf) { - return true, nil - } - } - return false, nil -} - -func openLocalPort(lp *localPort) (closeable, error) { +func openLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { // For ports on node IPs, open the actual port and hold it, even though we // use iptables to redirect traffic. // This ensures a) that it's safe to use that port and b) that (a) stays @@ -1695,16 +1649,16 @@ func openLocalPort(lp *localPort) (closeable, error) { // it. Tools like 'ss' and 'netstat' do not show sockets that are // bind()ed but not listen()ed, and at least the default debian netcat // has no way to avoid about 10 seconds of retries. - var socket closeable - switch lp.protocol { + var socket utilproxy.Closeable + switch lp.Protocol { case "tcp": - listener, err := net.Listen("tcp", net.JoinHostPort(lp.ip, strconv.Itoa(lp.port))) + listener, err := net.Listen("tcp", net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port))) if err != nil { return nil, err } socket = listener case "udp": - addr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(lp.ip, strconv.Itoa(lp.port))) + addr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port))) if err != nil { return nil, err } @@ -1714,20 +1668,8 @@ func openLocalPort(lp *localPort) (closeable, error) { } socket = conn default: - return nil, fmt.Errorf("unknown protocol %q", lp.protocol) + return nil, fmt.Errorf("unknown protocol %q", lp.Protocol) } glog.V(2).Infof("Opened local port %s", lp.String()) return socket, nil } - -// revertPorts is closing ports in replacementPortsMap but not in originalPortsMap. In other words, it only -// closes the ports opened in this sync. -func revertPorts(replacementPortsMap, originalPortsMap map[localPort]closeable) { - for k, v := range replacementPortsMap { - // Only close newly opened local ports - leave ones that were open before this update - if originalPortsMap[k] == nil { - glog.V(2).Infof("Closing local port %s after iptables-restore failure", k.String()) - v.Close() - } - } -} diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 35f288c53b370..ee900e3f0d14d 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/proxy" + utilproxy "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/util/async" utiliptables "k8s.io/kubernetes/pkg/util/iptables" iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing" @@ -178,8 +179,8 @@ func TestGetChainLinesMultipleTables(t *testing.T) { func newFakeServiceInfo(service proxy.ServicePortName, ip net.IP, port int, protocol api.Protocol, onlyNodeLocalEndpoints bool) *serviceInfo { return &serviceInfo{ - sessionAffinityType: api.ServiceAffinityNone, // default - stickyMaxAgeMinutes: 180, // TODO: paramaterize this in the API. + sessionAffinityType: api.ServiceAffinityNone, // default + stickyMaxAgeSeconds: int(api.DefaultClientIPServiceAffinitySeconds), // default clusterIP: ip, port: port, protocol: protocol, @@ -258,103 +259,14 @@ func (c *fakeClosable) Close() error { return nil } -func TestRevertPorts(t *testing.T) { - testCases := []struct { - replacementPorts []localPort - existingPorts []localPort - expectToBeClose []bool - }{ - { - replacementPorts: []localPort{ - {port: 5001}, - {port: 5002}, - {port: 5003}, - }, - existingPorts: []localPort{}, - expectToBeClose: []bool{true, true, true}, - }, - { - replacementPorts: []localPort{}, - existingPorts: []localPort{ - {port: 5001}, - {port: 5002}, - {port: 5003}, - }, - expectToBeClose: []bool{}, - }, - { - replacementPorts: []localPort{ - {port: 5001}, - {port: 5002}, - {port: 5003}, - }, - existingPorts: []localPort{ - {port: 5001}, - {port: 5002}, - {port: 5003}, - }, - expectToBeClose: []bool{false, false, false}, - }, - { - replacementPorts: []localPort{ - {port: 5001}, - {port: 5002}, - {port: 5003}, - }, - existingPorts: []localPort{ - {port: 5001}, - {port: 5003}, - }, - expectToBeClose: []bool{false, true, false}, - }, - { - replacementPorts: []localPort{ - {port: 5001}, - {port: 5002}, - {port: 5003}, - }, - existingPorts: []localPort{ - {port: 5001}, - {port: 5002}, - {port: 5003}, - {port: 5004}, - }, - expectToBeClose: []bool{false, false, false}, - }, - } - - for i, tc := range testCases { - replacementPortsMap := make(map[localPort]closeable) - for _, lp := range tc.replacementPorts { - replacementPortsMap[lp] = &fakeClosable{} - } - existingPortsMap := make(map[localPort]closeable) - for _, lp := range tc.existingPorts { - existingPortsMap[lp] = &fakeClosable{} - } - revertPorts(replacementPortsMap, existingPortsMap) - for j, expectation := range tc.expectToBeClose { - if replacementPortsMap[tc.replacementPorts[j]].(*fakeClosable).closed != expectation { - t.Errorf("Expect replacement localport %v to be %v in test case %v", tc.replacementPorts[j], expectation, i) - } - } - for _, lp := range tc.existingPorts { - if existingPortsMap[lp].(*fakeClosable).closed == true { - t.Errorf("Expect existing localport %v to be false in test case %v", lp, i) - } - } - } - -} - // fakePortOpener implements portOpener. type fakePortOpener struct { - openPorts []*localPort + openPorts []*utilproxy.LocalPort } // OpenLocalPort fakes out the listen() and bind() used by syncProxyRules // to lock a local port. -func (f *fakePortOpener) OpenLocalPort(lp *localPort) (closeable, error) { +func (f *fakePortOpener) OpenLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { f.openPorts = append(f.openPorts, lp) return nil, nil } @@ -395,8 +307,8 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier { iptables: ipt, clusterCIDR: "10.0.0.0/24", hostname: testHostname, - portsMap: make(map[localPort]closeable), - portMapper: &fakePortOpener{[]*localPort{}}, + portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable), + portMapper: &fakePortOpener{[]*utilproxy.LocalPort{}}, healthChecker: newFakeHealthChecker(), precomputedProbabilities: make([]string, 0, 1001), iptablesData: bytes.NewBuffer(nil), diff --git a/pkg/proxy/userspace/loadbalancer.go b/pkg/proxy/userspace/loadbalancer.go index 2bf23d5b940f0..369094e3296c4 100644 --- a/pkg/proxy/userspace/loadbalancer.go +++ b/pkg/proxy/userspace/loadbalancer.go @@ -28,7 +28,7 @@ type LoadBalancer interface { // NextEndpoint returns the endpoint to handle a request for the given // service-port and source address. NextEndpoint(service proxy.ServicePortName, srcAddr net.Addr, sessionAffinityReset bool) (string, error) - NewService(service proxy.ServicePortName, sessionAffinityType api.ServiceAffinity, stickyMaxAgeMinutes int) error + NewService(service proxy.ServicePortName, sessionAffinityType api.ServiceAffinity, stickyMaxAgeSeconds int) error DeleteService(service proxy.ServicePortName) CleanupStaleStickySessions(service proxy.ServicePortName) ServiceHasEndpoints(service proxy.ServicePortName) bool diff --git a/pkg/proxy/userspace/proxier.go b/pkg/proxy/userspace/proxier.go index 846915f7fbd0f..d299544da2e3a 100644 --- a/pkg/proxy/userspace/proxier.go +++ b/pkg/proxy/userspace/proxier.go @@ -61,7 +61,7 @@ type ServiceInfo struct { nodePort int loadBalancerStatus api.LoadBalancerStatus sessionAffinityType api.ServiceAffinity - stickyMaxAgeMinutes int + stickyMaxAgeSeconds int // Deprecated, but required for back-compat (including e2e) externalIPs []string } @@ -378,15 +378,13 @@ func (proxier *Proxier) addServiceOnPort(service proxy.ServicePortName, protocol return nil, err } si := &ServiceInfo{ - Timeout: timeout, - ActiveClients: newClientCache(), - + Timeout: timeout, + ActiveClients: newClientCache(), isAliveAtomic: 1, proxyPort: portNum, protocol: protocol, socket: sock, sessionAffinityType: api.ServiceAffinityNone, // default - stickyMaxAgeMinutes: 180, // TODO: parameterize this in the API. } proxier.setServiceInfo(service, si) @@ -450,12 +448,17 @@ func (proxier *Proxier) mergeService(service *api.Service) sets.String { info.loadBalancerStatus = *helper.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer) info.nodePort = int(servicePort.NodePort) info.sessionAffinityType = service.Spec.SessionAffinity + // Set session affinity timeout value when sessionAffinity==ClientIP + if service.Spec.SessionAffinity == api.ServiceAffinityClientIP { + info.stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds) + } + glog.V(4).Infof("info: %#v", info) if err := proxier.openPortal(serviceName, info); err != nil { glog.Errorf("Failed to open portal for %q: %v", serviceName, err) } - proxier.loadBalancer.NewService(serviceName, info.sessionAffinityType, info.stickyMaxAgeMinutes) + proxier.loadBalancer.NewService(serviceName, info.sessionAffinityType, info.stickyMaxAgeSeconds) } return existingPorts @@ -583,7 +586,7 @@ func (proxier *Proxier) openPortal(service proxy.ServicePortName, info *ServiceI } func (proxier *Proxier) openOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) error { - if local, err := isLocalIP(portal.ip); err != nil { + if local, err := utilproxy.IsLocalIP(portal.ip.String()); err != nil { return fmt.Errorf("can't determine if IP %s is local, assuming not: %v", portal.ip, err) } else if local { err := proxier.claimNodePort(portal.ip, portal.port, protocol, name) @@ -761,7 +764,7 @@ func (proxier *Proxier) closePortal(service proxy.ServicePortName, info *Service func (proxier *Proxier) closeOnePortal(portal portal, protocol api.Protocol, proxyIP net.IP, proxyPort int, name proxy.ServicePortName) []error { el := []error{} - if local, err := isLocalIP(portal.ip); err != nil { + if local, err := utilproxy.IsLocalIP(portal.ip.String()); err != nil { el = append(el, fmt.Errorf("can't determine if IP %s is local, assuming not: %v", portal.ip, err)) } else if local { if err := proxier.releaseNodePort(portal.ip, portal.port, protocol, name); err != nil { @@ -832,23 +835,6 @@ func (proxier *Proxier) closeNodePort(nodePort int, protocol api.Protocol, proxy return el } -func isLocalIP(ip net.IP) (bool, error) { - addrs, err := net.InterfaceAddrs() - if err != nil { - return false, err - } - for i := range addrs { - intf, _, err := net.ParseCIDR(addrs[i].String()) - if err != nil { - return false, err - } - if ip.Equal(intf) { - return true, nil - } - } - return false, nil -} - // See comments in the *PortalArgs() functions for some details about why we // use two chains for portals. var iptablesContainerPortalChain iptables.Chain = "KUBE-PORTALS-CONTAINER" diff --git a/pkg/proxy/userspace/roundrobin.go b/pkg/proxy/userspace/roundrobin.go index b610ebbbd7603..b32d69d8776a7 100644 --- a/pkg/proxy/userspace/roundrobin.go +++ b/pkg/proxy/userspace/roundrobin.go @@ -48,7 +48,7 @@ type affinityState struct { type affinityPolicy struct { affinityType api.ServiceAffinity affinityMap map[string]*affinityState // map client IP -> affinity info - ttlMinutes int + ttlSeconds int } // LoadBalancerRR is a round-robin load balancer. @@ -66,11 +66,11 @@ type balancerState struct { affinity affinityPolicy } -func newAffinityPolicy(affinityType api.ServiceAffinity, ttlMinutes int) *affinityPolicy { +func newAffinityPolicy(affinityType api.ServiceAffinity, ttlSeconds int) *affinityPolicy { return &affinityPolicy{ affinityType: affinityType, affinityMap: make(map[string]*affinityState), - ttlMinutes: ttlMinutes, + ttlSeconds: ttlSeconds, } } @@ -81,22 +81,22 @@ func NewLoadBalancerRR() *LoadBalancerRR { } } -func (lb *LoadBalancerRR) NewService(svcPort proxy.ServicePortName, affinityType api.ServiceAffinity, ttlMinutes int) error { +func (lb *LoadBalancerRR) NewService(svcPort proxy.ServicePortName, affinityType api.ServiceAffinity, ttlSeconds int) error { glog.V(4).Infof("LoadBalancerRR NewService %q", svcPort) lb.lock.Lock() defer lb.lock.Unlock() - lb.newServiceInternal(svcPort, affinityType, ttlMinutes) + lb.newServiceInternal(svcPort, affinityType, ttlSeconds) return nil } // This assumes that lb.lock is already held. -func (lb *LoadBalancerRR) newServiceInternal(svcPort proxy.ServicePortName, affinityType api.ServiceAffinity, ttlMinutes int) *balancerState { - if ttlMinutes == 0 { - ttlMinutes = 180 //default to 3 hours if not specified. Should 0 be unlimited instead???? +func (lb *LoadBalancerRR) newServiceInternal(svcPort proxy.ServicePortName, affinityType api.ServiceAffinity, ttlSeconds int) *balancerState { + if ttlSeconds == 0 { + ttlSeconds = int(api.DefaultClientIPServiceAffinitySeconds) //default to 3 hours if not specified. Should 0 be unlimited instead???? } if _, exists := lb.services[svcPort]; !exists { - lb.services[svcPort] = &balancerState{affinity: *newAffinityPolicy(affinityType, ttlMinutes)} + lb.services[svcPort] = &balancerState{affinity: *newAffinityPolicy(affinityType, ttlSeconds)} glog.V(4).Infof("LoadBalancerRR service %q did not exist, created", svcPort) } else if affinityType != "" { lb.services[svcPort].affinity.affinityType = affinityType @@ -159,7 +159,7 @@ func (lb *LoadBalancerRR) NextEndpoint(svcPort proxy.ServicePortName, srcAddr ne } if !sessionAffinityReset { sessionAffinity, exists := state.affinity.affinityMap[ipaddr] - if exists && int(time.Now().Sub(sessionAffinity.lastUsed).Minutes()) < state.affinity.ttlMinutes { + if exists && int(time.Now().Sub(sessionAffinity.lastUsed).Seconds()) < state.affinity.ttlSeconds { // Affinity wins. endpoint := sessionAffinity.endpoint sessionAffinity.lastUsed = time.Now() @@ -378,7 +378,7 @@ func (lb *LoadBalancerRR) CleanupStaleStickySessions(svcPort proxy.ServicePortNa return } for ip, affinity := range state.affinity.affinityMap { - if int(time.Now().Sub(affinity.lastUsed).Minutes()) >= state.affinity.ttlMinutes { + if int(time.Now().Sub(affinity.lastUsed).Seconds()) >= state.affinity.ttlSeconds { glog.V(4).Infof("Removing client %s from affinityMap for service %q", affinity.clientIP, svcPort) delete(state.affinity.affinityMap, ip) } diff --git a/pkg/proxy/userspace/roundrobin_test.go b/pkg/proxy/userspace/roundrobin_test.go index de3a64de77bc8..05ccbd37a817e 100644 --- a/pkg/proxy/userspace/roundrobin_test.go +++ b/pkg/proxy/userspace/roundrobin_test.go @@ -357,7 +357,7 @@ func TestStickyLoadBalanceWorksWithNewServiceCalledFirst(t *testing.T) { } // Call NewService() before OnEndpointsUpdate() - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpoints := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: []api.EndpointSubset{ @@ -421,7 +421,7 @@ func TestStickyLoadBalanceWorksWithNewServiceCalledSecond(t *testing.T) { }, } loadBalancer.OnEndpointsAdd(endpoints) - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) client1 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} client2 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 2), Port: 0} @@ -473,7 +473,7 @@ func TestStickyLoadBalanaceWorksWithMultipleEndpointsRemoveOne(t *testing.T) { t.Errorf("Didn't fail with non-existent service") } - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpointsv1 := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: []api.EndpointSubset{ @@ -546,7 +546,7 @@ func TestStickyLoadBalanceWorksWithMultipleEndpointsAndUpdates(t *testing.T) { t.Errorf("Didn't fail with non-existent service") } - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpointsv1 := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: []api.EndpointSubset{ @@ -605,7 +605,7 @@ func TestStickyLoadBalanceWorksWithServiceRemoval(t *testing.T) { if err == nil || len(endpoint) != 0 { t.Errorf("Didn't fail with non-existent service") } - loadBalancer.NewService(fooService, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(fooService, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpoints1 := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: fooService.Name, Namespace: fooService.Namespace}, Subsets: []api.EndpointSubset{ @@ -616,7 +616,7 @@ func TestStickyLoadBalanceWorksWithServiceRemoval(t *testing.T) { }, } barService := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: "testnamespace", Name: "bar"}, Port: ""} - loadBalancer.NewService(barService, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(barService, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpoints2 := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: barService.Name, Namespace: barService.Namespace}, Subsets: []api.EndpointSubset{ @@ -674,7 +674,7 @@ func TestStickyLoadBalanceWorksWithEndpointFails(t *testing.T) { } // Call NewService() before OnEndpointsUpdate() - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpoints := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: []api.EndpointSubset{ diff --git a/pkg/proxy/util/BUILD b/pkg/proxy/util/BUILD index 01c12f6e8d522..0fb554dc0cf7f 100644 --- a/pkg/proxy/util/BUILD +++ b/pkg/proxy/util/BUILD @@ -2,16 +2,33 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["conntrack.go"], + srcs = [ + "conntrack.go", + "port.go", + "utils.go", + ], visibility = ["//visibility:public"], - deps = ["//vendor/k8s.io/utils/exec:go_default_library"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/api/helper:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + ], ) go_test( name = "go_default_test", - srcs = ["conntrack_test.go"], + srcs = [ + "conntrack_test.go", + "port_test.go", + "utils_test.go", + ], library = ":go_default_library", deps = [ + "//pkg/api:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec/testing:go_default_library", ], diff --git a/pkg/proxy/util/port.go b/pkg/proxy/util/port.go new file mode 100644 index 0000000000000..fd1a024dac8a2 --- /dev/null +++ b/pkg/proxy/util/port.go @@ -0,0 +1,64 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + + "github.com/golang/glog" +) + +// LocalPort describes a port on specific IP address and protocol +type LocalPort struct { + // Description is the identity message of a given local port. + Description string + // IP is the IP address part of a given local port. + // If this string is empty, the port binds to all local IP addresses. + IP string + // Port is the port part of a given local port. + Port int + // Protocol is the protocol part of a given local port. + // The value is assumed to be lower-case. For example, "udp" not "UDP", "tcp" not "TCP". + Protocol string +} + +func (lp *LocalPort) String() string { + return fmt.Sprintf("%q (%s:%d/%s)", lp.Description, lp.IP, lp.Port, lp.Protocol) +} + +// Closeable is an interface around closing an port. +type Closeable interface { + Close() error +} + +// PortOpener is an interface around port opening/closing. +// Abstracted out for testing. +type PortOpener interface { + OpenLocalPort(lp *LocalPort) (Closeable, error) +} + +// RevertPorts is closing ports in replacementPortsMap but not in originalPortsMap. In other words, it only +// closes the ports opened in this sync. +func RevertPorts(replacementPortsMap, originalPortsMap map[LocalPort]Closeable) { + for k, v := range replacementPortsMap { + // Only close newly opened local ports - leave ones that were open before this update + if originalPortsMap[k] == nil { + glog.V(2).Infof("Closing local port %s", k.String()) + v.Close() + } + } +} diff --git a/pkg/proxy/util/port_test.go b/pkg/proxy/util/port_test.go new file mode 100644 index 0000000000000..7f1cb0f9e1b9c --- /dev/null +++ b/pkg/proxy/util/port_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import "testing" + +type fakeClosable struct { + closed bool +} + +func (c *fakeClosable) Close() error { + c.closed = true + return nil +} + +func TestRevertPorts(t *testing.T) { + testCases := []struct { + replacementPorts []LocalPort + existingPorts []LocalPort + expectToBeClose []bool + }{ + { + replacementPorts: []LocalPort{ + {Port: 5001}, + {Port: 5002}, + {Port: 5003}, + }, + existingPorts: []LocalPort{}, + expectToBeClose: []bool{true, true, true}, + }, + { + replacementPorts: []LocalPort{}, + existingPorts: []LocalPort{ + {Port: 5001}, + {Port: 5002}, + {Port: 5003}, + }, + expectToBeClose: []bool{}, + }, + { + replacementPorts: []LocalPort{ + {Port: 5001}, + {Port: 5002}, + {Port: 5003}, + }, + existingPorts: []LocalPort{ + {Port: 5001}, + {Port: 5002}, + {Port: 5003}, + }, + expectToBeClose: []bool{false, false, false}, + }, + { + replacementPorts: []LocalPort{ + {Port: 5001}, + {Port: 5002}, + {Port: 5003}, + }, + existingPorts: []LocalPort{ + {Port: 5001}, + {Port: 5003}, + }, + expectToBeClose: []bool{false, true, false}, + }, + { + replacementPorts: []LocalPort{ + {Port: 5001}, + {Port: 5002}, + {Port: 5003}, + }, + existingPorts: []LocalPort{ + {Port: 5001}, + {Port: 5002}, + {Port: 5003}, + {Port: 5004}, + }, + expectToBeClose: []bool{false, false, false}, + }, + } + + for i, tc := range testCases { + replacementPortsMap := make(map[LocalPort]Closeable) + for _, lp := range tc.replacementPorts { + replacementPortsMap[lp] = &fakeClosable{} + } + existingPortsMap := make(map[LocalPort]Closeable) + for _, lp := range tc.existingPorts { + existingPortsMap[lp] = &fakeClosable{} + } + RevertPorts(replacementPortsMap, existingPortsMap) + for j, expectation := range tc.expectToBeClose { + if replacementPortsMap[tc.replacementPorts[j]].(*fakeClosable).closed != expectation { + t.Errorf("Expect replacement localport %v to be %v in test case %v", tc.replacementPorts[j], expectation, i) + } + } + for _, lp := range tc.existingPorts { + if existingPortsMap[lp].(*fakeClosable).closed == true { + t.Errorf("Expect existing localport %v to be false in test case %v", lp, i) + } + } + } +} diff --git a/pkg/proxy/util/utils.go b/pkg/proxy/util/utils.go new file mode 100644 index 0000000000000..81734f5489271 --- /dev/null +++ b/pkg/proxy/util/utils.go @@ -0,0 +1,58 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "net" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/helper" + + "github.com/golang/glog" +) + +func IsLocalIP(ip string) (bool, error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return false, err + } + for i := range addrs { + intf, _, err := net.ParseCIDR(addrs[i].String()) + if err != nil { + return false, err + } + if net.ParseIP(ip).Equal(intf) { + return true, nil + } + } + return false, nil +} + +func ShouldSkipService(svcName types.NamespacedName, service *api.Service) bool { + // if ClusterIP is "None" or empty, skip proxying + if !helper.IsServiceIPSet(service) { + glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) + return true + } + // Even if ClusterIP is set, ServiceTypeExternalName services don't get proxied + if service.Spec.Type == api.ServiceTypeExternalName { + glog.V(3).Infof("Skipping service %s due to Type=ExternalName", svcName) + return true + } + return false +} diff --git a/pkg/proxy/util/utils_test.go b/pkg/proxy/util/utils_test.go new file mode 100644 index 0000000000000..b57850ae5f98f --- /dev/null +++ b/pkg/proxy/util/utils_test.go @@ -0,0 +1,111 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/api" +) + +func TestShouldSkipService(t *testing.T) { + testCases := []struct { + service *api.Service + svcName types.NamespacedName + shouldSkip bool + }{ + { + // Cluster IP is None + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, + Spec: api.ServiceSpec{ + ClusterIP: api.ClusterIPNone, + }, + }, + svcName: types.NamespacedName{Namespace: "foo", Name: "bar"}, + shouldSkip: true, + }, + { + // Cluster IP is empty + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, + Spec: api.ServiceSpec{ + ClusterIP: "", + }, + }, + svcName: types.NamespacedName{Namespace: "foo", Name: "bar"}, + shouldSkip: true, + }, + { + // ExternalName type service + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, + Spec: api.ServiceSpec{ + ClusterIP: "1.2.3.4", + Type: api.ServiceTypeExternalName, + }, + }, + svcName: types.NamespacedName{Namespace: "foo", Name: "bar"}, + shouldSkip: true, + }, + { + // ClusterIP type service with ClusterIP set + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, + Spec: api.ServiceSpec{ + ClusterIP: "1.2.3.4", + Type: api.ServiceTypeClusterIP, + }, + }, + svcName: types.NamespacedName{Namespace: "foo", Name: "bar"}, + shouldSkip: false, + }, + { + // NodePort type service with ClusterIP set + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, + Spec: api.ServiceSpec{ + ClusterIP: "1.2.3.4", + Type: api.ServiceTypeNodePort, + }, + }, + svcName: types.NamespacedName{Namespace: "foo", Name: "bar"}, + shouldSkip: false, + }, + { + // LoadBalancer type service with ClusterIP set + service: &api.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, + Spec: api.ServiceSpec{ + ClusterIP: "1.2.3.4", + Type: api.ServiceTypeLoadBalancer, + }, + }, + svcName: types.NamespacedName{Namespace: "foo", Name: "bar"}, + shouldSkip: false, + }, + } + + for i := range testCases { + skip := ShouldSkipService(testCases[i].svcName, testCases[i].service) + if skip != testCases[i].shouldSkip { + t.Errorf("case %d: expect %v, got %v", i, testCases[i].shouldSkip, skip) + } + } +} diff --git a/pkg/proxy/winuserspace/proxier.go b/pkg/proxy/winuserspace/proxier.go index 03fd9546c4524..4e6382918c658 100644 --- a/pkg/proxy/winuserspace/proxier.go +++ b/pkg/proxy/winuserspace/proxier.go @@ -36,7 +36,6 @@ import ( ) const allAvailableInterfaces string = "" -const stickyMaxAgeMinutes int = 180 // TODO: parameterize this in the API. type portal struct { ip string @@ -360,7 +359,11 @@ func (proxier *Proxier) mergeService(service *api.Service) map[ServicePortPortal }, Port: servicePort.Name, } - proxier.loadBalancer.NewService(servicePortName, service.Spec.SessionAffinity, stickyMaxAgeMinutes) + timeoutSeconds := 0 + if service.Spec.SessionAffinity == api.ServiceAffinityClientIP { + timeoutSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds) + } + proxier.loadBalancer.NewService(servicePortName, service.Spec.SessionAffinity, timeoutSeconds) } } diff --git a/pkg/proxy/winuserspace/roundrobin.go b/pkg/proxy/winuserspace/roundrobin.go index ff2026bd8fcf6..86d7614c85034 100644 --- a/pkg/proxy/winuserspace/roundrobin.go +++ b/pkg/proxy/winuserspace/roundrobin.go @@ -48,7 +48,7 @@ type affinityState struct { type affinityPolicy struct { affinityType api.ServiceAffinity affinityMap map[string]*affinityState // map client IP -> affinity info - ttlMinutes int + ttlSeconds int } // LoadBalancerRR is a round-robin load balancer. @@ -66,11 +66,11 @@ type balancerState struct { affinity affinityPolicy } -func newAffinityPolicy(affinityType api.ServiceAffinity, ttlMinutes int) *affinityPolicy { +func newAffinityPolicy(affinityType api.ServiceAffinity, ttlSeconds int) *affinityPolicy { return &affinityPolicy{ affinityType: affinityType, affinityMap: make(map[string]*affinityState), - ttlMinutes: ttlMinutes, + ttlSeconds: ttlSeconds, } } @@ -81,22 +81,22 @@ func NewLoadBalancerRR() *LoadBalancerRR { } } -func (lb *LoadBalancerRR) NewService(svcPort proxy.ServicePortName, affinityType api.ServiceAffinity, ttlMinutes int) error { +func (lb *LoadBalancerRR) NewService(svcPort proxy.ServicePortName, affinityType api.ServiceAffinity, ttlSeconds int) error { glog.V(4).Infof("LoadBalancerRR NewService %q", svcPort) lb.lock.Lock() defer lb.lock.Unlock() - lb.newServiceInternal(svcPort, affinityType, ttlMinutes) + lb.newServiceInternal(svcPort, affinityType, ttlSeconds) return nil } // This assumes that lb.lock is already held. -func (lb *LoadBalancerRR) newServiceInternal(svcPort proxy.ServicePortName, affinityType api.ServiceAffinity, ttlMinutes int) *balancerState { - if ttlMinutes == 0 { - ttlMinutes = 180 //default to 3 hours if not specified. Should 0 be unlimited instead???? +func (lb *LoadBalancerRR) newServiceInternal(svcPort proxy.ServicePortName, affinityType api.ServiceAffinity, ttlSeconds int) *balancerState { + if ttlSeconds == 0 { + ttlSeconds = int(api.DefaultClientIPServiceAffinitySeconds) //default to 3 hours if not specified. Should 0 be unlimited instead???? } if _, exists := lb.services[svcPort]; !exists { - lb.services[svcPort] = &balancerState{affinity: *newAffinityPolicy(affinityType, ttlMinutes)} + lb.services[svcPort] = &balancerState{affinity: *newAffinityPolicy(affinityType, ttlSeconds)} glog.V(4).Infof("LoadBalancerRR service %q did not exist, created", svcPort) } else if affinityType != "" { lb.services[svcPort].affinity.affinityType = affinityType @@ -149,7 +149,7 @@ func (lb *LoadBalancerRR) NextEndpoint(svcPort proxy.ServicePortName, srcAddr ne } if !sessionAffinityReset { sessionAffinity, exists := state.affinity.affinityMap[ipaddr] - if exists && int(time.Now().Sub(sessionAffinity.lastUsed).Minutes()) < state.affinity.ttlMinutes { + if exists && int(time.Now().Sub(sessionAffinity.lastUsed).Seconds()) < state.affinity.ttlSeconds { // Affinity wins. endpoint := sessionAffinity.endpoint sessionAffinity.lastUsed = time.Now() @@ -366,7 +366,7 @@ func (lb *LoadBalancerRR) CleanupStaleStickySessions(svcPort proxy.ServicePortNa return } for ip, affinity := range state.affinity.affinityMap { - if int(time.Now().Sub(affinity.lastUsed).Minutes()) >= state.affinity.ttlMinutes { + if int(time.Now().Sub(affinity.lastUsed).Seconds()) >= state.affinity.ttlSeconds { glog.V(4).Infof("Removing client %s from affinityMap for service %q", affinity.clientIP, svcPort) delete(state.affinity.affinityMap, ip) } diff --git a/pkg/proxy/winuserspace/roundrobin_test.go b/pkg/proxy/winuserspace/roundrobin_test.go index b334a3ccc241e..1f4973ee1a941 100644 --- a/pkg/proxy/winuserspace/roundrobin_test.go +++ b/pkg/proxy/winuserspace/roundrobin_test.go @@ -357,7 +357,7 @@ func TestStickyLoadBalanceWorksWithNewServiceCalledFirst(t *testing.T) { } // Call NewService() before OnEndpointsUpdate() - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpoints := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: []api.EndpointSubset{ @@ -421,7 +421,7 @@ func TestStickyLoadBalanceWorksWithNewServiceCalledSecond(t *testing.T) { }, } loadBalancer.OnEndpointsAdd(endpoints) - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) client1 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} client2 := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 2), Port: 0} @@ -473,7 +473,7 @@ func TestStickyLoadBalanaceWorksWithMultipleEndpointsRemoveOne(t *testing.T) { t.Errorf("Didn't fail with non-existent service") } - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpointsv1 := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: []api.EndpointSubset{ @@ -546,7 +546,7 @@ func TestStickyLoadBalanceWorksWithMultipleEndpointsAndUpdates(t *testing.T) { t.Errorf("Didn't fail with non-existent service") } - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpointsv1 := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: []api.EndpointSubset{ @@ -605,7 +605,7 @@ func TestStickyLoadBalanceWorksWithServiceRemoval(t *testing.T) { if err == nil || len(endpoint) != 0 { t.Errorf("Didn't fail with non-existent service") } - loadBalancer.NewService(fooService, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(fooService, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpoints1 := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: fooService.Name, Namespace: fooService.Namespace}, Subsets: []api.EndpointSubset{ @@ -616,7 +616,7 @@ func TestStickyLoadBalanceWorksWithServiceRemoval(t *testing.T) { }, } barService := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: "testnamespace", Name: "bar"}, Port: ""} - loadBalancer.NewService(barService, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(barService, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpoints2 := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: barService.Name, Namespace: barService.Namespace}, Subsets: []api.EndpointSubset{ @@ -674,7 +674,7 @@ func TestStickyLoadBalanceWorksWithEndpointFails(t *testing.T) { } // Call NewService() before OnEndpointsUpdate() - loadBalancer.NewService(service, api.ServiceAffinityClientIP, 0) + loadBalancer.NewService(service, api.ServiceAffinityClientIP, int(api.DefaultClientIPServiceAffinitySeconds)) endpoints := &api.Endpoints{ ObjectMeta: metav1.ObjectMeta{Name: service.Name, Namespace: service.Namespace}, Subsets: []api.EndpointSubset{ diff --git a/pkg/quota/evaluator/core/BUILD b/pkg/quota/evaluator/core/BUILD index efc0798d80a6e..b413ddac201a7 100644 --- a/pkg/quota/evaluator/core/BUILD +++ b/pkg/quota/evaluator/core/BUILD @@ -25,6 +25,7 @@ go_library( "//pkg/api/helper/qos:go_default_library", "//pkg/api/v1:go_default_library", "//pkg/api/validation:go_default_library", + "//pkg/kubeapiserver/admission/util:go_default_library", "//pkg/quota:go_default_library", "//pkg/quota/generic:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/quota/evaluator/core/persistent_volume_claims.go b/pkg/quota/evaluator/core/persistent_volume_claims.go index 86e7350dee706..ed2d8bef0d058 100644 --- a/pkg/quota/evaluator/core/persistent_volume_claims.go +++ b/pkg/quota/evaluator/core/persistent_volume_claims.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/helper" k8s_api_v1 "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/pkg/kubeapiserver/admission/util" "k8s.io/kubernetes/pkg/quota" "k8s.io/kubernetes/pkg/quota/generic" ) @@ -141,8 +142,18 @@ func (p *pvcEvaluator) GroupKind() schema.GroupKind { } // Handles returns true if the evaluator should handle the specified operation. -func (p *pvcEvaluator) Handles(operation admission.Operation) bool { - return admission.Create == operation +func (p *pvcEvaluator) Handles(a admission.Attributes) bool { + op := a.GetOperation() + if op == admission.Create { + return true + } + updateUninitialized, err := util.IsUpdatingUninitializedObject(a) + if err != nil { + // fail closed, will try to give an evaluation. + return true + } + // only uninitialized pvc might be updated. + return updateUninitialized } // Matches returns true if the evaluator matches the specified quota with the provided input item diff --git a/pkg/quota/evaluator/core/pods.go b/pkg/quota/evaluator/core/pods.go index 5773a4a229db7..0417bb8eccb06 100644 --- a/pkg/quota/evaluator/core/pods.go +++ b/pkg/quota/evaluator/core/pods.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/api/helper/qos" k8s_api_v1 "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/api/validation" + "k8s.io/kubernetes/pkg/kubeapiserver/admission/util" "k8s.io/kubernetes/pkg/quota" "k8s.io/kubernetes/pkg/quota/generic" ) @@ -42,10 +43,13 @@ import ( var podResources = []api.ResourceName{ api.ResourceCPU, api.ResourceMemory, + api.ResourceEphemeralStorage, api.ResourceRequestsCPU, api.ResourceRequestsMemory, + api.ResourceRequestsEphemeralStorage, api.ResourceLimitsCPU, api.ResourceLimitsMemory, + api.ResourceLimitsEphemeralStorage, api.ResourcePods, } @@ -131,10 +135,19 @@ func (p *podEvaluator) GroupKind() schema.GroupKind { return api.Kind("Pod") } -// Handles returns true of the evaluator should handle the specified operation. -func (p *podEvaluator) Handles(operation admission.Operation) bool { - // TODO: update this if/when pods support resizing resource requirements. - return admission.Create == operation +// Handles returns true of the evaluator should handle the specified attributes. +func (p *podEvaluator) Handles(a admission.Attributes) bool { + op := a.GetOperation() + if op == admission.Create { + return true + } + updateUninitialized, err := util.IsUpdatingUninitializedObject(a) + if err != nil { + // fail closed, will try to give an evaluation. + return true + } + // only uninitialized pods might be updated. + return updateUninitialized } // Matches returns true if the evaluator matches the specified quota with the provided input item @@ -191,6 +204,13 @@ func podUsageHelper(requests api.ResourceList, limits api.ResourceList) api.Reso if limit, found := limits[api.ResourceMemory]; found { result[api.ResourceLimitsMemory] = limit } + if request, found := requests[api.ResourceEphemeralStorage]; found { + result[api.ResourceEphemeralStorage] = request + result[api.ResourceRequestsEphemeralStorage] = request + } + if limit, found := limits[api.ResourceEphemeralStorage]; found { + result[api.ResourceLimitsEphemeralStorage] = limit + } return result } diff --git a/pkg/quota/evaluator/core/pods_test.go b/pkg/quota/evaluator/core/pods_test.go index b947e1f3229b5..3a93165b81a5e 100644 --- a/pkg/quota/evaluator/core/pods_test.go +++ b/pkg/quota/evaluator/core/pods_test.go @@ -42,7 +42,7 @@ func TestPodConstraintsFunc(t *testing.T) { }}, }, }, - err: `spec.initContainers[0].resources.limits: Invalid value: "1m": must be greater than or equal to cpu request`, + err: `spec.initContainers[0].resources.requests: Invalid value: "2m": must be less than or equal to cpu limit`, }, "container resource invalid": { pod: &api.Pod{ @@ -55,7 +55,7 @@ func TestPodConstraintsFunc(t *testing.T) { }}, }, }, - err: `spec.containers[0].resources.limits: Invalid value: "1m": must be greater than or equal to cpu request`, + err: `spec.containers[0].resources.requests: Invalid value: "2m": must be less than or equal to cpu limit`, }, "init container resource missing": { pod: &api.Pod{ @@ -142,6 +142,24 @@ func TestPodEvaluatorUsage(t *testing.T) { api.ResourceMemory: resource.MustParse("1m"), }, }, + "init container local ephemeral storage": { + pod: &api.Pod{ + Spec: api.PodSpec{ + InitContainers: []api.Container{{ + Resources: api.ResourceRequirements{ + Requests: api.ResourceList{api.ResourceEphemeralStorage: resource.MustParse("32Mi")}, + Limits: api.ResourceList{api.ResourceEphemeralStorage: resource.MustParse("64Mi")}, + }, + }}, + }, + }, + usage: api.ResourceList{ + api.ResourceEphemeralStorage: resource.MustParse("32Mi"), + api.ResourceRequestsEphemeralStorage: resource.MustParse("32Mi"), + api.ResourceLimitsEphemeralStorage: resource.MustParse("64Mi"), + api.ResourcePods: resource.MustParse("1"), + }, + }, "container CPU": { pod: &api.Pod{ Spec: api.PodSpec{ @@ -178,6 +196,24 @@ func TestPodEvaluatorUsage(t *testing.T) { api.ResourceMemory: resource.MustParse("1m"), }, }, + "container local ephemeral storage": { + pod: &api.Pod{ + Spec: api.PodSpec{ + Containers: []api.Container{{ + Resources: api.ResourceRequirements{ + Requests: api.ResourceList{api.ResourceEphemeralStorage: resource.MustParse("32Mi")}, + Limits: api.ResourceList{api.ResourceEphemeralStorage: resource.MustParse("64Mi")}, + }, + }}, + }, + }, + usage: api.ResourceList{ + api.ResourceEphemeralStorage: resource.MustParse("32Mi"), + api.ResourceRequestsEphemeralStorage: resource.MustParse("32Mi"), + api.ResourceLimitsEphemeralStorage: resource.MustParse("64Mi"), + api.ResourcePods: resource.MustParse("1"), + }, + }, "init container maximums override sum of containers": { pod: &api.Pod{ Spec: api.PodSpec{ diff --git a/pkg/quota/evaluator/core/services.go b/pkg/quota/evaluator/core/services.go index 457d8b4173543..91a5ef0a059b1 100644 --- a/pkg/quota/evaluator/core/services.go +++ b/pkg/quota/evaluator/core/services.go @@ -108,7 +108,8 @@ func (p *serviceEvaluator) GroupKind() schema.GroupKind { } // Handles returns true of the evaluator should handle the specified operation. -func (p *serviceEvaluator) Handles(operation admission.Operation) bool { +func (p *serviceEvaluator) Handles(a admission.Attributes) bool { + operation := a.GetOperation() // We handle create and update because a service type can change. return admission.Create == operation || admission.Update == operation } diff --git a/pkg/quota/generic/evaluator.go b/pkg/quota/generic/evaluator.go index ab3bbb531b4e9..1b574bb6b52ec 100644 --- a/pkg/quota/generic/evaluator.go +++ b/pkg/quota/generic/evaluator.go @@ -150,8 +150,9 @@ func (o *ObjectCountEvaluator) GroupKind() schema.GroupKind { return o.InternalGroupKind } -// Handles returns true if the object count evaluator needs to track this operation. -func (o *ObjectCountEvaluator) Handles(operation admission.Operation) bool { +// Handles returns true if the object count evaluator needs to track this attributes. +func (o *ObjectCountEvaluator) Handles(a admission.Attributes) bool { + operation := a.GetOperation() return operation == admission.Create || (o.AllowCreateOnUpdate && operation == admission.Update) } diff --git a/pkg/quota/interfaces.go b/pkg/quota/interfaces.go index 393e89791b57a..56eacfe2d47f5 100644 --- a/pkg/quota/interfaces.go +++ b/pkg/quota/interfaces.go @@ -45,9 +45,9 @@ type Evaluator interface { Constraints(required []api.ResourceName, item runtime.Object) error // GroupKind returns the groupKind that this object knows how to evaluate GroupKind() schema.GroupKind - // Handles determines if quota could be impacted by the specified operation. + // Handles determines if quota could be impacted by the specified attribute. // If true, admission control must perform quota processing for the operation, otherwise it is safe to ignore quota. - Handles(operation admission.Operation) bool + Handles(operation admission.Attributes) bool // Matches returns true if the specified quota matches the input item Matches(resourceQuota *api.ResourceQuota, item runtime.Object) (bool, error) // MatchingResources takes the input specified list of resources and returns the set of resources evaluator matches. diff --git a/pkg/registry/OWNERS b/pkg/registry/OWNERS index 1b51832e39e70..b26edf1880dc5 100644 --- a/pkg/registry/OWNERS +++ b/pkg/registry/OWNERS @@ -37,3 +37,4 @@ reviewers: - dims - madhusudancs - hongchaodeng +- enj diff --git a/pkg/registry/apps/statefulset/storage/BUILD b/pkg/registry/apps/statefulset/storage/BUILD index 1d3a58e657314..66767360ae97b 100644 --- a/pkg/registry/apps/statefulset/storage/BUILD +++ b/pkg/registry/apps/statefulset/storage/BUILD @@ -36,6 +36,9 @@ go_library( "//pkg/apis/apps:go_default_library", "//pkg/apis/extensions:go_default_library", "//pkg/apis/extensions/validation:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/apps/statefulset:go_default_library", "//pkg/registry/cachesize:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/pkg/registry/apps/statefulset/storage/storage.go b/pkg/registry/apps/statefulset/storage/storage.go index 80375251a9662..066e95116a645 100644 --- a/pkg/registry/apps/statefulset/storage/storage.go +++ b/pkg/registry/apps/statefulset/storage/storage.go @@ -30,6 +30,9 @@ import ( "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/extensions" extvalidation "k8s.io/kubernetes/pkg/apis/extensions/validation" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/apps/statefulset" "k8s.io/kubernetes/pkg/registry/cachesize" ) @@ -69,6 +72,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) { CreateStrategy: statefulset.Strategy, UpdateStrategy: statefulset.Strategy, DeleteStrategy: statefulset.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/authorization/OWNERS b/pkg/registry/authorization/OWNERS index 7c9eabed06954..d21ad8662fdcf 100755 --- a/pkg/registry/authorization/OWNERS +++ b/pkg/registry/authorization/OWNERS @@ -1,3 +1,4 @@ reviewers: - deads2k - liggitt +- enj diff --git a/pkg/registry/batch/cronjob/storage/BUILD b/pkg/registry/batch/cronjob/storage/BUILD index 2d8b18208f053..823dc973f701d 100644 --- a/pkg/registry/batch/cronjob/storage/BUILD +++ b/pkg/registry/batch/cronjob/storage/BUILD @@ -31,6 +31,9 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/apis/batch:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/batch/cronjob:go_default_library", "//pkg/registry/cachesize:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/registry/batch/cronjob/storage/storage.go b/pkg/registry/batch/cronjob/storage/storage.go index dbff7059b2424..9e48fb244f8c3 100644 --- a/pkg/registry/batch/cronjob/storage/storage.go +++ b/pkg/registry/batch/cronjob/storage/storage.go @@ -25,6 +25,9 @@ import ( "k8s.io/apiserver/pkg/registry/rest" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/batch" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/batch/cronjob" "k8s.io/kubernetes/pkg/registry/cachesize" ) @@ -42,9 +45,12 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) { NewListFunc: func() runtime.Object { return &batch.CronJobList{} }, DefaultQualifiedResource: batch.Resource("cronjobs"), WatchCacheSize: cachesize.GetWatchCacheSizeByResource("cronjobs"), - CreateStrategy: cronjob.Strategy, - UpdateStrategy: cronjob.Strategy, - DeleteStrategy: cronjob.Strategy, + + CreateStrategy: cronjob.Strategy, + UpdateStrategy: cronjob.Strategy, + DeleteStrategy: cronjob.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/batch/job/storage/BUILD b/pkg/registry/batch/job/storage/BUILD index 90b02b45ff2a0..3173c3ea3d355 100644 --- a/pkg/registry/batch/job/storage/BUILD +++ b/pkg/registry/batch/job/storage/BUILD @@ -29,6 +29,9 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/apis/batch:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/batch/job:go_default_library", "//pkg/registry/cachesize:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/registry/batch/job/storage/storage.go b/pkg/registry/batch/job/storage/storage.go index aa52d251023ca..9fa1ced76f9c0 100644 --- a/pkg/registry/batch/job/storage/storage.go +++ b/pkg/registry/batch/job/storage/storage.go @@ -25,6 +25,9 @@ import ( "k8s.io/apiserver/pkg/registry/rest" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/batch" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/batch/job" "k8s.io/kubernetes/pkg/registry/cachesize" ) @@ -62,6 +65,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) { CreateStrategy: job.Strategy, UpdateStrategy: job.Strategy, DeleteStrategy: job.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: job.GetAttrs} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/certificates/OWNERS b/pkg/registry/certificates/OWNERS index 07a69259288f4..ce89a734755bb 100755 --- a/pkg/registry/certificates/OWNERS +++ b/pkg/registry/certificates/OWNERS @@ -8,3 +8,4 @@ reviewers: - dims - hongchaodeng - david-mcmahon +- enj diff --git a/pkg/registry/core/endpoint/storage/BUILD b/pkg/registry/core/endpoint/storage/BUILD index 3721e1a89c90e..e348e08c7fbd9 100644 --- a/pkg/registry/core/endpoint/storage/BUILD +++ b/pkg/registry/core/endpoint/storage/BUILD @@ -27,6 +27,9 @@ go_library( srcs = ["storage.go"], deps = [ "//pkg/api:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/core/endpoint:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/pkg/registry/core/endpoint/storage/storage.go b/pkg/registry/core/endpoint/storage/storage.go index 28e30837199fe..d7ed2254745c0 100644 --- a/pkg/registry/core/endpoint/storage/storage.go +++ b/pkg/registry/core/endpoint/storage/storage.go @@ -22,6 +22,9 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/core/endpoint" ) @@ -42,6 +45,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) *REST { CreateStrategy: endpoint.Strategy, UpdateStrategy: endpoint.Strategy, DeleteStrategy: endpoint.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/core/node/storage/BUILD b/pkg/registry/core/node/storage/BUILD index 04d1942af5fff..27d3a8a0278e4 100644 --- a/pkg/registry/core/node/storage/BUILD +++ b/pkg/registry/core/node/storage/BUILD @@ -31,6 +31,9 @@ go_library( "//pkg/api:go_default_library", "//pkg/api/v1:go_default_library", "//pkg/kubelet/client:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/core/node:go_default_library", "//pkg/registry/core/node/rest:go_default_library", diff --git a/pkg/registry/core/node/storage/storage.go b/pkg/registry/core/node/storage/storage.go index fb9e6c2fcdb15..0f28cec47f0fa 100644 --- a/pkg/registry/core/node/storage/storage.go +++ b/pkg/registry/core/node/storage/storage.go @@ -31,6 +31,9 @@ import ( "k8s.io/kubernetes/pkg/api" k8s_api_v1 "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/kubelet/client" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/core/node" noderest "k8s.io/kubernetes/pkg/registry/core/node/rest" @@ -84,6 +87,8 @@ func NewStorage(optsGetter generic.RESTOptionsGetter, kubeletClientConfig client UpdateStrategy: node.Strategy, DeleteStrategy: node.Strategy, ExportStrategy: node.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: node.GetAttrs, TriggerFunc: node.NodeNameTriggerFunc} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/core/persistentvolume/storage/storage_test.go b/pkg/registry/core/persistentvolume/storage/storage_test.go index 264d159f03e6e..3b419ad342599 100644 --- a/pkg/registry/core/persistentvolume/storage/storage_test.go +++ b/pkg/registry/core/persistentvolume/storage/storage_test.go @@ -46,6 +46,12 @@ func newStorage(t *testing.T) (*REST, *StatusREST, *etcdtesting.EtcdTestServer) return persistentVolumeStorage, statusStorage, server } +func newHostPathType(pathType string) *api.HostPathType { + hostPathType := new(api.HostPathType) + *hostPathType = api.HostPathType(pathType) + return hostPathType +} + func validNewPersistentVolume(name string) *api.PersistentVolume { pv := &api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ @@ -57,7 +63,7 @@ func validNewPersistentVolume(name string) *api.PersistentVolume { }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: "/foo"}, + HostPath: &api.HostPathVolumeSource{Path: "/foo", Type: newHostPathType(string(api.HostPathDirectoryOrCreate))}, }, PersistentVolumeReclaimPolicy: api.PersistentVolumeReclaimRetain, }, diff --git a/pkg/registry/core/pod/strategy.go b/pkg/registry/core/pod/strategy.go index 9337ef9b9941b..2c6bbf5237630 100644 --- a/pkg/registry/core/pod/strategy.go +++ b/pkg/registry/core/pod/strategy.go @@ -196,9 +196,10 @@ func PodToSelectableFields(pod *api.Pod) fields.Set { // amount of allocations needed to create the fields.Set. If you add any // field here or the number of object-meta related fields changes, this should // be adjusted. - podSpecificFieldsSet := make(fields.Set, 6) + podSpecificFieldsSet := make(fields.Set, 7) podSpecificFieldsSet["spec.nodeName"] = pod.Spec.NodeName podSpecificFieldsSet["spec.restartPolicy"] = string(pod.Spec.RestartPolicy) + podSpecificFieldsSet["spec.schedulerName"] = string(pod.Spec.SchedulerName) podSpecificFieldsSet["status.phase"] = string(pod.Status.Phase) podSpecificFieldsSet["status.podIP"] = string(pod.Status.PodIP) return generic.AddObjectMetaFieldsSet(podSpecificFieldsSet, &pod.ObjectMeta, true) diff --git a/pkg/registry/core/pod/strategy_test.go b/pkg/registry/core/pod/strategy_test.go index e2f34621085d8..4d602cb06569a 100644 --- a/pkg/registry/core/pod/strategy_test.go +++ b/pkg/registry/core/pod/strategy_test.go @@ -71,6 +71,20 @@ func TestMatchPod(t *testing.T) { fieldSelector: fields.ParseSelectorOrDie("spec.restartPolicy=Never"), expectMatch: false, }, + { + in: &api.Pod{ + Spec: api.PodSpec{SchedulerName: "scheduler1"}, + }, + fieldSelector: fields.ParseSelectorOrDie("spec.schedulerName=scheduler1"), + expectMatch: true, + }, + { + in: &api.Pod{ + Spec: api.PodSpec{SchedulerName: "scheduler1"}, + }, + fieldSelector: fields.ParseSelectorOrDie("spec.schedulerName=scheduler2"), + expectMatch: false, + }, { in: &api.Pod{ Status: api.PodStatus{Phase: api.PodRunning}, diff --git a/pkg/registry/core/podtemplate/storage/BUILD b/pkg/registry/core/podtemplate/storage/BUILD index 94aeadfdbd65a..d408695edd18a 100644 --- a/pkg/registry/core/podtemplate/storage/BUILD +++ b/pkg/registry/core/podtemplate/storage/BUILD @@ -27,6 +27,9 @@ go_library( srcs = ["storage.go"], deps = [ "//pkg/api:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/core/podtemplate:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/pkg/registry/core/podtemplate/storage/storage.go b/pkg/registry/core/podtemplate/storage/storage.go index 9376d4b4ba49c..81b4d4e74f51e 100644 --- a/pkg/registry/core/podtemplate/storage/storage.go +++ b/pkg/registry/core/podtemplate/storage/storage.go @@ -21,6 +21,9 @@ import ( "k8s.io/apiserver/pkg/registry/generic" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/core/podtemplate" ) @@ -44,6 +47,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) *REST { ExportStrategy: podtemplate.Strategy, ReturnDeletedObject: true, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/core/replicationcontroller/storage/BUILD b/pkg/registry/core/replicationcontroller/storage/BUILD index ffef9f4d1d359..99928e1bde34a 100644 --- a/pkg/registry/core/replicationcontroller/storage/BUILD +++ b/pkg/registry/core/replicationcontroller/storage/BUILD @@ -35,6 +35,9 @@ go_library( "//pkg/api:go_default_library", "//pkg/apis/autoscaling:go_default_library", "//pkg/apis/autoscaling/validation:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/core/replicationcontroller:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/pkg/registry/core/replicationcontroller/storage/storage.go b/pkg/registry/core/replicationcontroller/storage/storage.go index 1d0bb1e76e988..210f29532d2c6 100644 --- a/pkg/registry/core/replicationcontroller/storage/storage.go +++ b/pkg/registry/core/replicationcontroller/storage/storage.go @@ -32,6 +32,9 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/autoscaling" "k8s.io/kubernetes/pkg/apis/autoscaling/validation" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/core/replicationcontroller" ) @@ -71,6 +74,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) { CreateStrategy: replicationcontroller.Strategy, UpdateStrategy: replicationcontroller.Strategy, DeleteStrategy: replicationcontroller.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: replicationcontroller.GetAttrs} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/core/service/storage/BUILD b/pkg/registry/core/service/storage/BUILD index faece9dd9ed9f..d96681f169615 100644 --- a/pkg/registry/core/service/storage/BUILD +++ b/pkg/registry/core/service/storage/BUILD @@ -28,6 +28,9 @@ go_library( srcs = ["storage.go"], deps = [ "//pkg/api:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/core/service:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/registry/core/service/storage/storage.go b/pkg/registry/core/service/storage/storage.go index 886f227592e7d..865e792098411 100644 --- a/pkg/registry/core/service/storage/storage.go +++ b/pkg/registry/core/service/storage/storage.go @@ -24,6 +24,9 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/core/service" ) @@ -45,6 +48,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) { UpdateStrategy: service.Strategy, DeleteStrategy: service.Strategy, ExportStrategy: service.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/extensions/daemonset/storage/BUILD b/pkg/registry/extensions/daemonset/storage/BUILD index 176dae98bc97e..d4e6ddfac9bb2 100644 --- a/pkg/registry/extensions/daemonset/storage/BUILD +++ b/pkg/registry/extensions/daemonset/storage/BUILD @@ -29,6 +29,9 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/apis/extensions:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/extensions/daemonset:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/registry/extensions/daemonset/storage/storage.go b/pkg/registry/extensions/daemonset/storage/storage.go index 0ba0dc9b3cded..5f0c4d3079314 100644 --- a/pkg/registry/extensions/daemonset/storage/storage.go +++ b/pkg/registry/extensions/daemonset/storage/storage.go @@ -25,6 +25,9 @@ import ( "k8s.io/apiserver/pkg/registry/rest" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/extensions" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/extensions/daemonset" ) @@ -46,6 +49,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) { CreateStrategy: daemonset.Strategy, UpdateStrategy: daemonset.Strategy, DeleteStrategy: daemonset.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/extensions/deployment/storage/storage.go b/pkg/registry/extensions/deployment/storage/storage.go index 1e6fcbe83d5b0..025f9cb52c311 100644 --- a/pkg/registry/extensions/deployment/storage/storage.go +++ b/pkg/registry/extensions/deployment/storage/storage.go @@ -146,6 +146,7 @@ func (r *RollbackREST) Create(ctx genericapirequest.Context, obj runtime.Object, return nil, err } return &metav1.Status{ + Status: metav1.StatusSuccess, Message: fmt.Sprintf("rollback request for deployment %q succeeded", rollback.Name), Code: http.StatusOK, }, nil diff --git a/pkg/registry/extensions/deployment/storage/storage_test.go b/pkg/registry/extensions/deployment/storage/storage_test.go index 07c84d58c2197..3bad7f5d04aba 100644 --- a/pkg/registry/extensions/deployment/storage/storage_test.go +++ b/pkg/registry/extensions/deployment/storage/storage_test.go @@ -17,6 +17,7 @@ limitations under the License. package storage import ( + "net/http" "reflect" "testing" @@ -343,10 +344,18 @@ func TestEtcdCreateDeploymentRollback(t *testing.T) { if _, err := storage.Deployment.Create(ctx, validNewDeployment(), false); err != nil { t.Fatalf("%s: unexpected error: %v", k, err) } - if _, err := rollbackStorage.Create(ctx, &test.rollback, false); !test.errOK(err) { + rollbackRespStatus, err := rollbackStorage.Create(ctx, &test.rollback, false) + if !test.errOK(err) { t.Errorf("%s: unexpected error: %v", k, err) } else if err == nil { - // If rollback succeeded, verify Rollback field of deployment + // If rollback succeeded, verify Rollback response and Rollback field of deployment + status, ok := rollbackRespStatus.(*metav1.Status) + if !ok { + t.Errorf("%s: unexpected response format", k) + } + if status.Code != http.StatusOK || status.Status != metav1.StatusSuccess { + t.Errorf("%s: unexpected response, code: %d, status: %s", k, status.Code, status.Status) + } d, err := storage.Deployment.Get(ctx, validNewDeployment().ObjectMeta.Name, &metav1.GetOptions{}) if err != nil { t.Errorf("%s: unexpected error: %v", k, err) diff --git a/pkg/registry/extensions/ingress/storage/BUILD b/pkg/registry/extensions/ingress/storage/BUILD index 0b538d6b2cf42..5bfdd5d06881f 100644 --- a/pkg/registry/extensions/ingress/storage/BUILD +++ b/pkg/registry/extensions/ingress/storage/BUILD @@ -30,6 +30,9 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/apis/extensions:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/extensions/ingress:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/registry/extensions/ingress/storage/storage.go b/pkg/registry/extensions/ingress/storage/storage.go index f6c7d6a915c13..7a9e6409e87b9 100644 --- a/pkg/registry/extensions/ingress/storage/storage.go +++ b/pkg/registry/extensions/ingress/storage/storage.go @@ -25,6 +25,9 @@ import ( "k8s.io/apiserver/pkg/registry/rest" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/extensions" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/extensions/ingress" ) @@ -46,6 +49,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) { CreateStrategy: ingress.Strategy, UpdateStrategy: ingress.Strategy, DeleteStrategy: ingress.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/extensions/replicaset/storage/BUILD b/pkg/registry/extensions/replicaset/storage/BUILD index 7ccbe4d101be7..a1cca2204b46e 100644 --- a/pkg/registry/extensions/replicaset/storage/BUILD +++ b/pkg/registry/extensions/replicaset/storage/BUILD @@ -35,6 +35,9 @@ go_library( "//pkg/api:go_default_library", "//pkg/apis/extensions:go_default_library", "//pkg/apis/extensions/validation:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/extensions/replicaset:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/pkg/registry/extensions/replicaset/storage/storage.go b/pkg/registry/extensions/replicaset/storage/storage.go index b10c455ae911b..e3b7d18745fb9 100644 --- a/pkg/registry/extensions/replicaset/storage/storage.go +++ b/pkg/registry/extensions/replicaset/storage/storage.go @@ -31,6 +31,9 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/extensions" extvalidation "k8s.io/kubernetes/pkg/apis/extensions/validation" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/extensions/replicaset" ) @@ -70,6 +73,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) { CreateStrategy: replicaset.Strategy, UpdateStrategy: replicaset.Strategy, DeleteStrategy: replicaset.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: replicaset.GetAttrs} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/policy/poddisruptionbudget/storage/BUILD b/pkg/registry/policy/poddisruptionbudget/storage/BUILD index 5c0c85eca3f05..fe817c6c6853a 100644 --- a/pkg/registry/policy/poddisruptionbudget/storage/BUILD +++ b/pkg/registry/policy/poddisruptionbudget/storage/BUILD @@ -31,6 +31,9 @@ go_library( deps = [ "//pkg/api:go_default_library", "//pkg/apis/policy:go_default_library", + "//pkg/printers:go_default_library", + "//pkg/printers/internalversion:go_default_library", + "//pkg/printers/storage:go_default_library", "//pkg/registry/cachesize:go_default_library", "//pkg/registry/policy/poddisruptionbudget:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/registry/policy/poddisruptionbudget/storage/storage.go b/pkg/registry/policy/poddisruptionbudget/storage/storage.go index e97136c295865..2f196e85b1a6c 100644 --- a/pkg/registry/policy/poddisruptionbudget/storage/storage.go +++ b/pkg/registry/policy/poddisruptionbudget/storage/storage.go @@ -25,6 +25,9 @@ import ( "k8s.io/apiserver/pkg/registry/rest" "k8s.io/kubernetes/pkg/api" policyapi "k8s.io/kubernetes/pkg/apis/policy" + "k8s.io/kubernetes/pkg/printers" + printersinternal "k8s.io/kubernetes/pkg/printers/internalversion" + printerstorage "k8s.io/kubernetes/pkg/printers/storage" "k8s.io/kubernetes/pkg/registry/cachesize" "k8s.io/kubernetes/pkg/registry/policy/poddisruptionbudget" ) @@ -46,6 +49,8 @@ func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) { CreateStrategy: poddisruptionbudget.Strategy, UpdateStrategy: poddisruptionbudget.Strategy, DeleteStrategy: poddisruptionbudget.Strategy, + + TableConvertor: printerstorage.TableConvertor{TablePrinter: printers.NewTablePrinter().With(printersinternal.AddHandlers)}, } options := &generic.StoreOptions{RESTOptions: optsGetter} if err := store.CompleteWithOptions(options); err != nil { diff --git a/pkg/registry/rbac/OWNERS b/pkg/registry/rbac/OWNERS index 7c9eabed06954..d21ad8662fdcf 100755 --- a/pkg/registry/rbac/OWNERS +++ b/pkg/registry/rbac/OWNERS @@ -1,3 +1,4 @@ reviewers: - deads2k - liggitt +- enj diff --git a/pkg/registry/registrytest/OWNERS b/pkg/registry/registrytest/OWNERS index 2bfe9edfb7b83..c00d143160e21 100755 --- a/pkg/registry/registrytest/OWNERS +++ b/pkg/registry/registrytest/OWNERS @@ -21,3 +21,4 @@ reviewers: - ddysher - mqliang - sdminonne +- enj diff --git a/pkg/registry/registrytest/etcd.go b/pkg/registry/registrytest/etcd.go index 13abeac2baac6..344ce4629b2a2 100644 --- a/pkg/registry/registrytest/etcd.go +++ b/pkg/registry/registrytest/etcd.go @@ -202,6 +202,10 @@ func (t *Tester) createObject(ctx genericapirequest.Context, obj runtime.Object) func (t *Tester) setObjectsForList(objects []runtime.Object) []runtime.Object { key := t.storage.KeyRootFunc(t.tester.TestContext()) + if _, err := t.storage.DeleteCollection(t.tester.TestContext(), nil, nil); err != nil { + t.tester.Errorf("unable to clear collection: %v", err) + return nil + } if err := storagetesting.CreateObjList(key, t.storage.Storage, objects); err != nil { t.tester.Errorf("unexpected error: %v", err) return nil diff --git a/pkg/security/apparmor/validate.go b/pkg/security/apparmor/validate.go index 1036a43b3d533..13788876fb6a0 100644 --- a/pkg/security/apparmor/validate.go +++ b/pkg/security/apparmor/validate.go @@ -111,8 +111,8 @@ func validateHost(runtime string) error { } // Check runtime support. Currently only Docker is supported. - if runtime != "docker" { - return fmt.Errorf("AppArmor is only enabled for 'docker' runtime. Found: %q.", runtime) + if runtime != "docker" && runtime != "remote" { + return fmt.Errorf("AppArmor is only enabled for 'docker' and 'remote' runtimes. Found: %q.", runtime) } return nil diff --git a/pkg/serviceaccount/OWNERS b/pkg/serviceaccount/OWNERS index 019de5ebe2443..c799f8b6ab193 100644 --- a/pkg/serviceaccount/OWNERS +++ b/pkg/serviceaccount/OWNERS @@ -6,3 +6,4 @@ reviewers: - deads2k - mikedanese - ericchiang +- enj diff --git a/pkg/serviceaccount/util.go b/pkg/serviceaccount/util.go index 2ab8484a830bf..df4dc3a9140d1 100644 --- a/pkg/serviceaccount/util.go +++ b/pkg/serviceaccount/util.go @@ -28,7 +28,7 @@ func UserInfo(namespace, name, uid string) user.Info { return &user.DefaultInfo{ Name: apiserverserviceaccount.MakeUsername(namespace, name), UID: uid, - Groups: apiserverserviceaccount.MakeGroupNames(namespace, name), + Groups: apiserverserviceaccount.MakeGroupNames(namespace), } } diff --git a/pkg/util/BUILD b/pkg/util/BUILD index d0ec4a60e5082..d8a7be64ef3ba 100644 --- a/pkg/util/BUILD +++ b/pkg/util/BUILD @@ -60,3 +60,10 @@ filegroup( ], tags = ["automanaged"], ) + +sh_test( + name = "verify-util-pkg", + size = "small", + srcs = ["verify-util-pkg.sh"], + data = glob(["*.go"]), +) diff --git a/pkg/util/mount/BUILD b/pkg/util/mount/BUILD index 66aa2cbb6d071..3bfcded823f71 100644 --- a/pkg/util/mount/BUILD +++ b/pkg/util/mount/BUILD @@ -10,6 +10,7 @@ go_library( name = "go_default_library", srcs = [ "doc.go", + "exec.go", "fake.go", "mount.go", "mount_unsupported.go", @@ -45,10 +46,7 @@ go_test( "//conditions:default": [], }), library = ":go_default_library", - deps = [ - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", - ], + deps = ["//vendor/k8s.io/utils/exec/testing:go_default_library"], ) filegroup( diff --git a/pkg/util/mount/exec.go b/pkg/util/mount/exec.go new file mode 100644 index 0000000000000..716cda0a0c381 --- /dev/null +++ b/pkg/util/mount/exec.go @@ -0,0 +1,50 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import "k8s.io/utils/exec" + +func NewOsExec() Exec { + return &osExec{} +} + +// Real implementation of Exec interface that uses simple util.Exec +type osExec struct{} + +var _ Exec = &osExec{} + +func (e *osExec) Run(cmd string, args ...string) ([]byte, error) { + exe := exec.New() + return exe.Command(cmd, args...).CombinedOutput() +} + +func NewFakeExec(run runHook) *FakeExec { + return &FakeExec{runHook: run} +} + +// Fake for testing. +type FakeExec struct { + runHook runHook +} +type runHook func(cmd string, args ...string) ([]byte, error) + +func (f *FakeExec) Run(cmd string, args ...string) ([]byte, error) { + if f.runHook != nil { + return f.runHook(cmd, args...) + } + return nil, nil +} diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index eab2889951d78..7bdc4fe64603b 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -25,7 +25,6 @@ import ( "strings" "github.com/golang/glog" - "k8s.io/utils/exec" ) const ( @@ -70,6 +69,16 @@ type Interface interface { GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) } +// Exec executes command where mount utilities are. This can be either the host, +// container where kubelet runs or even a remote pod with mount utilities. +// Usual pkg/util/exec interface is not used because kubelet.RunInContainer does +// not provide stdin/stdout/stderr streams. +type Exec interface { + // Run executes a command and returns its stdout + stderr combined in one + // stream. + Run(cmd string, args ...string) ([]byte, error) +} + // Compile-time check to ensure all Mounter implementations satisfy // the mount interface var _ Interface = &Mounter{} @@ -89,7 +98,7 @@ type MountPoint struct { // mounts it otherwise the device is formatted first then mounted. type SafeFormatAndMount struct { Interface - Runner exec.Interface + Exec } // FormatAndMount formats the given disk, if needed, and mounts it. @@ -114,7 +123,6 @@ func GetMountRefs(mounter Interface, mountPath string) ([]string, error) { if err != nil { return nil, err } - // Find the device name. deviceName := "" // If mountPath is symlink, need get its target path. @@ -143,6 +151,39 @@ func GetMountRefs(mounter Interface, mountPath string) ([]string, error) { return refs, nil } +// GetMountRefsByDev finds all references to the device provided +// by mountPath; returns a list of paths. +func GetMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { + mps, err := mounter.List() + if err != nil { + return nil, err + } + slTarget, err := filepath.EvalSymlinks(mountPath) + if err != nil { + slTarget = mountPath + } + + // Finding the device mounted to mountPath + diskDev := "" + for i := range mps { + if slTarget == mps[i].Path { + diskDev = mps[i].Device + break + } + } + + // Find all references to the device. + var refs []string + for i := range mps { + if mps[i].Device == diskDev || mps[i].Device == slTarget { + if mps[i].Path != slTarget { + refs = append(refs, mps[i].Path) + } + } + } + return refs, nil +} + // GetDeviceNameFromMount: given a mnt point, find the device from /proc/mounts // returns the device name, reference count, and error code func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, error) { diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index 65715db6cc210..88396708a4f49 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -412,8 +412,7 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, // Run fsck on the disk to fix repairable issues glog.V(4).Infof("Checking for issues with fsck on disk: %s", source) args := []string{"-a", source} - cmd := mounter.Runner.Command("fsck", args...) - out, err := cmd.CombinedOutput() + out, err := mounter.Exec.Run("fsck", args...) if err != nil { ee, isExitError := err.(utilexec.ExitError) switch { @@ -450,8 +449,7 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, args = []string{"-F", source} } glog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) - cmd := mounter.Runner.Command("mkfs."+fstype, args...) - _, err := cmd.CombinedOutput() + _, err := mounter.Exec.Run("mkfs."+fstype, args...) if err == nil { // the disk has been formatted successfully try to mount it again. glog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) @@ -476,9 +474,8 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, // diskLooksUnformatted uses 'lsblk' to see if the given disk is unformated func (mounter *SafeFormatAndMount) getDiskFormat(disk string) (string, error) { args := []string{"-n", "-o", "FSTYPE", disk} - cmd := mounter.Runner.Command("lsblk", args...) glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args) - dataOut, err := cmd.CombinedOutput() + dataOut, err := mounter.Exec.Run("lsblk", args...) output := string(dataOut) glog.V(4).Infof("Output: %q", output) diff --git a/pkg/util/mount/mount_linux_test.go b/pkg/util/mount/mount_linux_test.go index e13cb7a0f9886..ccd5c46b66a94 100644 --- a/pkg/util/mount/mount_linux_test.go +++ b/pkg/util/mount/mount_linux_test.go @@ -19,6 +19,7 @@ limitations under the License. package mount import ( + "reflect" "strings" "testing" ) @@ -75,24 +76,12 @@ func TestReadProcMountsFrom(t *testing.T) { } func mountPointsEqual(a, b *MountPoint) bool { - if a.Device != b.Device || a.Path != b.Path || a.Type != b.Type || !slicesEqual(a.Opts, b.Opts) || a.Pass != b.Pass || a.Freq != b.Freq { + if a.Device != b.Device || a.Path != b.Path || a.Type != b.Type || !reflect.DeepEqual(a.Opts, b.Opts) || a.Pass != b.Pass || a.Freq != b.Freq { return false } return true } -func slicesEqual(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} - func TestGetMountRefs(t *testing.T) { fm := &FakeMounter{ MountPoints: []MountPoint{ @@ -181,3 +170,41 @@ func TestGetDeviceNameFromMount(t *testing.T) { } } } + +func TestGetMountRefsByDev(t *testing.T) { + fm := &FakeMounter{ + MountPoints: []MountPoint{ + {Device: "/dev/sdb", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd"}, + {Device: "/dev/sdb", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2"}, + }, + } + + tests := []struct { + mountPath string + expectedRefs []string + }{ + { + "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd", + []string{ + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod", + }, + }, + { + "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2", + []string{ + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1", + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2", + }, + }, + } + + for i, test := range tests { + + if refs, err := GetMountRefsByDev(fm, test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) { + t.Errorf("%d. getMountRefsByDev(%q) = %v, %v; expected %v, nil", i, test.mountPath, refs, err, test.expectedRefs) + } + } +} diff --git a/pkg/util/mount/safe_format_and_mount_test.go b/pkg/util/mount/safe_format_and_mount_test.go index 3ea9d575b1c59..72b768f3bf436 100644 --- a/pkg/util/mount/safe_format_and_mount_test.go +++ b/pkg/util/mount/safe_format_and_mount_test.go @@ -21,7 +21,6 @@ import ( "runtime" "testing" - "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" ) @@ -181,40 +180,30 @@ func TestSafeFormatAndMount(t *testing.T) { } for _, test := range tests { - commandScripts := []fakeexec.FakeCommandAction{} - for _, expected := range test.execScripts { - ecmd := expected.command - eargs := expected.args - output := expected.output - err := expected.err - commandScript := func(cmd string, args ...string) exec.Cmd { - if cmd != ecmd { - t.Errorf("Unexpected command %s. Expecting %s", cmd, ecmd) - } - - for j := range args { - if args[j] != eargs[j] { - t.Errorf("Unexpected args %v. Expecting %v", args, eargs) - } - } - fake := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ - func() ([]byte, error) { return []byte(output), err }, - }, + execCallCount := 0 + execCallback := func(cmd string, args ...string) ([]byte, error) { + if len(test.execScripts) <= execCallCount { + t.Errorf("Unexpected command: %s %v", cmd, args) + return nil, nil + } + script := test.execScripts[execCallCount] + execCallCount++ + if script.command != cmd { + t.Errorf("Unexpected command %s. Expecting %s", cmd, script.command) + } + for j := range args { + if args[j] != script.args[j] { + t.Errorf("Unexpected args %v. Expecting %v", args, script.args) } - return fakeexec.InitFakeCmd(&fake, cmd, args...) } - commandScripts = append(commandScripts, commandScript) - } - - fake := fakeexec.FakeExec{ - CommandScript: commandScripts, + return []byte(script.output), script.err } fakeMounter := ErrorMounter{&FakeMounter{}, 0, test.mountErrs} + fakeExec := NewFakeExec(execCallback) mounter := SafeFormatAndMount{ Interface: &fakeMounter, - Runner: &fake, + Exec: fakeExec, } device := "/dev/foo" diff --git a/pkg/util/verify-util-pkg.sh b/pkg/util/verify-util-pkg.sh new file mode 100755 index 0000000000000..755924a099af3 --- /dev/null +++ b/pkg/util/verify-util-pkg.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# verify-util-pkg.sh checks whether *.go except doc.go in pkg/util have been moved into +# sub-pkgs, see issue #15634. + +set -o errexit +set -o nounset +set -o pipefail + +BASH_DIR=$(dirname "${BASH_SOURCE}") + +find_go_files() { + find . -maxdepth 1 -not \( \ + \( \ + -wholename './doc.go' \ + \) -prune \ + \) -name '*.go' +} + +ret=0 + +pushd "${BASH_DIR}" > /dev/null + for path in `find_go_files`; do + file=$(basename $path) + echo "Found pkg/util/${file}, but should be moved into util sub-pkgs." 1>&2 + ret=1 + done +popd > /dev/null + +if [[ ${ret} > 0 ]]; then + exit ${ret} +fi + +echo "Util Package Verified." diff --git a/pkg/util/version/version.go b/pkg/util/version/version.go index 327f2e67f4080..e8cd0cecfa7f1 100644 --- a/pkg/util/version/version.go +++ b/pkg/util/version/version.go @@ -121,11 +121,39 @@ func MustParseSemantic(str string) *Version { return v } +// Major returns the major release number +func (v *Version) Major() uint { + return v.components[0] +} + +// Minor returns the minor release number +func (v *Version) Minor() uint { + return v.components[1] +} + +// Patch returns the patch release number if v is a Semantic Version, or 0 +func (v *Version) Patch() uint { + if len(v.components) < 3 { + return 0 + } + return v.components[2] +} + // BuildMetadata returns the build metadata, if v is a Semantic Version, or "" func (v *Version) BuildMetadata() string { return v.buildMetadata } +// PreRelease returns the prerelease metadata, if v is a Semantic Version, or "" +func (v *Version) PreRelease() string { + return v.preRelease +} + +// Components returns the version number components +func (v *Version) Components() []uint { + return v.components +} + // String converts a Version back to a string; note that for versions parsed with // ParseGeneric, this will not include the trailing uninterpreted portion of the version // number. diff --git a/pkg/util/version/version_test.go b/pkg/util/version/version_test.go index 555c59b4b75f5..eb231b1e2bda6 100644 --- a/pkg/util/version/version_test.go +++ b/pkg/util/version/version_test.go @@ -18,6 +18,7 @@ package version import ( "fmt" + "reflect" "testing" ) @@ -257,3 +258,75 @@ func TestBadGenericVersions(t *testing.T) { } } } + +func TestComponents(t *testing.T) { + + var tests = []struct { + version string + semver bool + expectedComponents []uint + expectedMajor uint + expectedMinor uint + expectedPatch uint + expectedPreRelease string + expectedBuildMetadata string + }{ + { + version: "1.0.2", + semver: true, + expectedComponents: []uint{1, 0, 2}, + expectedMajor: 1, + expectedMinor: 0, + expectedPatch: 2, + }, + { + version: "1.0.2-alpha+001", + semver: true, + expectedComponents: []uint{1, 0, 2}, + expectedMajor: 1, + expectedMinor: 0, + expectedPatch: 2, + expectedPreRelease: "alpha", + expectedBuildMetadata: "001", + }, + { + version: "1.2", + semver: false, + expectedComponents: []uint{1, 2}, + expectedMajor: 1, + expectedMinor: 2, + }, + { + version: "1.0.2-beta+exp.sha.5114f85", + semver: true, + expectedComponents: []uint{1, 0, 2}, + expectedMajor: 1, + expectedMinor: 0, + expectedPatch: 2, + expectedPreRelease: "beta", + expectedBuildMetadata: "exp.sha.5114f85", + }, + } + + for _, test := range tests { + version, _ := parse(test.version, test.semver) + if !reflect.DeepEqual(test.expectedComponents, version.Components()) { + t.Error("parse returned un'expected components") + } + if test.expectedMajor != version.Major() { + t.Errorf("parse returned version.Major %d, expected %d", test.expectedMajor, version.Major()) + } + if test.expectedMinor != version.Minor() { + t.Errorf("parse returned version.Minor %d, expected %d", test.expectedMinor, version.Minor()) + } + if test.expectedPatch != version.Patch() { + t.Errorf("parse returned version.Patch %d, expected %d", test.expectedPatch, version.Patch()) + } + if test.expectedPreRelease != version.PreRelease() { + t.Errorf("parse returned version.PreRelease %s, expected %s", test.expectedPreRelease, version.PreRelease()) + } + if test.expectedBuildMetadata != version.BuildMetadata() { + t.Errorf("parse returned version.BuildMetadata %s, expected %s", test.expectedBuildMetadata, version.BuildMetadata()) + } + } +} diff --git a/pkg/volume/aws_ebs/BUILD b/pkg/volume/aws_ebs/BUILD index 7b623b5f43a45..a8d96f4b4ba10 100644 --- a/pkg/volume/aws_ebs/BUILD +++ b/pkg/volume/aws_ebs/BUILD @@ -27,7 +27,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/aws_ebs/attacher.go b/pkg/volume/aws_ebs/attacher.go index 2ccf0aa885f2e..bf14142469011 100644 --- a/pkg/volume/aws_ebs/attacher.go +++ b/pkg/volume/aws_ebs/attacher.go @@ -24,12 +24,13 @@ import ( "time" "github.com/golang/glog" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" + "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) type awsElasticBlockStoreAttacher struct { @@ -54,7 +55,7 @@ func (plugin *awsElasticBlockStorePlugin) NewAttacher() (volume.Attacher, error) } func (plugin *awsElasticBlockStorePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mount.GetMountRefs(mounter, deviceMountPath) } @@ -142,7 +143,7 @@ func (attacher *awsElasticBlockStoreAttacher) BulkVerifyVolumes(volumesByNode ma return volumesAttachedCheck, nil } -func (attacher *awsElasticBlockStoreAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { +func (attacher *awsElasticBlockStoreAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { volumeSource, _, err := getVolumeSource(spec) if err != nil { return "", err @@ -199,7 +200,7 @@ func (attacher *awsElasticBlockStoreAttacher) GetDeviceMountPath( // FIXME: this method can be further pruned. func (attacher *awsElasticBlockStoreAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { - mounter := attacher.host.GetMounter() + mounter := attacher.host.GetMounter(awsElasticBlockStorePluginName) notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { if os.IsNotExist(err) { @@ -222,7 +223,7 @@ func (attacher *awsElasticBlockStoreAttacher) MountDevice(spec *volume.Spec, dev options = append(options, "ro") } if notMnt { - diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()} + diskMounter := volumehelper.NewSafeFormatAndMountFromHost(awsElasticBlockStorePluginName, attacher.host) mountOptions := volume.MountOptionFromSpec(spec, options...) err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions) if err != nil { @@ -247,7 +248,7 @@ func (plugin *awsElasticBlockStorePlugin) NewDetacher() (volume.Detacher, error) } return &awsElasticBlockStoreDetacher{ - mounter: plugin.host.GetMounter(), + mounter: plugin.host.GetMounter(plugin.GetPluginName()), awsVolumes: awsCloud, }, nil } diff --git a/pkg/volume/aws_ebs/aws_ebs.go b/pkg/volume/aws_ebs/aws_ebs.go index 84498e4b9b997..7a81f7605e71e 100644 --- a/pkg/volume/aws_ebs/aws_ebs.go +++ b/pkg/volume/aws_ebs/aws_ebs.go @@ -35,7 +35,6 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) // This is the primary entrypoint for volume plugins. @@ -104,7 +103,7 @@ func (plugin *awsElasticBlockStorePlugin) GetAccessModes() []v1.PersistentVolume func (plugin *awsElasticBlockStorePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newMounterInternal(spec, pod.UID, &AWSDiskUtil{}, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod.UID, &AWSDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *awsElasticBlockStorePlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager ebsManager, mounter mount.Interface) (volume.Mounter, error) { @@ -135,12 +134,12 @@ func (plugin *awsElasticBlockStorePlugin) newMounterInternal(spec *volume.Spec, }, fsType: fsType, readOnly: readOnly, - diskMounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(), Runner: exec.New()}}, nil + diskMounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host)}, nil } func (plugin *awsElasticBlockStorePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newUnmounterInternal(volName, podUID, &AWSDiskUtil{}, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, &AWSDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *awsElasticBlockStorePlugin) newUnmounterInternal(volName string, podUID types.UID, manager ebsManager, mounter mount.Interface) (volume.Unmounter, error) { @@ -199,7 +198,7 @@ func getVolumeSource( } func (plugin *awsElasticBlockStorePlugin) ConstructVolumeSpec(volName, mountPath string) (*volume.Spec, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName()) volumeID, err := mounter.GetDeviceNameFromMount(mountPath, pluginDir) if err != nil { diff --git a/pkg/volume/aws_ebs/aws_ebs_test.go b/pkg/volume/aws_ebs/aws_ebs_test.go index 3766b21254a1e..85538b3649a4a 100644 --- a/pkg/volume/aws_ebs/aws_ebs_test.go +++ b/pkg/volume/aws_ebs/aws_ebs_test.go @@ -40,7 +40,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/aws-ebs") if err != nil { @@ -64,7 +64,7 @@ func TestGetAccessModes(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/aws-ebs") if err != nil { @@ -113,7 +113,7 @@ func TestPlugin(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/aws-ebs") if err != nil { @@ -251,7 +251,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, clientset, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, clientset, nil)) plug, _ := plugMgr.FindPluginByName(awsElasticBlockStorePluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes @@ -271,7 +271,7 @@ func TestMounterAndUnmounterTypeAssert(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/aws-ebs") if err != nil { diff --git a/pkg/volume/azure_dd/BUILD b/pkg/volume/azure_dd/BUILD index db2313fe564bf..ba57ee1ea6ab3 100644 --- a/pkg/volume/azure_dd/BUILD +++ b/pkg/volume/azure_dd/BUILD @@ -24,6 +24,7 @@ go_library( "//pkg/util/strings:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", + "//pkg/volume/util/volumehelper:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/arm/compute:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/arm/storage:go_default_library", "//vendor/github.com/golang/glog:go_default_library", @@ -33,7 +34,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -62,7 +62,5 @@ go_test( "//pkg/volume/testing:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/client-go/util/testing:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", ], ) diff --git a/pkg/volume/azure_dd/attacher.go b/pkg/volume/azure_dd/attacher.go index cf71f65b1a2c8..0ef1c6c04779b 100644 --- a/pkg/volume/azure_dd/attacher.go +++ b/pkg/volume/azure_dd/attacher.go @@ -35,7 +35,7 @@ import ( "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" + "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) type azureDiskDetacher struct { @@ -150,7 +150,7 @@ func (a *azureDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName ty return volumesAttachedCheck, nil } -func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { +func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { var err error lun, err := strconv.Atoi(devicePath) if err != nil { @@ -170,9 +170,7 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, newDevicePath := "" err = wait.Poll(1*time.Second, timeout, func() (bool, error) { - exe := exec.New() - - if newDevicePath, err = findDiskByLun(lun, io, exe); err != nil { + if newDevicePath, err = findDiskByLun(lun, io); err != nil { return false, fmt.Errorf("azureDisk - WaitForAttach ticker failed node (%s) disk (%s) lun(%v) err(%s)", nodeName, diskName, lun, err) } @@ -181,7 +179,7 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, // the curent sequence k8s uses for unformated disk (check-disk, mount, fail, mkfs.extX) hangs on // Azure Managed disk scsi interface. this is a hack and will be replaced once we identify and solve // the root case on Azure. - formatIfNotFormatted(newDevicePath, *volumeSource.FSType) + formatIfNotFormatted(newDevicePath, *volumeSource.FSType, a.plugin.host.GetExec(a.plugin.GetPluginName())) return true, nil } @@ -211,7 +209,7 @@ func (a *azureDiskAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error } func (attacher *azureDiskAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { - mounter := attacher.plugin.host.GetMounter() + mounter := attacher.plugin.host.GetMounter(azureDataDiskPluginName) notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { @@ -232,7 +230,7 @@ func (attacher *azureDiskAttacher) MountDevice(spec *volume.Spec, devicePath str options := []string{} if notMnt { - diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()} + diskMounter := volumehelper.NewSafeFormatAndMountFromHost(azureDataDiskPluginName, attacher.plugin.host) mountOptions := volume.MountOptionFromSpec(spec, options...) err = diskMounter.FormatAndMount(devicePath, deviceMountPath, *volumeSource.FSType, mountOptions) if err != nil { @@ -277,7 +275,7 @@ func (d *azureDiskDetacher) Detach(diskURI string, nodeName types.NodeName) erro // UnmountDevice unmounts the volume on the node func (detacher *azureDiskDetacher) UnmountDevice(deviceMountPath string) error { - err := volumeutil.UnmountPath(deviceMountPath, detacher.plugin.host.GetMounter()) + err := volumeutil.UnmountPath(deviceMountPath, detacher.plugin.host.GetMounter(detacher.plugin.GetPluginName())) if err == nil { glog.V(4).Infof("azureDisk - Device %s was unmounted", deviceMountPath) } else { diff --git a/pkg/volume/azure_dd/azure_common.go b/pkg/volume/azure_dd/azure_common.go index 7c36b3984a697..ec3d69fb76894 100644 --- a/pkg/volume/azure_dd/azure_common.go +++ b/pkg/volume/azure_dd/azure_common.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "os" "path" - "regexp" "strconv" libstrings "strings" @@ -35,7 +34,6 @@ import ( "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" - "k8s.io/utils/exec" ) const ( @@ -158,6 +156,7 @@ type ioHandler interface { ReadDir(dirname string) ([]os.FileInfo, error) WriteFile(filename string, data []byte, perm os.FileMode) error Readlink(name string) (string, error) + ReadFile(filename string) ([]byte, error) } //TODO: check if priming the iscsi interface is actually needed @@ -176,6 +175,10 @@ func (handler *osIOHandler) Readlink(name string) (string, error) { return os.Readlink(name) } +func (handler *osIOHandler) ReadFile(filename string) ([]byte, error) { + return ioutil.ReadFile(filename) +} + // exclude those used by azure as resource and OS root in /dev/disk/azure func listAzureDiskPath(io ioHandler) []string { azureDiskPath := "/dev/disk/azure/" @@ -209,13 +212,13 @@ func scsiHostRescan(io ioHandler) { } } -func findDiskByLun(lun int, io ioHandler, exe exec.Interface) (string, error) { +func findDiskByLun(lun int, io ioHandler) (string, error) { azureDisks := listAzureDiskPath(io) - return findDiskByLunWithConstraint(lun, io, exe, azureDisks) + return findDiskByLunWithConstraint(lun, io, azureDisks) } // finds a device mounted to "current" node -func findDiskByLunWithConstraint(lun int, io ioHandler, exe exec.Interface, azureDisks []string) (string, error) { +func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (string, error) { var err error sys_path := "/sys/bus/scsi/devices" if dirs, err := io.ReadDir(sys_path); err == nil { @@ -237,18 +240,30 @@ func findDiskByLunWithConstraint(lun int, io ioHandler, exe exec.Interface, azur if lun == l { // find the matching LUN // read vendor and model to ensure it is a VHD disk - vendor := path.Join(sys_path, name, "vendor") - model := path.Join(sys_path, name, "model") - out, err := exe.Command("cat", vendor, model).CombinedOutput() + vendorPath := path.Join(sys_path, name, "vendor") + vendorBytes, err := io.ReadFile(vendorPath) + if err != nil { + glog.Errorf("failed to read device vendor, err: %v", err) + continue + } + vendor := libstrings.TrimSpace(string(vendorBytes)) + if libstrings.ToUpper(vendor) != "MSFT" { + glog.V(4).Infof("vendor doesn't match VHD, got %s", vendor) + continue + } + + modelPath := path.Join(sys_path, name, "model") + modelBytes, err := io.ReadFile(modelPath) if err != nil { - glog.V(4).Infof("azure disk - failed to cat device vendor and model, err: %v", err) + glog.Errorf("failed to read device model, err: %v", err) continue } - matched, err := regexp.MatchString("^MSFT[ ]{0,}\nVIRTUAL DISK[ ]{0,}\n$", libstrings.ToUpper(string(out))) - if err != nil || !matched { - glog.V(4).Infof("azure disk - doesn't match VHD, output %v, error %v", string(out), err) + model := libstrings.TrimSpace(string(modelBytes)) + if libstrings.ToUpper(model) != "VIRTUAL DISK" { + glog.V(4).Infof("model doesn't match VHD, got %s", model) continue } + // find a disk, validate name dir := path.Join(sys_path, name, "block") if dev, err := io.ReadDir(dir); err == nil { @@ -270,8 +285,8 @@ func findDiskByLunWithConstraint(lun int, io ioHandler, exe exec.Interface, azur return "", err } -func formatIfNotFormatted(disk string, fstype string) { - notFormatted, err := diskLooksUnformatted(disk) +func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) { + notFormatted, err := diskLooksUnformatted(disk, exec) if err == nil && notFormatted { args := []string{disk} // Disk is unformatted so format it. @@ -283,9 +298,8 @@ func formatIfNotFormatted(disk string, fstype string) { args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", disk} } glog.Infof("azureDisk - Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", disk, fstype, args) - runner := exec.New() - cmd := runner.Command("mkfs."+fstype, args...) - _, err := cmd.CombinedOutput() + + _, err := exec.Run("mkfs."+fstype, args...) if err == nil { // the disk has been formatted successfully try to mount it again. glog.Infof("azureDisk - Disk successfully formatted (mkfs): %s - %s %s", fstype, disk, "tt") @@ -300,12 +314,10 @@ func formatIfNotFormatted(disk string, fstype string) { } } -func diskLooksUnformatted(disk string) (bool, error) { +func diskLooksUnformatted(disk string, exec mount.Exec) (bool, error) { args := []string{"-nd", "-o", "FSTYPE", disk} - runner := exec.New() - cmd := runner.Command("lsblk", args...) glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args) - dataOut, err := cmd.CombinedOutput() + dataOut, err := exec.Run("lsblk", args...) if err != nil { glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) return false, err diff --git a/pkg/volume/azure_dd/azure_common_test.go b/pkg/volume/azure_dd/azure_common_test.go index d026e2b92eeea..00a35b98caa57 100644 --- a/pkg/volume/azure_dd/azure_common_test.go +++ b/pkg/volume/azure_dd/azure_common_test.go @@ -19,11 +19,9 @@ package azure_dd import ( "fmt" "os" + "strings" "testing" "time" - - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" ) type fakeFileInfo struct { @@ -107,27 +105,18 @@ func (handler *fakeIOHandler) Readlink(name string) (string, error) { return "/dev/azure/disk/sda", nil } -func TestIoHandler(t *testing.T) { - var fcmd fakeexec.FakeCmd - fcmd = fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ - // cat - func() ([]byte, error) { - return []byte("Msft \nVirtual Disk \n"), nil - }, - // cat - func() ([]byte, error) { - return []byte("Msft \nVirtual Disk \n"), nil - }, - }, +func (handler *fakeIOHandler) ReadFile(filename string) ([]byte, error) { + if strings.HasSuffix(filename, "vendor") { + return []byte("Msft \n"), nil } - fake := fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, + if strings.HasSuffix(filename, "model") { + return []byte("Virtual Disk \n"), nil } - disk, err := findDiskByLun(lun, &fakeIOHandler{}, &fake) + return nil, fmt.Errorf("unknown file") +} + +func TestIoHandler(t *testing.T) { + disk, err := findDiskByLun(lun, &fakeIOHandler{}) // if no disk matches lun, exit if disk != "/dev/"+devName || err != nil { t.Errorf("no data disk found: disk %v err %v", disk, err) diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index 49b68cdd43a71..80d9b98b17cea 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -190,7 +190,7 @@ func (plugin *azureDataDiskPlugin) NewUnmounter(volName string, podUID types.UID } func (plugin *azureDataDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName()) sourceName, err := mounter.GetDeviceNameFromMount(mountPath, pluginDir) @@ -210,6 +210,6 @@ func (plugin *azureDataDiskPlugin) ConstructVolumeSpec(volumeName, mountPath str } func (plugin *azureDataDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { - m := plugin.host.GetMounter() + m := plugin.host.GetMounter(plugin.GetPluginName()) return mount.GetMountRefs(m, deviceMountPath) } diff --git a/pkg/volume/azure_dd/azure_dd_test.go b/pkg/volume/azure_dd/azure_dd_test.go index 59becdeeadadf..e1d24c9141e7e 100644 --- a/pkg/volume/azure_dd/azure_dd_test.go +++ b/pkg/volume/azure_dd/azure_dd_test.go @@ -33,7 +33,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName(azureDataDiskPluginName) if err != nil { diff --git a/pkg/volume/azure_dd/azure_mounter.go b/pkg/volume/azure_dd/azure_mounter.go index eedb5535f7529..2af044c369409 100644 --- a/pkg/volume/azure_dd/azure_mounter.go +++ b/pkg/volume/azure_dd/azure_mounter.go @@ -63,7 +63,7 @@ func (m *azureDiskMounter) GetPath() string { } func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { - mounter := m.plugin.host.GetMounter() + mounter := m.plugin.host.GetMounter(m.plugin.GetPluginName()) volumeSource, err := getVolumeSource(m.spec) if err != nil { @@ -154,7 +154,7 @@ func (u *azureDiskUnmounter) TearDownAt(dir string) error { } glog.V(4).Infof("azureDisk - TearDownAt: %s", dir) - mounter := u.plugin.host.GetMounter() + mounter := u.plugin.host.GetMounter(u.plugin.GetPluginName()) mountPoint, err := mounter.IsLikelyNotMountPoint(dir) if err != nil { return fmt.Errorf("azureDisk - TearDownAt: %s failed to do IsLikelyNotMountPoint %s", dir, err) diff --git a/pkg/volume/azure_file/azure_file.go b/pkg/volume/azure_file/azure_file.go index 426900a1f289b..2e51e46853a91 100644 --- a/pkg/volume/azure_file/azure_file.go +++ b/pkg/volume/azure_file/azure_file.go @@ -63,12 +63,12 @@ func (plugin *azureFilePlugin) GetPluginName() string { } func (plugin *azureFilePlugin) GetVolumeName(spec *volume.Spec) (string, error) { - volumeSource, _, err := getVolumeSource(spec) + share, _, err := getVolumeSource(spec) if err != nil { return "", err } - return volumeSource.ShareName, nil + return share, nil } func (plugin *azureFilePlugin) CanSupport(spec *volume.Spec) bool { @@ -98,15 +98,15 @@ func (plugin *azureFilePlugin) GetAccessModes() []v1.PersistentVolumeAccessMode } func (plugin *azureFilePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - return plugin.newMounterInternal(spec, pod, &azureSvc{}, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod, &azureSvc{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *azureFilePlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, util azureUtil, mounter mount.Interface) (volume.Mounter, error) { - source, readOnly, err := getVolumeSource(spec) + share, readOnly, err := getVolumeSource(spec) if err != nil { return nil, err } - + secretName, secretNamespace, err := getSecretNameAndNamespace(spec, pod.Namespace) return &azureFileMounter{ azureFile: &azureFile{ volName: spec.Name(), @@ -115,16 +115,17 @@ func (plugin *azureFilePlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod plugin: plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host)), }, - util: util, - secretName: source.SecretName, - shareName: source.ShareName, - readOnly: readOnly, - mountOptions: volume.MountOptionFromSpec(spec), + util: util, + secretNamespace: secretNamespace, + secretName: secretName, + shareName: share, + readOnly: readOnly, + mountOptions: volume.MountOptionFromSpec(spec), }, nil } func (plugin *azureFilePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *azureFilePlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) { @@ -166,11 +167,12 @@ func (azureFileVolume *azureFile) GetPath() string { type azureFileMounter struct { *azureFile - util azureUtil - secretName string - shareName string - readOnly bool - mountOptions []string + util azureUtil + secretName string + secretNamespace string + shareName string + readOnly bool + mountOptions []string } var _ volume.Mounter = &azureFileMounter{} @@ -205,7 +207,7 @@ func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error { return nil } var accountKey, accountName string - if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.pod.Namespace, b.secretName); err != nil { + if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.secretNamespace, b.secretName); err != nil { return err } os.MkdirAll(dir, 0700) @@ -260,16 +262,43 @@ func (c *azureFileUnmounter) TearDownAt(dir string) error { return util.UnmountPath(dir, c.mounter) } -func getVolumeSource( - spec *volume.Spec) (*v1.AzureFileVolumeSource, bool, error) { +func getVolumeSource(spec *volume.Spec) (string, bool, error) { if spec.Volume != nil && spec.Volume.AzureFile != nil { - return spec.Volume.AzureFile, spec.Volume.AzureFile.ReadOnly, nil + share := spec.Volume.AzureFile.ShareName + readOnly := spec.Volume.AzureFile.ReadOnly + return share, readOnly, nil } else if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.AzureFile != nil { - return spec.PersistentVolume.Spec.AzureFile, spec.ReadOnly, nil + share := spec.PersistentVolume.Spec.AzureFile.ShareName + readOnly := spec.ReadOnly + return share, readOnly, nil + } + return "", false, fmt.Errorf("Spec does not reference an AzureFile volume type") +} + +func getSecretNameAndNamespace(spec *volume.Spec, defaultNamespace string) (string, string, error) { + secretName := "" + secretNamespace := "" + if spec.Volume != nil && spec.Volume.AzureFile != nil { + secretName = spec.Volume.AzureFile.SecretName + secretNamespace = defaultNamespace + + } else if spec.PersistentVolume != nil && + spec.PersistentVolume.Spec.AzureFile != nil { + secretNamespace = defaultNamespace + if spec.PersistentVolume.Spec.AzureFile.SecretNamespace != nil { + secretNamespace = *spec.PersistentVolume.Spec.AzureFile.SecretNamespace + } + secretName = spec.PersistentVolume.Spec.AzureFile.SecretName + } else { + return "", "", fmt.Errorf("Spec does not reference an AzureFile volume type") + } + + if len(secretNamespace) == 0 { + return "", "", fmt.Errorf("invalid Azure volume: nil namespace") } + return secretName, secretNamespace, nil - return nil, false, fmt.Errorf("Spec does not reference an AzureFile volume type") } func getAzureCloud(cloudProvider cloudprovider.Interface) (*azure.Cloud, error) { diff --git a/pkg/volume/azure_file/azure_file_test.go b/pkg/volume/azure_file/azure_file_test.go index bb1a5ad3ad243..d6fc125963e64 100644 --- a/pkg/volume/azure_file/azure_file_test.go +++ b/pkg/volume/azure_file/azure_file_test.go @@ -41,7 +41,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/azure-file") if err != nil { @@ -53,7 +53,7 @@ func TestCanSupport(t *testing.T) { if !plug.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{AzureFile: &v1.AzureFileVolumeSource{}}}}) { t.Errorf("Expected true") } - if !plug.CanSupport(&volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{AzureFile: &v1.AzureFileVolumeSource{}}}}}) { + if !plug.CanSupport(&volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{AzureFile: &v1.AzureFilePersistentVolumeSource{}}}}}) { t.Errorf("Expected true") } } @@ -65,7 +65,7 @@ func TestGetAccessModes(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/azure-file") if err != nil { @@ -131,7 +131,7 @@ func TestPluginWithOtherCloudProvider(t *testing.T) { func testPlugin(t *testing.T, tmpDir string, volumeHost volume.VolumeHost) { plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumeHost) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumeHost) plug, err := plugMgr.FindPluginByName("kubernetes.io/azure-file") if err != nil { @@ -204,7 +204,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { }, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ - AzureFile: &v1.AzureFileVolumeSource{}, + AzureFile: &v1.AzureFilePersistentVolumeSource{}, }, ClaimRef: &v1.ObjectReference{ Name: "claimA", @@ -228,13 +228,16 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { client := fake.NewSimpleClientset(pv, claim) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost("/tmp/fake", client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost("/tmp/fake", client, nil)) plug, _ := plugMgr.FindPluginByName(azureFilePluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes spec := volume.NewSpecFromPersistentVolume(pv, true) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{}) + if mounter == nil { + t.Fatalf("Got a nil Mounter") + } if !mounter.GetAttributes().ReadOnly { t.Errorf("Expected true for mounter.IsReadOnly") @@ -257,7 +260,7 @@ func TestMounterAndUnmounterTypeAssert(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/azure-file") if err != nil { @@ -284,3 +287,83 @@ func TestMounterAndUnmounterTypeAssert(t *testing.T) { t.Errorf("Volume Unmounter can be type-assert to Mounter") } } + +type testcase struct { + name string + defaultNs string + spec *volume.Spec + // Expected return of the test + expectedName string + expectedNs string + expectedError error +} + +func TestGetSecretNameAndNamespaceForPV(t *testing.T) { + secretNs := "ns" + tests := []testcase{ + { + name: "persistent volume source", + defaultNs: "default", + spec: &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AzureFile: &v1.AzureFilePersistentVolumeSource{ + ShareName: "share", + SecretName: "name", + SecretNamespace: &secretNs, + }, + }, + }, + }, + }, + expectedName: "name", + expectedNs: "ns", + expectedError: nil, + }, + { + name: "persistent volume source without namespace", + defaultNs: "default", + spec: &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + AzureFile: &v1.AzureFilePersistentVolumeSource{ + ShareName: "share", + SecretName: "name", + }, + }, + }, + }, + }, + expectedName: "name", + expectedNs: "default", + expectedError: nil, + }, + { + name: "pod volume source", + defaultNs: "default", + spec: &volume.Spec{ + Volume: &v1.Volume{ + VolumeSource: v1.VolumeSource{ + AzureFile: &v1.AzureFileVolumeSource{ + ShareName: "share", + SecretName: "name", + }, + }, + }, + }, + expectedName: "name", + expectedNs: "default", + expectedError: nil, + }, + } + for _, testcase := range tests { + resultName, resultNs, err := getSecretNameAndNamespace(testcase.spec, testcase.defaultNs) + if err != testcase.expectedError || resultName != testcase.expectedName || resultNs != testcase.expectedNs { + t.Errorf("%s failed: expected err=%v ns=%q name=%q, got %v/%q/%q", testcase.name, testcase.expectedError, testcase.expectedNs, testcase.expectedName, + err, resultNs, resultName) + } + } + +} diff --git a/pkg/volume/azure_file/azure_provision.go b/pkg/volume/azure_file/azure_provision.go index a74fbc2cefa88..bebec4599fb5e 100644 --- a/pkg/volume/azure_file/azure_provision.go +++ b/pkg/volume/azure_file/azure_provision.go @@ -63,15 +63,13 @@ func (plugin *azureFilePlugin) newDeleterInternal(spec *volume.Spec, util azureU if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.AzureFile == nil { return nil, fmt.Errorf("invalid PV spec") } - pvSpec := spec.PersistentVolume - if pvSpec.Spec.ClaimRef.Namespace == "" { - glog.Errorf("namespace cannot be nil") - return nil, fmt.Errorf("invalid PV spec: nil namespace") + + secretName, secretNamespace, err := getSecretNameAndNamespace(spec, spec.PersistentVolume.Spec.ClaimRef.Namespace) + if err != nil { + return nil, err } - nameSpace := pvSpec.Spec.ClaimRef.Namespace - secretName := pvSpec.Spec.AzureFile.SecretName - shareName := pvSpec.Spec.AzureFile.ShareName - if accountName, accountKey, err := util.GetAzureCredentials(plugin.host, nameSpace, secretName); err != nil { + shareName := spec.PersistentVolume.Spec.AzureFile.ShareName + if accountName, accountKey, err := util.GetAzureCredentials(plugin.host, secretNamespace, secretName); err != nil { return nil, err } else { return &azureFileDeleter{ @@ -144,7 +142,7 @@ func (a *azureFileProvisioner) Provision() (*v1.PersistentVolume, error) { capacity := a.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] requestBytes := capacity.Value() requestGB := int(volume.RoundUpSize(requestBytes, 1024*1024*1024)) - + secretNamespace := a.options.PVC.Namespace // Apply ProvisionerParameters (case-insensitive). We leave validation of // the values to the cloud provider. for k, v := range a.options.Parameters { @@ -155,6 +153,8 @@ func (a *azureFileProvisioner) Provision() (*v1.PersistentVolume, error) { location = v case "storageaccount": account = v + case "secretnamespace": + secretNamespace = v default: return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, a.plugin.GetPluginName()) } @@ -168,8 +168,9 @@ func (a *azureFileProvisioner) Provision() (*v1.PersistentVolume, error) { if err != nil { return nil, err } + // create a secret for storage account and key - secretName, err := a.util.SetAzureCredentials(a.plugin.host, a.options.PVC.Namespace, account, key) + secretName, err := a.util.SetAzureCredentials(a.plugin.host, secretNamespace, account, key) if err != nil { return nil, err } @@ -189,9 +190,10 @@ func (a *azureFileProvisioner) Provision() (*v1.PersistentVolume, error) { v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", requestGB)), }, PersistentVolumeSource: v1.PersistentVolumeSource{ - AzureFile: &v1.AzureFileVolumeSource{ - SecretName: secretName, - ShareName: name, + AzureFile: &v1.AzureFilePersistentVolumeSource{ + SecretName: secretName, + ShareName: name, + SecretNamespace: &secretNamespace, }, }, }, diff --git a/pkg/volume/cephfs/cephfs.go b/pkg/volume/cephfs/cephfs.go index 5ea4e6137e7d7..5c0fdd04a747d 100644 --- a/pkg/volume/cephfs/cephfs.go +++ b/pkg/volume/cephfs/cephfs.go @@ -56,12 +56,12 @@ func (plugin *cephfsPlugin) GetPluginName() string { } func (plugin *cephfsPlugin) GetVolumeName(spec *volume.Spec) (string, error) { - volumeSource, _, err := getVolumeSource(spec) + mon, _, _, _, _, err := getVolumeSource(spec) if err != nil { return "", err } - return fmt.Sprintf("%v", volumeSource.Monitors), nil + return fmt.Sprintf("%v", mon), nil } func (plugin *cephfsPlugin) CanSupport(spec *volume.Spec) bool { @@ -89,62 +89,60 @@ func (plugin *cephfsPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { } func (plugin *cephfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - cephvs, _, err := getVolumeSource(spec) + secretName, secretNs, err := getSecretNameAndNamespace(spec, pod.Namespace) if err != nil { return nil, err } secret := "" - if cephvs.SecretRef != nil { + if len(secretName) > 0 && len(secretNs) > 0 { + // if secret is provideded, retrieve it kubeClient := plugin.host.GetKubeClient() if kubeClient == nil { return nil, fmt.Errorf("Cannot get kube client") } - - secretName, err := kubeClient.Core().Secrets(pod.Namespace).Get(cephvs.SecretRef.Name, metav1.GetOptions{}) + secrets, err := kubeClient.Core().Secrets(secretNs).Get(secretName, metav1.GetOptions{}) if err != nil { - err = fmt.Errorf("Couldn't get secret %v/%v err: %v", pod.Namespace, cephvs.SecretRef, err) + err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNs, secretName, err) return nil, err } - for name, data := range secretName.Data { + for name, data := range secrets.Data { secret = string(data) glog.V(4).Infof("found ceph secret info: %s", name) } } - return plugin.newMounterInternal(spec, pod.UID, plugin.host.GetMounter(), secret) + return plugin.newMounterInternal(spec, pod.UID, plugin.host.GetMounter(plugin.GetPluginName()), secret) } func (plugin *cephfsPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, mounter mount.Interface, secret string) (volume.Mounter, error) { - cephvs, _, err := getVolumeSource(spec) + mon, path, id, secretFile, readOnly, err := getVolumeSource(spec) if err != nil { return nil, err } - id := cephvs.User if id == "" { id = "admin" } - path := cephvs.Path if path == "" { path = "/" } if !strings.HasPrefix(path, "/") { path = "/" + path } - secret_file := cephvs.SecretFile - if secret_file == "" { - secret_file = "/etc/ceph/" + id + ".secret" + + if secretFile == "" { + secretFile = "/etc/ceph/" + id + ".secret" } return &cephfsMounter{ cephfs: &cephfs{ podUID: podUID, volName: spec.Name(), - mon: cephvs.Monitors, + mon: mon, path: path, secret: secret, id: id, - secret_file: secret_file, - readonly: cephvs.ReadOnly, + secret_file: secretFile, + readonly: readOnly, mounter: mounter, plugin: plugin, mountOptions: volume.MountOptionFromSpec(spec), @@ -153,7 +151,7 @@ func (plugin *cephfsPlugin) newMounterInternal(spec *volume.Spec, podUID types.U } func (plugin *cephfsPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *cephfsPlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) { @@ -260,7 +258,7 @@ func (cephfsVolume *cephfsUnmounter) TearDownAt(dir string) error { return util.UnmountPath(dir, cephfsVolume.mounter) } -// GatePath creates global mount path +// GetPath creates global mount path func (cephfsVolume *cephfs) GetPath() string { name := cephfsPluginName return cephfsVolume.plugin.host.GetPodVolumeDir(cephfsVolume.podUID, utilstrings.EscapeQualifiedNameForDisk(name), cephfsVolume.volName) @@ -301,13 +299,46 @@ func (cephfsVolume *cephfs) execMount(mountpoint string) error { return nil } -func getVolumeSource(spec *volume.Spec) (*v1.CephFSVolumeSource, bool, error) { +func getVolumeSource(spec *volume.Spec) ([]string, string, string, string, bool, error) { if spec.Volume != nil && spec.Volume.CephFS != nil { - return spec.Volume.CephFS, spec.Volume.CephFS.ReadOnly, nil + mon := spec.Volume.CephFS.Monitors + path := spec.Volume.CephFS.Path + user := spec.Volume.CephFS.User + secretFile := spec.Volume.CephFS.SecretFile + readOnly := spec.Volume.CephFS.ReadOnly + return mon, path, user, secretFile, readOnly, nil } else if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CephFS != nil { - return spec.PersistentVolume.Spec.CephFS, spec.ReadOnly, nil + mon := spec.PersistentVolume.Spec.CephFS.Monitors + path := spec.PersistentVolume.Spec.CephFS.Path + user := spec.PersistentVolume.Spec.CephFS.User + secretFile := spec.PersistentVolume.Spec.CephFS.SecretFile + readOnly := spec.PersistentVolume.Spec.CephFS.ReadOnly + return mon, path, user, secretFile, readOnly, nil } - return nil, false, fmt.Errorf("Spec does not reference a CephFS volume type") + return nil, "", "", "", false, fmt.Errorf("Spec does not reference a CephFS volume type") +} + +func getSecretNameAndNamespace(spec *volume.Spec, defaultNamespace string) (string, string, error) { + if spec.Volume != nil && spec.Volume.CephFS != nil { + localSecretRef := spec.Volume.CephFS.SecretRef + if localSecretRef != nil { + return localSecretRef.Name, defaultNamespace, nil + } + return "", "", nil + + } else if spec.PersistentVolume != nil && + spec.PersistentVolume.Spec.CephFS != nil { + secretRef := spec.PersistentVolume.Spec.CephFS.SecretRef + secretNs := defaultNamespace + if secretRef != nil { + if len(secretRef.Namespace) != 0 { + secretNs = secretRef.Namespace + } + return secretRef.Name, secretNs, nil + } + return "", "", nil + } + return "", "", fmt.Errorf("Spec does not reference an CephFS volume type") } diff --git a/pkg/volume/cephfs/cephfs_test.go b/pkg/volume/cephfs/cephfs_test.go index 87912a784cc33..da9d85f11e133 100644 --- a/pkg/volume/cephfs/cephfs_test.go +++ b/pkg/volume/cephfs/cephfs_test.go @@ -36,7 +36,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/cephfs") if err != nil { t.Errorf("Can't find the plugin by name") @@ -59,7 +59,7 @@ func TestPlugin(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/cephfs") if err != nil { t.Errorf("Can't find the plugin by name") @@ -123,7 +123,7 @@ func TestConstructVolumeSpec(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/cephfs") if err != nil { t.Errorf("can't find cephfs plugin by name") @@ -135,3 +135,94 @@ func TestConstructVolumeSpec(t *testing.T) { t.Errorf("Get wrong cephfs spec name, got: %s", cephfsSpec.Name()) } } + +type testcase struct { + name string + defaultNs string + spec *volume.Spec + // Expected return of the test + expectedName string + expectedNs string + expectedError error +} + +func TestGetSecretNameAndNamespaceForPV(t *testing.T) { + tests := []testcase{ + { + name: "persistent volume source", + defaultNs: "default", + spec: &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + CephFS: &v1.CephFSPersistentVolumeSource{ + Monitors: []string{"a", "b"}, + User: "user", + SecretRef: &v1.SecretReference{ + Name: "name", + Namespace: "ns", + }, + SecretFile: "/etc/ceph/user.secret", + }, + }, + }, + }, + }, + expectedName: "name", + expectedNs: "ns", + expectedError: nil, + }, + { + name: "persistent volume source without namespace", + defaultNs: "default", + spec: &volume.Spec{ + PersistentVolume: &v1.PersistentVolume{ + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + CephFS: &v1.CephFSPersistentVolumeSource{ + Monitors: []string{"a", "b"}, + User: "user", + SecretRef: &v1.SecretReference{ + Name: "name", + }, + SecretFile: "/etc/ceph/user.secret", + }, + }, + }, + }, + }, + expectedName: "name", + expectedNs: "default", + expectedError: nil, + }, + { + name: "pod volume source", + defaultNs: "default", + spec: &volume.Spec{ + Volume: &v1.Volume{ + VolumeSource: v1.VolumeSource{ + CephFS: &v1.CephFSVolumeSource{ + Monitors: []string{"a", "b"}, + User: "user", + SecretRef: &v1.LocalObjectReference{ + Name: "name", + }, + SecretFile: "/etc/ceph/user.secret", + }, + }, + }, + }, + expectedName: "name", + expectedNs: "default", + expectedError: nil, + }, + } + for _, testcase := range tests { + resultName, resultNs, err := getSecretNameAndNamespace(testcase.spec, testcase.defaultNs) + if err != testcase.expectedError || resultName != testcase.expectedName || resultNs != testcase.expectedNs { + t.Errorf("%s failed: expected err=%v ns=%q name=%q, got %v/%q/%q", testcase.name, testcase.expectedError, testcase.expectedNs, testcase.expectedName, + err, resultNs, resultName) + } + } + +} diff --git a/pkg/volume/cinder/attacher.go b/pkg/volume/cinder/attacher.go index 901876d16b885..f6d1e4be69909 100644 --- a/pkg/volume/cinder/attacher.go +++ b/pkg/volume/cinder/attacher.go @@ -24,12 +24,14 @@ import ( "time" "github.com/golang/glog" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" + "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) type cinderDiskAttacher struct { @@ -66,7 +68,7 @@ func (plugin *cinderPlugin) NewAttacher() (volume.Attacher, error) { } func (plugin *cinderPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mount.GetMountRefs(mounter, deviceMountPath) } @@ -186,6 +188,17 @@ func (attacher *cinderDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nod instanceID, err := attacher.nodeInstanceID(nodeName) if err != nil { + if err == cloudprovider.InstanceNotFound { + // If node doesn't exist, OpenStack Nova will assume the volumes are not attached to it. + // Mark the volumes as detached and return false without error. + glog.Warningf("VolumesAreAttached: node %q does not exist.", nodeName) + for spec := range volumesAttachedCheck { + volumesAttachedCheck[spec] = false + } + + return volumesAttachedCheck, nil + } + return volumesAttachedCheck, err } @@ -208,7 +221,7 @@ func (attacher *cinderDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nod return volumesAttachedCheck, nil } -func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { +func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { // NOTE: devicePath is is path as reported by Cinder, which may be incorrect and should not be used. See Issue #33128 volumeSource, _, err := getVolumeSource(spec) if err != nil { @@ -262,7 +275,7 @@ func (attacher *cinderDiskAttacher) GetDeviceMountPath( // FIXME: this method can be further pruned. func (attacher *cinderDiskAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { - mounter := attacher.host.GetMounter() + mounter := attacher.host.GetMounter(cinderVolumePluginName) notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { if os.IsNotExist(err) { @@ -285,7 +298,7 @@ func (attacher *cinderDiskAttacher) MountDevice(spec *volume.Spec, devicePath st options = append(options, "ro") } if notMnt { - diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()} + diskMounter := volumehelper.NewSafeFormatAndMountFromHost(cinderVolumePluginName, attacher.host) mountOptions := volume.MountOptionFromSpec(spec, options...) err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions) if err != nil { @@ -309,7 +322,7 @@ func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) { return nil, err } return &cinderDiskDetacher{ - mounter: plugin.host.GetMounter(), + mounter: plugin.host.GetMounter(plugin.GetPluginName()), cinderProvider: cinder, }, nil } diff --git a/pkg/volume/cinder/attacher_test.go b/pkg/volume/cinder/attacher_test.go index e1818d58447e9..d6a4b5b06e639 100644 --- a/pkg/volume/cinder/attacher_test.go +++ b/pkg/volume/cinder/attacher_test.go @@ -650,6 +650,10 @@ func (instances *instances) InstanceTypeByProviderID(providerID string) (string, return "", errors.New("Not implemented") } +func (instances *instances) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + func (instances *instances) List(filter string) ([]types.NodeName, error) { return []types.NodeName{}, errors.New("Not implemented") } diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index 034a5f8e8cdde..58bb713322e5b 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -36,7 +36,6 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) // This is the primary entrypoint for volume plugins. @@ -116,7 +115,7 @@ func (plugin *cinderPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { } func (plugin *cinderPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - return plugin.newMounterInternal(spec, pod.UID, &CinderDiskUtil{}, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod.UID, &CinderDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *cinderPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager cdManager, mounter mount.Interface) (volume.Mounter, error) { @@ -139,11 +138,11 @@ func (plugin *cinderPlugin) newMounterInternal(spec *volume.Spec, podUID types.U }, fsType: fsType, readOnly: readOnly, - blockDeviceMounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}}, nil + blockDeviceMounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host)}, nil } func (plugin *cinderPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, &CinderDiskUtil{}, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, &CinderDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *cinderPlugin) newUnmounterInternal(volName string, podUID types.UID, manager cdManager, mounter mount.Interface) (volume.Unmounter, error) { @@ -216,7 +215,7 @@ func (plugin *cinderPlugin) getCloudProvider() (CinderProvider, error) { } func (plugin *cinderPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName()) sourceName, err := mounter.GetDeviceNameFromMount(mountPath, pluginDir) if err != nil { @@ -276,13 +275,6 @@ type cinderVolume struct { volume.MetricsNil } -func detachDiskLogError(cd *cinderVolume) { - err := cd.manager.DetachDisk(&cinderVolumeUnmounter{cd}) - if err != nil { - glog.Warningf("Failed to detach disk: %v (%v)", cd, err) - } -} - func (b *cinderVolumeMounter) GetAttributes() volume.Attributes { return volume.Attributes{ ReadOnly: b.readOnly, @@ -309,7 +301,6 @@ func (b *cinderVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { b.plugin.volumeLocks.LockKey(b.pdName) defer b.plugin.volumeLocks.UnlockKey(b.pdName) - // TODO: handle failed mounts here. notmnt, err := b.mounter.IsLikelyNotMountPoint(dir) if err != nil && !os.IsNotExist(err) { glog.Errorf("Cannot validate mount point: %s %v", dir, err) @@ -327,9 +318,7 @@ func (b *cinderVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } if err := os.MkdirAll(dir, 0750); err != nil { - // TODO: we should really eject the attach/detach out into its own control loop. glog.V(4).Infof("Could not create directory %s: %v", dir, err) - detachDiskLogError(b.cinderVolume) return err } @@ -360,8 +349,6 @@ func (b *cinderVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { } } os.Remove(dir) - // TODO: we should really eject the attach/detach out into its own control loop. - detachDiskLogError(b.cinderVolume) glog.Errorf("Failed to mount %s: %v", dir, err) return err } diff --git a/pkg/volume/cinder/cinder_test.go b/pkg/volume/cinder/cinder_test.go index d4cf53cdc7190..a5fc228814f83 100644 --- a/pkg/volume/cinder/cinder_test.go +++ b/pkg/volume/cinder/cinder_test.go @@ -38,7 +38,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/cinder") if err != nil { @@ -134,7 +134,7 @@ func TestPlugin(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/cinder") if err != nil { diff --git a/pkg/volume/configmap/configmap.go b/pkg/volume/configmap/configmap.go index af05aef114529..94b83b7ed7f32 100644 --- a/pkg/volume/configmap/configmap.go +++ b/pkg/volume/configmap/configmap.go @@ -92,7 +92,7 @@ func (plugin *configMapPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts v spec.Name(), pod.UID, plugin, - plugin.host.GetMounter(), + plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetWriter(), volume.MetricsNil{}, }, @@ -109,7 +109,7 @@ func (plugin *configMapPlugin) NewUnmounter(volName string, podUID types.UID) (v volName, podUID, plugin, - plugin.host.GetMounter(), + plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetWriter(), volume.MetricsNil{}, }, diff --git a/pkg/volume/configmap/configmap_test.go b/pkg/volume/configmap/configmap_test.go index ce300e976b07f..9b2388517520c 100644 --- a/pkg/volume/configmap/configmap_test.go +++ b/pkg/volume/configmap/configmap_test.go @@ -272,7 +272,7 @@ func TestCanSupport(t *testing.T) { pluginMgr := volume.VolumePluginMgr{} tempDir, host := newTestHost(t, nil) defer os.RemoveAll(tempDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(configMapPluginName) if err != nil { @@ -304,7 +304,7 @@ func TestPlugin(t *testing.T) { ) defer os.RemoveAll(tempDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(configMapPluginName) if err != nil { @@ -317,7 +317,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Failed to make a new Mounter: %v", err) } if mounter == nil { - t.Errorf("Got a nil Mounter") + t.Fatalf("Got a nil Mounter") } vName, err := plugin.GetVolumeName(volume.NewSpecFromVolume(volumeSpec)) @@ -368,7 +368,7 @@ func TestPluginReboot(t *testing.T) { ) defer os.RemoveAll(rootDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(configMapPluginName) if err != nil { @@ -381,7 +381,7 @@ func TestPluginReboot(t *testing.T) { t.Errorf("Failed to make a new Mounter: %v", err) } if mounter == nil { - t.Errorf("Got a nil Mounter") + t.Fatalf("Got a nil Mounter") } podMetadataDir := fmt.Sprintf("%v/pods/test_pod_uid3/plugins/kubernetes.io~configmap/test_volume_name", rootDir) @@ -424,7 +424,7 @@ func TestPluginOptional(t *testing.T) { volumeSpec.VolumeSource.ConfigMap.Optional = &trueVal defer os.RemoveAll(tempDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(configMapPluginName) if err != nil { @@ -499,7 +499,7 @@ func TestPluginKeysOptional(t *testing.T) { volumeSpec.VolumeSource.ConfigMap.Optional = &trueVal defer os.RemoveAll(tempDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(configMapPluginName) if err != nil { @@ -597,7 +597,7 @@ func doTestCleanAndTeardown(plugin volume.VolumePlugin, podUID types.UID, testVo t.Errorf("Failed to make a new Unmounter: %v", err) } if unmounter == nil { - t.Errorf("Got a nil Unmounter") + t.Fatalf("Got a nil Unmounter") } if err := unmounter.TearDown(); err != nil { diff --git a/pkg/volume/downwardapi/downwardapi_test.go b/pkg/volume/downwardapi/downwardapi_test.go index 393e158b7a98c..4f68da8456629 100644 --- a/pkg/volume/downwardapi/downwardapi_test.go +++ b/pkg/volume/downwardapi/downwardapi_test.go @@ -54,7 +54,7 @@ func TestCanSupport(t *testing.T) { pluginMgr := volume.VolumePluginMgr{} tmpDir, host := newTestHost(t, nil) defer os.RemoveAll(tmpDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(downwardAPIPluginName) if err != nil { @@ -219,7 +219,7 @@ func newDownwardAPITest(t *testing.T, name string, volumeFiles, podLabels, podAn pluginMgr := volume.VolumePluginMgr{} rootDir, host := newTestHost(t, clientset) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(downwardAPIPluginName) if err != nil { t.Errorf("Can't find the plugin by name") @@ -241,7 +241,7 @@ func newDownwardAPITest(t *testing.T, name string, volumeFiles, podLabels, podAn t.Errorf("Failed to make a new Mounter: %v", err) } if mounter == nil { - t.Errorf("Got a nil Mounter") + t.Fatalf("Got a nil Mounter") } volumePath := mounter.GetPath() @@ -278,7 +278,7 @@ func (test *downwardAPITest) tearDown() { test.t.Errorf("Failed to make a new Unmounter: %v", err) } if unmounter == nil { - test.t.Errorf("Got a nil Unmounter") + test.t.Fatalf("Got a nil Unmounter") } if err := unmounter.TearDown(); err != nil { diff --git a/pkg/volume/empty_dir/empty_dir.go b/pkg/volume/empty_dir/empty_dir.go index 2cb1aba212f55..76da7c3cffe8b 100644 --- a/pkg/volume/empty_dir/empty_dir.go +++ b/pkg/volume/empty_dir/empty_dir.go @@ -99,7 +99,7 @@ func (plugin *emptyDirPlugin) SupportsBulkVolumeVerification() bool { } func (plugin *emptyDirPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { - return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(), &realMountDetector{plugin.host.GetMounter()}, opts) + return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName()), &realMountDetector{plugin.host.GetMounter(plugin.GetPluginName())}, opts) } func (plugin *emptyDirPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, mounter mount.Interface, mountDetector mountDetector, opts volume.VolumeOptions) (volume.Mounter, error) { @@ -120,7 +120,7 @@ func (plugin *emptyDirPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, func (plugin *emptyDirPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(), &realMountDetector{plugin.host.GetMounter()}) + return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName()), &realMountDetector{plugin.host.GetMounter(plugin.GetPluginName())}) } func (plugin *emptyDirPlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface, mountDetector mountDetector) (volume.Unmounter, error) { diff --git a/pkg/volume/empty_dir/empty_dir_test.go b/pkg/volume/empty_dir/empty_dir_test.go index 765581b5bb655..31cc7f5a31228 100644 --- a/pkg/volume/empty_dir/empty_dir_test.go +++ b/pkg/volume/empty_dir/empty_dir_test.go @@ -36,7 +36,7 @@ import ( // Construct an instance of a plugin, by name. func makePluginUnderTest(t *testing.T, plugName, basePath string) volume.VolumePlugin { plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(basePath, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(basePath, nil, nil)) plug, err := plugMgr.FindPluginByName(plugName) if err != nil { @@ -221,7 +221,7 @@ func TestPluginBackCompat(t *testing.T) { t.Errorf("Failed to make a new Mounter: %v", err) } if mounter == nil { - t.Errorf("Got a nil Mounter") + t.Fatalf("Got a nil Mounter") } volPath := mounter.GetPath() @@ -249,6 +249,9 @@ func TestMetrics(t *testing.T) { if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) } + if mounter == nil { + t.Fatalf("Got a nil Mounter") + } // Need to create the subdirectory os.MkdirAll(mounter.GetPath(), 0755) diff --git a/pkg/volume/fc/BUILD b/pkg/volume/fc/BUILD index df6f13d053faf..5bc5321da7d94 100644 --- a/pkg/volume/fc/BUILD +++ b/pkg/volume/fc/BUILD @@ -20,10 +20,10 @@ go_library( "//pkg/util/strings:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", + "//pkg/volume/util/volumehelper:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/fc/attacher.go b/pkg/volume/fc/attacher.go index c0e469c35bfcd..4a8121faee896 100644 --- a/pkg/volume/fc/attacher.go +++ b/pkg/volume/fc/attacher.go @@ -24,17 +24,17 @@ import ( "time" "github.com/golang/glog" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" + "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) type fcAttacher struct { host volume.VolumeHost manager diskManager - exe exec.Interface } var _ volume.Attacher = &fcAttacher{} @@ -45,12 +45,11 @@ func (plugin *fcPlugin) NewAttacher() (volume.Attacher, error) { return &fcAttacher{ host: plugin.host, manager: &FCUtil{}, - exe: exec.New(), }, nil } func (plugin *fcPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mount.GetMountRefs(mounter, deviceMountPath) } @@ -67,7 +66,7 @@ func (attacher *fcAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName ty return volumesAttachedCheck, nil } -func (attacher *fcAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { +func (attacher *fcAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { mounter, err := volumeSpecToMounter(spec, attacher.host) if err != nil { glog.Warningf("failed to get fc mounter: %v", err) @@ -88,7 +87,7 @@ func (attacher *fcAttacher) GetDeviceMountPath( } func (attacher *fcAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { - mounter := attacher.host.GetMounter() + mounter := attacher.host.GetMounter(fcPluginName) notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { if os.IsNotExist(err) { @@ -111,7 +110,7 @@ func (attacher *fcAttacher) MountDevice(spec *volume.Spec, devicePath string, de options = append(options, "ro") } if notMnt { - diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()} + diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Exec: attacher.host.GetExec(fcPluginName)} mountOptions := volume.MountOptionFromSpec(spec, options...) err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions) if err != nil { @@ -125,16 +124,14 @@ func (attacher *fcAttacher) MountDevice(spec *volume.Spec, devicePath string, de type fcDetacher struct { mounter mount.Interface manager diskManager - exe exec.Interface } var _ volume.Detacher = &fcDetacher{} func (plugin *fcPlugin) NewDetacher() (volume.Detacher, error) { return &fcDetacher{ - mounter: plugin.host.GetMounter(), + mounter: plugin.host.GetMounter(plugin.GetPluginName()), manager: &FCUtil{}, - exe: exec.New(), }, nil } @@ -192,7 +189,7 @@ func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost) (*fcDiskMoun }, fsType: fc.FSType, readOnly: readOnly, - mounter: &mount.SafeFormatAndMount{Interface: host.GetMounter(), Runner: exec.New()}, + mounter: volumehelper.NewSafeFormatAndMountFromHost(fcPluginName, host), }, nil } diff --git a/pkg/volume/fc/disk_manager.go b/pkg/volume/fc/disk_manager.go index b14d84ff31f1c..33a0c422cfff8 100644 --- a/pkg/volume/fc/disk_manager.go +++ b/pkg/volume/fc/disk_manager.go @@ -36,7 +36,6 @@ type diskManager interface { // utility to mount a disk based filesystem func diskSetUp(manager diskManager, b fcDiskMounter, volPath string, mounter mount.Interface, fsGroup *int64) error { globalPDPath := manager.MakeGlobalPDName(*b.fcDisk) - // TODO: handle failed mounts here. noMnt, err := mounter.IsLikelyNotMountPoint(volPath) if err != nil && !os.IsNotExist(err) { @@ -57,7 +56,30 @@ func diskSetUp(manager diskManager, b fcDiskMounter, volPath string, mounter mou } err = mounter.Mount(globalPDPath, volPath, "", options) if err != nil { - glog.Errorf("failed to bind mount:%s", globalPDPath) + glog.Errorf("Failed to bind mount: source:%s, target:%s, err:%v", globalPDPath, volPath, err) + noMnt, mntErr := b.mounter.IsLikelyNotMountPoint(volPath) + if mntErr != nil { + glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + return err + } + if !noMnt { + if mntErr = b.mounter.Unmount(volPath); mntErr != nil { + glog.Errorf("Failed to unmount: %v", mntErr) + return err + } + noMnt, mntErr = b.mounter.IsLikelyNotMountPoint(volPath) + if mntErr != nil { + glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + return err + } + if !noMnt { + // will most likely retry on next sync loop. + glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", volPath) + return err + } + } + os.Remove(volPath) + return err } diff --git a/pkg/volume/fc/fc.go b/pkg/volume/fc/fc.go index 26e69bd709f32..096a24df387e9 100644 --- a/pkg/volume/fc/fc.go +++ b/pkg/volume/fc/fc.go @@ -28,17 +28,15 @@ import ( utilstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" ) // This is the primary entrypoint for volume plugins. func ProbeVolumePlugins() []volume.VolumePlugin { - return []volume.VolumePlugin{&fcPlugin{nil, exec.New()}} + return []volume.VolumePlugin{&fcPlugin{nil}} } type fcPlugin struct { host volume.VolumeHost - exe exec.Interface } var _ volume.VolumePlugin = &fcPlugin{} @@ -103,10 +101,10 @@ func (plugin *fcPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { func (plugin *fcPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newMounterInternal(spec, pod.UID, &FCUtil{}, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod.UID, &FCUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) } -func (plugin *fcPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface) (volume.Mounter, error) { +func (plugin *fcPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec) (volume.Mounter, error) { // fc volumes used directly in a pod have a ReadOnly flag set by the pod author. // fc volumes used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV fc, readOnly, err := getVolumeSource(spec) @@ -138,13 +136,13 @@ func (plugin *fcPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, plugin: plugin}, fsType: fc.FSType, readOnly: readOnly, - mounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}, + mounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}, }, nil } func (plugin *fcPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newUnmounterInternal(volName, podUID, &FCUtil{}, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, &FCUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *fcPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface) (volume.Unmounter, error) { @@ -160,11 +158,6 @@ func (plugin *fcPlugin) newUnmounterInternal(volName string, podUID types.UID, m }, nil } -func (plugin *fcPlugin) execCommand(command string, args []string) ([]byte, error) { - cmd := plugin.exe.Command(command, args...) - return cmd.CombinedOutput() -} - func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { fcVolume := &v1.Volume{ Name: volumeName, diff --git a/pkg/volume/fc/fc_test.go b/pkg/volume/fc/fc_test.go index e2662d2c90922..b800694281744 100644 --- a/pkg/volume/fc/fc_test.go +++ b/pkg/volume/fc/fc_test.go @@ -39,7 +39,7 @@ func TestCanSupport(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/fc") if err != nil { @@ -61,7 +61,7 @@ func TestGetAccessModes(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/fc") if err != nil { @@ -132,7 +132,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/fc") if err != nil { @@ -141,7 +141,8 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { fakeManager := NewFakeDiskManager() defer fakeManager.Cleanup() fakeMounter := &mount.FakeMounter{} - mounter, err := plug.(*fcPlugin).newMounterInternal(spec, types.UID("poduid"), fakeManager, fakeMounter) + fakeExec := mount.NewFakeExec(nil) + mounter, err := plug.(*fcPlugin).newMounterInternal(spec, types.UID("poduid"), fakeManager, fakeMounter, fakeExec) if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) } @@ -201,7 +202,7 @@ func doTestPluginNilMounter(t *testing.T, spec *volume.Spec) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/fc") if err != nil { @@ -210,7 +211,8 @@ func doTestPluginNilMounter(t *testing.T, spec *volume.Spec) { fakeManager := NewFakeDiskManager() defer fakeManager.Cleanup() fakeMounter := &mount.FakeMounter{} - mounter, err := plug.(*fcPlugin).newMounterInternal(spec, types.UID("poduid"), fakeManager, fakeMounter) + fakeExec := mount.NewFakeExec(nil) + mounter, err := plug.(*fcPlugin).newMounterInternal(spec, types.UID("poduid"), fakeManager, fakeMounter, fakeExec) if err == nil { t.Errorf("Error failed to make a new Mounter is expected: %v", err) } @@ -353,13 +355,16 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { client := fake.NewSimpleClientset(pv, claim) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, client, nil)) plug, _ := plugMgr.FindPluginByName(fcPluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes spec := volume.NewSpecFromPersistentVolume(pv, true) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{}) + if mounter == nil { + t.Fatalf("Got a nil Mounter") + } if !mounter.GetAttributes().ReadOnly { t.Errorf("Expected true for mounter.IsReadOnly") diff --git a/pkg/volume/flexvolume/BUILD b/pkg/volume/flexvolume/BUILD index 3a78103ab8624..02f3602940803 100644 --- a/pkg/volume/flexvolume/BUILD +++ b/pkg/volume/flexvolume/BUILD @@ -29,9 +29,11 @@ go_library( "//pkg/util/strings:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", + "//vendor/github.com/fsnotify/fsnotify:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -42,6 +44,7 @@ go_test( "attacher_test.go", "common_test.go", "detacher_test.go", + "driver-call_test.go", "flexvolume_test.go", "mounter_test.go", "plugin_test.go", diff --git a/pkg/volume/flexvolume/attacher-defaults.go b/pkg/volume/flexvolume/attacher-defaults.go index 3073f4def051b..bf8dcfe825414 100644 --- a/pkg/volume/flexvolume/attacher-defaults.go +++ b/pkg/volume/flexvolume/attacher-defaults.go @@ -24,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" - "k8s.io/utils/exec" ) type attacherDefaults flexVolumeAttacher @@ -59,7 +58,7 @@ func (a *attacherDefaults) MountDevice(spec *volume.Spec, devicePath string, dev options = append(options, "rw") } - diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()} + diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Exec: a.plugin.host.GetExec(a.plugin.GetPluginName())} return diskMounter.FormatAndMount(devicePath, deviceMountPath, volSource.FSType, options) } diff --git a/pkg/volume/flexvolume/attacher.go b/pkg/volume/flexvolume/attacher.go index 73575be4c4e96..c2efcffa1c37a 100644 --- a/pkg/volume/flexvolume/attacher.go +++ b/pkg/volume/flexvolume/attacher.go @@ -20,6 +20,7 @@ import ( "time" "github.com/golang/glog" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/volume" ) @@ -47,7 +48,7 @@ func (a *flexVolumeAttacher) Attach(spec *volume.Spec, hostName types.NodeName) } // WaitForAttach is part of the volume.Attacher interface -func (a *flexVolumeAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { +func (a *flexVolumeAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { call := a.plugin.NewDriverCallWithTimeout(waitForAttachCmd, timeout) call.Append(devicePath) call.AppendSpec(spec, a.plugin.host, nil) @@ -69,7 +70,7 @@ func (a *flexVolumeAttacher) GetDeviceMountPath(spec *volume.Spec) (string, erro // MountDevice is part of the volume.Attacher interface func (a *flexVolumeAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { // Mount only once. - alreadyMounted, err := prepareForMount(a.plugin.host.GetMounter(), deviceMountPath) + alreadyMounted, err := prepareForMount(a.plugin.host.GetMounter(a.plugin.GetPluginName()), deviceMountPath) if err != nil { return err } @@ -87,7 +88,7 @@ func (a *flexVolumeAttacher) MountDevice(spec *volume.Spec, devicePath string, d // Devicepath is empty if the plugin does not support attach calls. Ignore mountDevice calls if the // plugin does not implement attach interface. if devicePath != "" { - return (*attacherDefaults)(a).MountDevice(spec, devicePath, deviceMountPath, a.plugin.host.GetMounter()) + return (*attacherDefaults)(a).MountDevice(spec, devicePath, deviceMountPath, a.plugin.host.GetMounter(a.plugin.GetPluginName())) } else { return nil } diff --git a/pkg/volume/flexvolume/attacher_test.go b/pkg/volume/flexvolume/attacher_test.go index 8807cc8162229..f4b4ab7b8395c 100644 --- a/pkg/volume/flexvolume/attacher_test.go +++ b/pkg/volume/flexvolume/attacher_test.go @@ -17,9 +17,11 @@ limitations under the License. package flexvolume import ( - "k8s.io/kubernetes/pkg/volume" "testing" "time" + + "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/volume" ) func TestAttach(t *testing.T) { @@ -37,7 +39,7 @@ func TestAttach(t *testing.T) { func TestWaitForAttach(t *testing.T) { spec := fakeVolumeSpec() - + var pod *v1.Pod plugin, _ := testPlugin() plugin.runner = fakeRunner( assertDriverCall(t, notSupportedOutput(), waitForAttachCmd, "/dev/sdx", @@ -45,7 +47,7 @@ func TestWaitForAttach(t *testing.T) { ) a, _ := plugin.NewAttacher() - a.WaitForAttach(spec, "/dev/sdx", 1*time.Second) + a.WaitForAttach(spec, "/dev/sdx", pod, 1*time.Second) } func TestMountDevice(t *testing.T) { diff --git a/pkg/volume/flexvolume/detacher-defaults.go b/pkg/volume/flexvolume/detacher-defaults.go index 3226836f90e15..17c85a3653203 100644 --- a/pkg/volume/flexvolume/detacher-defaults.go +++ b/pkg/volume/flexvolume/detacher-defaults.go @@ -41,5 +41,5 @@ func (d *detacherDefaults) WaitForDetach(devicePath string, timeout time.Duratio // UnmountDevice is part of the volume.Detacher interface. func (d *detacherDefaults) UnmountDevice(deviceMountPath string) error { glog.Warning(logPrefix(d.plugin.flexVolumePlugin), "using default UnmountDevice for device mount path ", deviceMountPath) - return util.UnmountPath(deviceMountPath, d.plugin.host.GetMounter()) + return util.UnmountPath(deviceMountPath, d.plugin.host.GetMounter(d.plugin.GetPluginName())) } diff --git a/pkg/volume/flexvolume/detacher.go b/pkg/volume/flexvolume/detacher.go index dcca403619e33..4acbd03eaef80 100644 --- a/pkg/volume/flexvolume/detacher.go +++ b/pkg/volume/flexvolume/detacher.go @@ -69,7 +69,7 @@ func (d *flexVolumeDetacher) UnmountDevice(deviceMountPath string) error { return nil } - notmnt, err := isNotMounted(d.plugin.host.GetMounter(), deviceMountPath) + notmnt, err := isNotMounted(d.plugin.host.GetMounter(d.plugin.GetPluginName()), deviceMountPath) if err != nil { return err } diff --git a/pkg/volume/flexvolume/driver-call.go b/pkg/volume/flexvolume/driver-call.go index 4640c51d8bb35..e3e4527ddc97d 100644 --- a/pkg/volume/flexvolume/driver-call.go +++ b/pkg/volume/flexvolume/driver-call.go @@ -58,8 +58,6 @@ const ( optionKeyPodUID = "kubernetes.io/pod.uid" optionKeyServiceAccountName = "kubernetes.io/serviceAccount.name" - - attachCapability = "attach" ) const ( @@ -204,7 +202,19 @@ type DriverStatus struct { // Returns capabilities of the driver. // By default we assume all the capabilities are supported. // If the plugin does not support a capability, it can return false for that capability. - Capabilities map[string]bool + Capabilities *DriverCapabilities `json:",omitempty"` +} + +type DriverCapabilities struct { + Attach bool `json:"attach"` + SELinuxRelabel bool `json:"selinuxRelabel"` +} + +func defaultCapabilities() *DriverCapabilities { + return &DriverCapabilities{ + Attach: true, + SELinuxRelabel: true, + } } // isCmdNotSupportedErr checks if the error corresponds to command not supported by @@ -220,7 +230,9 @@ func isCmdNotSupportedErr(err error) bool { // handleCmdResponse processes the command output and returns the appropriate // error code or message. func handleCmdResponse(cmd string, output []byte) (*DriverStatus, error) { - var status DriverStatus + status := DriverStatus{ + Capabilities: defaultCapabilities(), + } if err := json.Unmarshal(output, &status); err != nil { glog.Errorf("Failed to unmarshal output for command: %s, output: %q, error: %s", cmd, string(output), err.Error()) return nil, err diff --git a/cmd/mungedocs/whitespace.go b/pkg/volume/flexvolume/driver-call_test.go similarity index 58% rename from cmd/mungedocs/whitespace.go rename to pkg/volume/flexvolume/driver-call_test.go index 37a1b1d16b854..77f71f7b8f87c 100644 --- a/cmd/mungedocs/whitespace.go +++ b/pkg/volume/flexvolume/driver-call_test.go @@ -1,5 +1,5 @@ /* -Copyright 2015 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,18 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package flexvolume -// Remove all trailing whitespace -func updateWhitespace(file string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - for _, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - newline := trimRightSpace(mline.data) - out = append(out, newMungeLine(newline)) +import ( + "testing" +) + +func TestHandleResponseDefaults(t *testing.T) { + ds, err := handleCmdResponse("test", []byte(`{"status": "Success"}`)) + if err != nil { + t.Error("error: ", err) + } + + if *ds.Capabilities != *defaultCapabilities() { + t.Error("wrong default capabilities: ", *ds.Capabilities) } - return out, nil } diff --git a/pkg/volume/flexvolume/flexvolume_test.go b/pkg/volume/flexvolume/flexvolume_test.go index 76ade7b85450d..292beb54506f3 100644 --- a/pkg/volume/flexvolume/flexvolume_test.go +++ b/pkg/volume/flexvolume/flexvolume_test.go @@ -174,7 +174,7 @@ func TestCanSupport(t *testing.T) { plugMgr := volume.VolumePluginMgr{} installPluginUnderTest(t, "kubernetes.io", "fakeAttacher", tmpDir, execScriptTempl1, nil) - plugMgr.InitPlugins(ProbeVolumePlugins(tmpDir), volumetest.NewFakeVolumeHost("fake", nil, nil)) + plugMgr.InitPlugins(nil, GetDynamicPluginProber(tmpDir), volumetest.NewFakeVolumeHost("fake", nil, nil)) plugin, err := plugMgr.FindPluginByName("flexvolume-kubernetes.io/fakeAttacher") if err != nil { t.Fatalf("Can't find the plugin by name") @@ -202,7 +202,7 @@ func TestGetAccessModes(t *testing.T) { plugMgr := volume.VolumePluginMgr{} installPluginUnderTest(t, "kubernetes.io", "fakeAttacher", tmpDir, execScriptTempl1, nil) - plugMgr.InitPlugins(ProbeVolumePlugins(tmpDir), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(nil, GetDynamicPluginProber(tmpDir), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plugin, err := plugMgr.FindPersistentPluginByName("flexvolume-kubernetes.io/fakeAttacher") if err != nil { diff --git a/pkg/volume/flexvolume/mounter-defaults.go b/pkg/volume/flexvolume/mounter-defaults.go index a8586b46ec934..b3cd9430ff532 100644 --- a/pkg/volume/flexvolume/mounter-defaults.go +++ b/pkg/volume/flexvolume/mounter-defaults.go @@ -47,7 +47,7 @@ func (f *mounterDefaults) GetAttributes() volume.Attributes { return volume.Attributes{ ReadOnly: f.readOnly, Managed: !f.readOnly, - SupportsSELinux: true, + SupportsSELinux: f.flexVolume.plugin.capabilities.SELinuxRelabel, } } diff --git a/pkg/volume/flexvolume/mounter.go b/pkg/volume/flexvolume/mounter.go index 316928c429ba6..94b2ffd748867 100644 --- a/pkg/volume/flexvolume/mounter.go +++ b/pkg/volume/flexvolume/mounter.go @@ -19,7 +19,6 @@ package flexvolume import ( "strconv" - "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/utils/exec" ) @@ -29,9 +28,6 @@ type flexVolumeMounter struct { *flexVolume // Runner used to setup the volume. runner exec.Interface - // blockDeviceMounter provides the interface to create filesystem if the - // filesystem doesn't exist. - blockDeviceMounter mount.Interface // the considered volume spec spec *volume.Spec readOnly bool diff --git a/pkg/volume/flexvolume/plugin.go b/pkg/volume/flexvolume/plugin.go index 7b37514009404..4df126ed873c8 100644 --- a/pkg/volume/flexvolume/plugin.go +++ b/pkg/volume/flexvolume/plugin.go @@ -43,6 +43,7 @@ type flexVolumePlugin struct { sync.Mutex unsupportedCommands []string + capabilities DriverCapabilities } type flexVolumeAttachablePlugin struct { @@ -64,13 +65,15 @@ func NewFlexVolumePlugin(pluginDir, name string) (volume.VolumePlugin, error) { unsupportedCommands: []string{}, } - // Check whether the plugin is attachable. - ok, err := isAttachable(flexPlugin) + // Initialize the plugin and probe the capabilities + call := flexPlugin.NewDriverCall(initCmd) + ds, err := call.Run() if err != nil { return nil, err } + flexPlugin.capabilities = *ds.Capabilities - if ok { + if flexPlugin.capabilities.Attach { // Plugin supports attach/detach, so return flexVolumeAttachablePlugin return &flexVolumeAttachablePlugin{flexVolumePlugin: flexPlugin}, nil } else { @@ -78,30 +81,11 @@ func NewFlexVolumePlugin(pluginDir, name string) (volume.VolumePlugin, error) { } } -func isAttachable(plugin *flexVolumePlugin) (bool, error) { - call := plugin.NewDriverCall(initCmd) - res, err := call.Run() - if err != nil { - return false, err - } - - // By default all plugins are attachable, unless they report otherwise. - cap, ok := res.Capabilities[attachCapability] - if ok { - // cap is false, so plugin does not support attach/detach calls. - return cap, nil - } - - return true, nil -} - // Init is part of the volume.VolumePlugin interface. func (plugin *flexVolumePlugin) Init(host volume.VolumeHost) error { plugin.host = host - // call the init script - call := plugin.NewDriverCall(initCmd) - _, err := call.Run() - return err + // Hardwired 'success' as any errors from calling init() will be caught by NewFlexVolumePlugin() + return nil } func (plugin *flexVolumePlugin) getExecutable() string { @@ -158,7 +142,7 @@ func (plugin *flexVolumePlugin) GetAccessModes() []api.PersistentVolumeAccessMod // NewMounter is part of the volume.VolumePlugin interface. func (plugin *flexVolumePlugin) NewMounter(spec *volume.Spec, pod *api.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(), plugin.runner) + return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName()), plugin.runner) } // newMounterInternal is the internal mounter routine to build the volume. @@ -176,16 +160,15 @@ func (plugin *flexVolumePlugin) newMounterInternal(spec *volume.Spec, pod *api.P podServiceAccountName: pod.Spec.ServiceAccountName, volName: spec.Name(), }, - runner: runner, - spec: spec, - readOnly: readOnly, - blockDeviceMounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: runner}, + runner: runner, + spec: spec, + readOnly: readOnly, }, nil } // NewUnmounter is part of the volume.VolumePlugin interface. func (plugin *flexVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(), plugin.runner) + return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName()), plugin.runner) } // newUnmounterInternal is the internal unmounter routine to clean the volume. @@ -254,7 +237,7 @@ func (plugin *flexVolumePlugin) isUnsupported(command string) bool { } func (plugin *flexVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mount.GetMountRefs(mounter, deviceMountPath) } diff --git a/pkg/volume/flexvolume/probe.go b/pkg/volume/flexvolume/probe.go index 431c340a22e28..4acdb9f932ec7 100644 --- a/pkg/volume/flexvolume/probe.go +++ b/pkg/volume/flexvolume/probe.go @@ -19,28 +19,217 @@ package flexvolume import ( "io/ioutil" + "github.com/golang/glog" "k8s.io/kubernetes/pkg/volume" + + "os" + + "fmt" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + "k8s.io/apimachinery/pkg/util/errors" +) + +type flexVolumeProber struct { + mutex sync.Mutex + pluginDir string // Flexvolume driver directory + watcher *fsnotify.Watcher + probeNeeded bool // Must only read and write this through testAndSetProbeNeeded. + lastUpdated time.Time // Last time probeNeeded was updated. + watchEventCount int +} + +const ( + // TODO (cxing) Tune these params based on test results. + // watchEventLimit is the max allowable number of processed watches within watchEventInterval. + watchEventInterval = 5 * time.Second + watchEventLimit = 20 ) -// This is the primary entrypoint for volume plugins. -func ProbeVolumePlugins(pluginDir string) []volume.VolumePlugin { - plugins := []volume.VolumePlugin{} +func GetDynamicPluginProber(pluginDir string) volume.DynamicPluginProber { + return &flexVolumeProber{pluginDir: pluginDir} +} + +func (prober *flexVolumeProber) Init() error { + prober.testAndSetProbeNeeded(true) + prober.lastUpdated = time.Now() + + if err := prober.createPluginDir(); err != nil { + return err + } + if err := prober.initWatcher(); err != nil { + return err + } + + go func() { + defer prober.watcher.Close() + for { + select { + case event := <-prober.watcher.Events: + if err := prober.handleWatchEvent(event); err != nil { + glog.Errorf("Flexvolume prober watch: %s", err) + } + case err := <-prober.watcher.Errors: + glog.Errorf("Received an error from watcher: %s", err) + } + } + }() - files, _ := ioutil.ReadDir(pluginDir) + return nil +} + +// Probes for Flexvolume drivers. +// If a filesystem update has occurred since the last probe, updated = true +// and the list of probed plugins is returned. +// Otherwise, update = false and probedPlugins = nil. +// +// If an error occurs, updated and plugins are set arbitrarily. +func (prober *flexVolumeProber) Probe() (updated bool, plugins []volume.VolumePlugin, err error) { + probeNeeded := prober.testAndSetProbeNeeded(false) + + if !probeNeeded { + return false, nil, nil + } + + files, err := ioutil.ReadDir(prober.pluginDir) + if err != nil { + return false, nil, fmt.Errorf("Error reading the Flexvolume directory: %s", err) + } + + plugins = []volume.VolumePlugin{} + allErrs := []error{} for _, f := range files { - // only directories are counted as plugins + // only directories with names that do not begin with '.' are counted as plugins // and pluginDir/dirname/dirname should be an executable // unless dirname contains '~' for escaping namespace // e.g. dirname = vendor~cifs // then, executable will be pluginDir/dirname/cifs - if f.IsDir() { - plugin, err := NewFlexVolumePlugin(pluginDir, f.Name()) - if err != nil { + if f.IsDir() && filepath.Base(f.Name())[0] != '.' { + plugin, pluginErr := NewFlexVolumePlugin(prober.pluginDir, f.Name()) + if pluginErr != nil { + pluginErr = fmt.Errorf( + "Error creating Flexvolume plugin from directory %s, skipping. Error: %s", + f.Name(), pluginErr) + allErrs = append(allErrs, pluginErr) continue } plugins = append(plugins, plugin) } } - return plugins + + return true, plugins, errors.NewAggregate(allErrs) +} + +func (prober *flexVolumeProber) handleWatchEvent(event fsnotify.Event) error { + // event.Name is the watched path. + if filepath.Base(event.Name)[0] == '.' { + // Ignore files beginning with '.' + return nil + } + + eventPathAbs, err := filepath.Abs(event.Name) + if err != nil { + return err + } + + pluginDirAbs, err := filepath.Abs(prober.pluginDir) + if err != nil { + return err + } + + // If the Flexvolume plugin directory is removed, need to recreate it + // in order to keep it under watch. + if eventOpIs(event, fsnotify.Remove) && eventPathAbs == pluginDirAbs { + glog.Warningf("Flexvolume plugin directory at %s is removed. Recreating.", pluginDirAbs) + if err := prober.createPluginDir(); err != nil { + return err + } + if err := prober.addWatchRecursive(pluginDirAbs); err != nil { + return err + } + } else if eventOpIs(event, fsnotify.Create) { + if err := prober.addWatchRecursive(eventPathAbs); err != nil { + return err + } + } + + prober.updateProbeNeeded() + + return nil +} + +func (prober *flexVolumeProber) updateProbeNeeded() { + // Within 'watchEventInterval' seconds, a max of 'watchEventLimit' watch events is processed. + // The watch event will not be registered if the limit is reached. + // This prevents increased disk usage from Probe() being triggered too frequently (either + // accidentally or maliciously). + if time.Since(prober.lastUpdated) > watchEventInterval { + // Update, then reset the timer and watch count. + prober.testAndSetProbeNeeded(true) + prober.lastUpdated = time.Now() + prober.watchEventCount = 1 + } else if prober.watchEventCount < watchEventLimit { + prober.testAndSetProbeNeeded(true) + prober.watchEventCount++ + } +} + +// Recursively adds to watch all directories inside and including the file specified by the given filename. +// If the file is a symlink to a directory, it will watch the symlink but not any of the subdirectories. +// +// Each file or directory change triggers two events: one from the watch on itself, another from the watch +// on its parent directory. +func (prober *flexVolumeProber) addWatchRecursive(filename string) error { + addWatch := func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + if err := prober.watcher.Add(path); err != nil { + glog.Errorf("Error recursively adding watch: %v", err) + } + } + return nil + } + + return filepath.Walk(filename, addWatch) +} + +// Creates a new filesystem watcher and adds watches for the plugin directory +// and all of its subdirectories. +func (prober *flexVolumeProber) initWatcher() error { + var err error + if prober.watcher, err = fsnotify.NewWatcher(); err != nil { + return fmt.Errorf("Error creating new watcher: %s", err) + } + + if err = prober.addWatchRecursive(prober.pluginDir); err != nil { + return fmt.Errorf("Error adding watch on Flexvolume directory: %s", err) + } + + return nil +} + +// Creates the plugin directory, if it doesn't already exist. +func (prober *flexVolumeProber) createPluginDir() error { + if _, err := os.Stat(prober.pluginDir); os.IsNotExist(err) { + err := os.MkdirAll(prober.pluginDir, 0755) + if err != nil { + return fmt.Errorf("Error (re-)creating driver directory: %s", err) + } + } + + return nil +} + +func (prober *flexVolumeProber) testAndSetProbeNeeded(newval bool) (oldval bool) { + prober.mutex.Lock() + defer prober.mutex.Unlock() + oldval, prober.probeNeeded = prober.probeNeeded, newval + return +} + +func eventOpIs(event fsnotify.Event, op fsnotify.Op) bool { + return event.Op&op == op } diff --git a/pkg/volume/flocker/flocker.go b/pkg/volume/flocker/flocker.go index 33acb0a81fc83..18a59b7422827 100644 --- a/pkg/volume/flocker/flocker.go +++ b/pkg/volume/flocker/flocker.go @@ -138,7 +138,7 @@ func (p *flockerPlugin) getFlockerVolumeSource(spec *volume.Spec) (*v1.FlockerVo func (plugin *flockerPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newMounterInternal(spec, pod.UID, &FlockerUtil{}, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod.UID, &FlockerUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *flockerPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager volumeManager, mounter mount.Interface) (volume.Mounter, error) { @@ -166,7 +166,7 @@ func (plugin *flockerPlugin) newMounterInternal(spec *volume.Spec, podUID types. func (p *flockerPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { // Inject real implementations here, test through the internal function. - return p.newUnmounterInternal(volName, podUID, &FlockerUtil{}, p.host.GetMounter()) + return p.newUnmounterInternal(volName, podUID, &FlockerUtil{}, p.host.GetMounter(p.GetPluginName())) } func (p *flockerPlugin) newUnmounterInternal(volName string, podUID types.UID, manager volumeManager, mounter mount.Interface) (volume.Unmounter, error) { diff --git a/pkg/volume/flocker/flocker_test.go b/pkg/volume/flocker/flocker_test.go index 9ffba24befe00..78930d8945129 100644 --- a/pkg/volume/flocker/flocker_test.go +++ b/pkg/volume/flocker/flocker_test.go @@ -127,7 +127,7 @@ func newInitializedVolumePlugMgr(t *testing.T) (*volume.VolumePluginMgr, string) plugMgr := &volume.VolumePluginMgr{} dir, err := utiltesting.MkTmpdir("flocker") assert.NoError(t, err) - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(dir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(dir, nil, nil)) return plugMgr, dir } @@ -138,7 +138,7 @@ func TestPlugin(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/flocker") if err != nil { diff --git a/pkg/volume/flocker/flocker_volume_test.go b/pkg/volume/flocker/flocker_volume_test.go index 8abc62c82e2bd..7f1a2e3071530 100644 --- a/pkg/volume/flocker/flocker_volume_test.go +++ b/pkg/volume/flocker/flocker_volume_test.go @@ -35,7 +35,7 @@ func newTestableProvisioner(assert *assert.Assertions, options volume.VolumeOpti assert.NoError(err, fmt.Sprintf("can't make a temp dir: %v", err)) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName(pluginName) assert.NoError(err, "Can't find the plugin by name") diff --git a/pkg/volume/gce_pd/attacher.go b/pkg/volume/gce_pd/attacher.go index 0cf8b3d6727d8..588d9aabe76d1 100644 --- a/pkg/volume/gce_pd/attacher.go +++ b/pkg/volume/gce_pd/attacher.go @@ -25,13 +25,14 @@ import ( "time" "github.com/golang/glog" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" + "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) type gcePersistentDiskAttacher struct { @@ -56,7 +57,7 @@ func (plugin *gcePersistentDiskPlugin) NewAttacher() (volume.Attacher, error) { } func (plugin *gcePersistentDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mount.GetMountRefs(mounter, deviceMountPath) } @@ -130,7 +131,7 @@ func (attacher *gcePersistentDiskAttacher) VolumesAreAttached(specs []*volume.Sp return volumesAttachedCheck, nil } -func (attacher *gcePersistentDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { +func (attacher *gcePersistentDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { ticker := time.NewTicker(checkSleepDuration) defer ticker.Stop() timer := time.NewTimer(timeout) @@ -185,7 +186,7 @@ func (attacher *gcePersistentDiskAttacher) GetDeviceMountPath( func (attacher *gcePersistentDiskAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { // Only mount the PD globally once. - mounter := attacher.host.GetMounter() + mounter := attacher.host.GetMounter(gcePersistentDiskPluginName) notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { if os.IsNotExist(err) { @@ -208,7 +209,7 @@ func (attacher *gcePersistentDiskAttacher) MountDevice(spec *volume.Spec, device options = append(options, "ro") } if notMnt { - diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()} + diskMounter := volumehelper.NewSafeFormatAndMountFromHost(gcePersistentDiskPluginName, attacher.host) mountOptions := volume.MountOptionFromSpec(spec, options...) err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions) if err != nil { @@ -272,5 +273,5 @@ func (detacher *gcePersistentDiskDetacher) Detach(deviceMountPath string, nodeNa } func (detacher *gcePersistentDiskDetacher) UnmountDevice(deviceMountPath string) error { - return volumeutil.UnmountPath(deviceMountPath, detacher.host.GetMounter()) + return volumeutil.UnmountPath(deviceMountPath, detacher.host.GetMounter(gcePersistentDiskPluginName)) } diff --git a/pkg/volume/gce_pd/attacher_test.go b/pkg/volume/gce_pd/attacher_test.go index 84de7c55b4bbf..cccb876376b4d 100644 --- a/pkg/volume/gce_pd/attacher_test.go +++ b/pkg/volume/gce_pd/attacher_test.go @@ -202,7 +202,8 @@ func newPlugin() *gcePersistentDiskPlugin { host := volumetest.NewFakeVolumeHost( "/tmp", /* rootDir */ nil, /* kubeClient */ - nil /* plugins */) + nil, /* plugins */ + ) plugins := ProbeVolumePlugins() plugin := plugins[0] plugin.Init(host) diff --git a/pkg/volume/gce_pd/gce_pd.go b/pkg/volume/gce_pd/gce_pd.go index b59835b387729..25f7127fe8419 100644 --- a/pkg/volume/gce_pd/gce_pd.go +++ b/pkg/volume/gce_pd/gce_pd.go @@ -100,7 +100,7 @@ func (plugin *gcePersistentDiskPlugin) GetAccessModes() []v1.PersistentVolumeAcc func (plugin *gcePersistentDiskPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newMounterInternal(spec, pod.UID, &GCEDiskUtil{}, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod.UID, &GCEDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func getVolumeSource( @@ -145,7 +145,7 @@ func (plugin *gcePersistentDiskPlugin) newMounterInternal(spec *volume.Spec, pod func (plugin *gcePersistentDiskPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newUnmounterInternal(volName, podUID, &GCEDiskUtil{}, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, &GCEDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *gcePersistentDiskPlugin) newUnmounterInternal(volName string, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Unmounter, error) { @@ -191,7 +191,7 @@ func (plugin *gcePersistentDiskPlugin) newProvisionerInternal(options volume.Vol } func (plugin *gcePersistentDiskPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName()) sourceName, err := mounter.GetDeviceNameFromMount(mountPath, pluginDir) if err != nil { diff --git a/pkg/volume/gce_pd/gce_pd_test.go b/pkg/volume/gce_pd/gce_pd_test.go index 5f9b3619fe352..285e7b5faf0fc 100644 --- a/pkg/volume/gce_pd/gce_pd_test.go +++ b/pkg/volume/gce_pd/gce_pd_test.go @@ -39,7 +39,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/gce-pd") if err != nil { @@ -63,7 +63,7 @@ func TestGetAccessModes(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/gce-pd") if err != nil { @@ -106,7 +106,7 @@ func TestPlugin(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/gce-pd") if err != nil { @@ -244,13 +244,16 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, client, nil)) plug, _ := plugMgr.FindPluginByName(gcePersistentDiskPluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes spec := volume.NewSpecFromPersistentVolume(pv, true) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{}) + if mounter == nil { + t.Fatalf("Got a nil Mounter") + } if !mounter.GetAttributes().ReadOnly { t.Errorf("Expected true for mounter.IsReadOnly") diff --git a/pkg/volume/git_repo/git_repo_test.go b/pkg/volume/git_repo/git_repo_test.go index e851e43c475b6..1bee3ad218586 100644 --- a/pkg/volume/git_repo/git_repo_test.go +++ b/pkg/volume/git_repo/git_repo_test.go @@ -47,7 +47,7 @@ func TestCanSupport(t *testing.T) { plugMgr := volume.VolumePluginMgr{} tempDir, host := newTestHost(t) defer os.RemoveAll(tempDir) - plugMgr.InitPlugins(ProbeVolumePlugins(), host) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plug, err := plugMgr.FindPluginByName("kubernetes.io/git-repo") if err != nil { @@ -225,7 +225,7 @@ func doTestPlugin(scenario struct { plugMgr := volume.VolumePluginMgr{} rootDir, host := newTestHost(t) defer os.RemoveAll(rootDir) - plugMgr.InitPlugins(ProbeVolumePlugins(), host) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plug, err := plugMgr.FindPluginByName("kubernetes.io/git-repo") if err != nil { diff --git a/pkg/volume/glusterfs/BUILD b/pkg/volume/glusterfs/BUILD index b895c99116a71..558fb615ab9c6 100644 --- a/pkg/volume/glusterfs/BUILD +++ b/pkg/volume/glusterfs/BUILD @@ -32,7 +32,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -55,8 +54,6 @@ go_test( "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library", "//vendor/k8s.io/client-go/testing:go_default_library", "//vendor/k8s.io/client-go/util/testing:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", ], ) diff --git a/pkg/volume/glusterfs/OWNERS b/pkg/volume/glusterfs/OWNERS index 07387b03d2691..75150d589380d 100644 --- a/pkg/volume/glusterfs/OWNERS +++ b/pkg/volume/glusterfs/OWNERS @@ -2,6 +2,7 @@ approvers: - rootfs - saad-ali - jingxu97 +- humblec reviewers: - thockin - smarterclayton diff --git a/pkg/volume/glusterfs/glusterfs.go b/pkg/volume/glusterfs/glusterfs.go index 9088688f3b1db..cb62d07470a64 100644 --- a/pkg/volume/glusterfs/glusterfs.go +++ b/pkg/volume/glusterfs/glusterfs.go @@ -43,17 +43,15 @@ import ( "k8s.io/kubernetes/pkg/volume" volutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) // ProbeVolumePlugins is the primary entrypoint for volume plugins. func ProbeVolumePlugins() []volume.VolumePlugin { - return []volume.VolumePlugin{&glusterfsPlugin{host: nil, exe: exec.New(), gidTable: make(map[string]*MinMaxAllocator)}} + return []volume.VolumePlugin{&glusterfsPlugin{host: nil, gidTable: make(map[string]*MinMaxAllocator)}} } type glusterfsPlugin struct { host volume.VolumeHost - exe exec.Interface gidTable map[string]*MinMaxAllocator gidTableLock sync.Mutex } @@ -144,15 +142,15 @@ func (plugin *glusterfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volu podNs := pod.Namespace kubeClient := plugin.host.GetKubeClient() if kubeClient == nil { - return nil, fmt.Errorf("glusterfs: failed to get kube client to initialize mounter") + return nil, fmt.Errorf("failed to get kube client to initialize mounter") } ep, err := kubeClient.Core().Endpoints(podNs).Get(epName, metav1.GetOptions{}) if err != nil { - glog.Errorf("glusterfs: failed to get endpoints %s[%v]", epName, err) + glog.Errorf("failed to get endpoints %s[%v]", epName, err) return nil, err } - glog.V(1).Infof("glusterfs: endpoints %v", ep) - return plugin.newMounterInternal(spec, ep, pod, plugin.host.GetMounter(), exec.New()) + glog.V(1).Infof("glusterfs pv endpoint %v", ep) + return plugin.newMounterInternal(spec, ep, pod, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *glusterfsPlugin) getGlusterVolumeSource(spec *volume.Spec) (*v1.GlusterfsVolumeSource, bool) { @@ -164,7 +162,7 @@ func (plugin *glusterfsPlugin) getGlusterVolumeSource(spec *volume.Spec) (*v1.Gl return spec.PersistentVolume.Spec.Glusterfs, spec.ReadOnly } -func (plugin *glusterfsPlugin) newMounterInternal(spec *volume.Spec, ep *v1.Endpoints, pod *v1.Pod, mounter mount.Interface, exe exec.Interface) (volume.Mounter, error) { +func (plugin *glusterfsPlugin) newMounterInternal(spec *volume.Spec, ep *v1.Endpoints, pod *v1.Pod, mounter mount.Interface) (volume.Mounter, error) { source, readOnly := plugin.getGlusterVolumeSource(spec) return &glusterfsMounter{ glusterfs: &glusterfs{ @@ -176,13 +174,12 @@ func (plugin *glusterfsPlugin) newMounterInternal(spec *volume.Spec, ep *v1.Endp hosts: ep, path: source.Path, readOnly: readOnly, - exe: exe, mountOptions: volume.MountOptionFromSpec(spec), }, nil } func (plugin *glusterfsPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *glusterfsPlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) { @@ -194,11 +191,6 @@ func (plugin *glusterfsPlugin) newUnmounterInternal(volName string, podUID types }}, nil } -func (plugin *glusterfsPlugin) execCommand(command string, args []string) ([]byte, error) { - cmd := plugin.exe.Command(command, args...) - return cmd.CombinedOutput() -} - func (plugin *glusterfsPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { glusterfsVolume := &v1.Volume{ Name: volumeName, @@ -226,7 +218,6 @@ type glusterfsMounter struct { hosts *v1.Endpoints path string readOnly bool - exe exec.Interface mountOptions []string } @@ -244,10 +235,10 @@ func (b *glusterfsMounter) GetAttributes() volume.Attributes { // to mount the volume are available on the underlying node. // If not, it returns an error func (b *glusterfsMounter) CanMount() error { - exe := exec.New() + exe := b.plugin.host.GetExec(b.plugin.GetPluginName()) switch runtime.GOOS { case "linux": - if _, err := exe.Command("/bin/ls", gciLinuxGlusterMountBinaryPath).CombinedOutput(); err != nil { + if _, err := exe.Run("/bin/ls", gciLinuxGlusterMountBinaryPath); err != nil { return fmt.Errorf("Required binary %s is missing", gciLinuxGlusterMountBinaryPath) } } @@ -261,7 +252,7 @@ func (b *glusterfsMounter) SetUp(fsGroup *int64) error { func (b *glusterfsMounter) SetUpAt(dir string, fsGroup *int64) error { notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) - glog.V(4).Infof("glusterfs: mount set up: %s %v %v", dir, !notMnt, err) + glog.V(4).Infof("mount setup: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { return err } @@ -309,7 +300,7 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error { p := path.Join(b.glusterfs.plugin.host.GetPluginDir(glusterfsPluginName), b.glusterfs.volName) if err := os.MkdirAll(p, 0750); err != nil { - return fmt.Errorf("glusterfs: mkdir failed: %v", err) + return fmt.Errorf("Error creating directory %v: %v", p, err) } // adding log-level ERROR to remove noise @@ -364,12 +355,12 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error { } errs = b.mounter.Mount(ip+":"+b.path, dir, "glusterfs", noAutoMountOptions) if errs == nil { - glog.Infof("glusterfs: successfully mounted %s", dir) + glog.Infof("successfully mounted %s", dir) return nil } } } else { - return fmt.Errorf("glusterfs: failed to execute mount command:[no valid ipaddress found in endpoint address list]") + return fmt.Errorf("failed to execute mount command:[no valid ipaddress found in endpoint address list]") } // Failed mount scenario. @@ -377,9 +368,9 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error { // it all goes in a log file, we will read the log file logErr := readGlusterLog(log, b.pod.Name) if logErr != nil { - return fmt.Errorf("glusterfs: mount failed: %v the following error information was pulled from the glusterfs log to help diagnose this issue: %v", errs, logErr) + return fmt.Errorf("mount failed: %v the following error information was pulled from the glusterfs log to help diagnose this issue: %v", errs, logErr) } - return fmt.Errorf("glusterfs: mount failed: %v", errs) + return fmt.Errorf("mount failed: %v", errs) } @@ -433,11 +424,11 @@ type glusterfsVolumeProvisioner struct { func convertGid(gidString string) (int, error) { gid64, err := strconv.ParseInt(gidString, 10, 32) if err != nil { - return 0, fmt.Errorf("glusterfs: failed to parse gid %v ", gidString) + return 0, fmt.Errorf("failed to parse gid %v ", gidString) } if gid64 < 0 { - return 0, fmt.Errorf("glusterfs: negative GIDs are not allowed: %v", gidString) + return 0, fmt.Errorf("negative GIDs are not allowed: %v", gidString) } // ParseInt returns a int64, but since we parsed only @@ -495,11 +486,11 @@ func (d *glusterfsVolumeDeleter) GetPath() string { func (plugin *glusterfsPlugin) collectGids(className string, gidTable *MinMaxAllocator) error { kubeClient := plugin.host.GetKubeClient() if kubeClient == nil { - return fmt.Errorf("glusterfs: failed to get kube client when collecting gids") + return fmt.Errorf("failed to get kube client when collecting gids") } pvList, err := kubeClient.Core().PersistentVolumes().List(metav1.ListOptions{LabelSelector: labels.Everything().String()}) if err != nil { - glog.Errorf("glusterfs: failed to get existing persistent volumes") + glog.Errorf("failed to get existing persistent volumes") return err } @@ -513,7 +504,7 @@ func (plugin *glusterfsPlugin) collectGids(className string, gidTable *MinMaxAll gidStr, ok := pv.Annotations[volumehelper.VolumeGidAnnotationKey] if !ok { - glog.Warningf("glusterfs: no gid found in pv '%v'", pvName) + glog.Warningf("no GID found in pv '%v'", pvName) continue } @@ -525,9 +516,9 @@ func (plugin *glusterfsPlugin) collectGids(className string, gidTable *MinMaxAll _, err = gidTable.Allocate(gid) if err == ErrConflict { - glog.Warningf("glusterfs: gid %v found in pv %v was already allocated", gid) + glog.Warningf("GID %v found in pv %v was already allocated", gid) } else if err != nil { - glog.Errorf("glusterfs: failed to store gid %v found in pv '%v': %v", gid, pvName, err) + glog.Errorf("failed to store gid %v found in pv '%v': %v", gid, pvName, err) return err } } @@ -604,7 +595,7 @@ func (d *glusterfsVolumeDeleter) getGid() (int, bool, error) { } func (d *glusterfsVolumeDeleter) Delete() error { - glog.V(2).Infof("glusterfs: delete volume: %s ", d.glusterfsMounter.path) + glog.V(2).Infof("delete volume: %s ", d.glusterfsMounter.path) volumeName := d.glusterfsMounter.path volumeID := dstrings.TrimPrefix(volumeName, volPrefix) class, err := volutil.GetClassForVolume(d.plugin.host.GetKubeClient(), d.spec) @@ -618,7 +609,7 @@ func (d *glusterfsVolumeDeleter) Delete() error { } d.provisionerConfig = *cfg - glog.V(4).Infof("glusterfs: deleting volume %q with configuration %+v", volumeID, d.provisionerConfig) + glog.V(4).Infof("deleting volume %q with configuration %+v", volumeID, d.provisionerConfig) gid, exists, err := d.getGid() if err != nil { @@ -626,48 +617,48 @@ func (d *glusterfsVolumeDeleter) Delete() error { } else if exists { gidTable, err := d.plugin.getGidTable(class.Name, cfg.gidMin, cfg.gidMax) if err != nil { - return fmt.Errorf("glusterfs: failed to get gidTable: %v", err) + return fmt.Errorf("failed to get gidTable: %v", err) } err = gidTable.Release(gid) if err != nil { - return fmt.Errorf("glusterfs: failed to release gid %v: %v", gid, err) + return fmt.Errorf("failed to release gid %v: %v", gid, err) } } cli := gcli.NewClient(d.url, d.user, d.secretValue) if cli == nil { - glog.Errorf("glusterfs: failed to create glusterfs rest client") - return fmt.Errorf("glusterfs: failed to create glusterfs rest client, REST server authentication failed") + glog.Errorf("failed to create glusterfs rest client") + return fmt.Errorf("failed to create glusterfs rest client, REST server authentication failed") } err = cli.VolumeDelete(volumeID) if err != nil { - glog.Errorf("glusterfs: error when deleting the volume :%v", err) + glog.Errorf("error when deleting the volume :%v", err) return err } - glog.V(2).Infof("glusterfs: volume %s deleted successfully", volumeName) + glog.V(2).Infof("volume %s deleted successfully", volumeName) //Deleter takes endpoint and endpointnamespace from pv spec. pvSpec := d.spec.Spec var dynamicEndpoint, dynamicNamespace string if pvSpec.ClaimRef == nil { - glog.Errorf("glusterfs: ClaimRef is nil") - return fmt.Errorf("glusterfs: ClaimRef is nil") + glog.Errorf("ClaimRef is nil") + return fmt.Errorf("ClaimRef is nil") } if pvSpec.ClaimRef.Namespace == "" { - glog.Errorf("glusterfs: namespace is nil") - return fmt.Errorf("glusterfs: namespace is nil") + glog.Errorf("namespace is nil") + return fmt.Errorf("namespace is nil") } dynamicNamespace = pvSpec.ClaimRef.Namespace if pvSpec.Glusterfs.EndpointsName != "" { dynamicEndpoint = pvSpec.Glusterfs.EndpointsName } - glog.V(3).Infof("glusterfs: dynamic namespace and endpoint : [%v/%v]", dynamicNamespace, dynamicEndpoint) + glog.V(3).Infof("dynamic namespace and endpoint : [%v/%v]", dynamicNamespace, dynamicEndpoint) err = d.deleteEndpointService(dynamicNamespace, dynamicEndpoint) if err != nil { - glog.Errorf("glusterfs: error when deleting endpoint/service :%v", err) + glog.Errorf("error when deleting endpoint/service :%v", err) } else { - glog.V(1).Infof("glusterfs: [%v/%v] deleted successfully ", dynamicNamespace, dynamicEndpoint) + glog.V(1).Infof("endpoint: %v and service: %v deleted successfully ", dynamicNamespace, dynamicEndpoint) } return nil } @@ -678,10 +669,10 @@ func (p *glusterfsVolumeProvisioner) Provision() (*v1.PersistentVolume, error) { } if p.options.PVC.Spec.Selector != nil { - glog.V(4).Infof("glusterfs: not able to parse your claim Selector") - return nil, fmt.Errorf("glusterfs: not able to parse your claim Selector") + glog.V(4).Infof("not able to parse your claim Selector") + return nil, fmt.Errorf("not able to parse your claim Selector") } - glog.V(4).Infof("glusterfs: Provison VolumeOptions %v", p.options) + glog.V(4).Infof("Provision VolumeOptions %v", p.options) scName := v1helper.GetPersistentVolumeClaimClass(p.options.PVC) cfg, err := parseClassParameters(p.options.Parameters, p.plugin.host.GetKubeClient()) if err != nil { @@ -689,29 +680,29 @@ func (p *glusterfsVolumeProvisioner) Provision() (*v1.PersistentVolume, error) { } p.provisionerConfig = *cfg - glog.V(4).Infof("glusterfs: creating volume with configuration %+v", p.provisionerConfig) + glog.V(4).Infof("creating volume with configuration %+v", p.provisionerConfig) gidTable, err := p.plugin.getGidTable(scName, cfg.gidMin, cfg.gidMax) if err != nil { - return nil, fmt.Errorf("glusterfs: failed to get gidTable: %v", err) + return nil, fmt.Errorf("failed to get gidTable: %v", err) } gid, _, err := gidTable.AllocateNext() if err != nil { - glog.Errorf("glusterfs: failed to reserve gid from table: %v", err) - return nil, fmt.Errorf("glusterfs: failed to reserve gid from table: %v", err) + glog.Errorf("failed to reserve GID from table: %v", err) + return nil, fmt.Errorf("failed to reserve GID from table: %v", err) } - glog.V(2).Infof("glusterfs: got gid [%d] for PVC %s", gid, p.options.PVC.Name) + glog.V(2).Infof("Allocated GID [%d] for PVC %s", gid, p.options.PVC.Name) glusterfs, sizeGB, err := p.CreateVolume(gid) if err != nil { if releaseErr := gidTable.Release(gid); releaseErr != nil { - glog.Errorf("glusterfs: error when releasing gid in storageclass: %s", scName) + glog.Errorf("error when releasing GID in storageclass: %s", scName) } - glog.Errorf("glusterfs: create volume err: %v.", err) - return nil, fmt.Errorf("glusterfs: create volume err: %v", err) + glog.Errorf("create volume error: %v.", err) + return nil, fmt.Errorf("create volume error: %v", err) } pv := new(v1.PersistentVolume) pv.Spec.PersistentVolumeSource.Glusterfs = glusterfs @@ -742,31 +733,31 @@ func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsVolum capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] volSizeBytes := capacity.Value() sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024)) - glog.V(2).Infof("glusterfs: create volume of size: %d bytes and configuration %+v", volSizeBytes, p.provisionerConfig) + glog.V(2).Infof("create volume of size: %d bytes and configuration %+v", volSizeBytes, p.provisionerConfig) if p.url == "" { - glog.Errorf("glusterfs : rest server endpoint is empty") + glog.Errorf("REST server endpoint is empty") return nil, 0, fmt.Errorf("failed to create glusterfs REST client, REST URL is empty") } cli := gcli.NewClient(p.url, p.user, p.secretValue) if cli == nil { - glog.Errorf("glusterfs: failed to create glusterfs rest client") + glog.Errorf("failed to create glusterfs rest client") return nil, 0, fmt.Errorf("failed to create glusterfs REST client, REST server authentication failed") } if p.provisionerConfig.clusterID != "" { clusterIDs = dstrings.Split(p.clusterID, ",") - glog.V(4).Infof("glusterfs: provided clusterIDs: %v", clusterIDs) + glog.V(4).Infof("provided clusterIDs: %v", clusterIDs) } gid64 := int64(gid) volumeReq := &gapi.VolumeCreateRequest{Size: sz, Clusters: clusterIDs, Gid: gid64, Durability: p.volumeType, GlusterVolumeOptions: p.volumeOptions} volume, err := cli.VolumeCreate(volumeReq) if err != nil { - glog.Errorf("glusterfs: error creating volume %v ", err) + glog.Errorf("error creating volume %v ", err) return nil, 0, fmt.Errorf("error creating volume %v", err) } - glog.V(1).Infof("glusterfs: volume with size: %d and name: %s created", volume.Size, volume.Name) + glog.V(1).Infof("volume with size: %d and name: %s created", volume.Size, volume.Name) dynamicHostIps, err := getClusterNodes(cli, volume.Cluster) if err != nil { - glog.Errorf("glusterfs: error [%v] when getting cluster nodes for volume %s", err, volume) + glog.Errorf("error [%v] when getting cluster nodes for volume %s", err, volume) return nil, 0, fmt.Errorf("error [%v] when getting cluster nodes for volume %s", err, volume) } @@ -778,14 +769,14 @@ func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsVolum epNamespace := p.options.PVC.Namespace endpoint, service, err := p.createEndpointService(epNamespace, epServiceName, dynamicHostIps, p.options.PVC.Name) if err != nil { - glog.Errorf("glusterfs: failed to create endpoint/service: %v", err) + glog.Errorf("failed to create endpoint/service: %v", err) deleteErr := cli.VolumeDelete(volume.Id) if deleteErr != nil { - glog.Errorf("glusterfs: error when deleting the volume :%v , manual deletion required", deleteErr) + glog.Errorf("error when deleting the volume :%v , manual deletion required", deleteErr) } return nil, 0, fmt.Errorf("failed to create endpoint/service %v", err) } - glog.V(3).Infof("glusterfs: dynamic ep %v and svc : %v ", endpoint, service) + glog.V(3).Infof("dynamic ep %v and svc : %v ", endpoint, service) return &v1.GlusterfsVolumeSource{ EndpointsName: endpoint.Name, Path: volume.Name, @@ -814,15 +805,15 @@ func (p *glusterfsVolumeProvisioner) createEndpointService(namespace string, epS } kubeClient := p.plugin.host.GetKubeClient() if kubeClient == nil { - return nil, nil, fmt.Errorf("glusterfs: failed to get kube client when creating endpoint service") + return nil, nil, fmt.Errorf("failed to get kube client when creating endpoint service") } _, err = kubeClient.Core().Endpoints(namespace).Create(endpoint) if err != nil && errors.IsAlreadyExists(err) { - glog.V(1).Infof("glusterfs: endpoint [%s] already exist in namespace [%s]", endpoint, namespace) + glog.V(1).Infof("endpoint [%s] already exist in namespace [%s]", endpoint, namespace) err = nil } if err != nil { - glog.Errorf("glusterfs: failed to create endpoint: %v", err) + glog.Errorf("failed to create endpoint: %v", err) return nil, nil, fmt.Errorf("error creating endpoint: %v", err) } service = &v1.Service{ @@ -838,11 +829,11 @@ func (p *glusterfsVolumeProvisioner) createEndpointService(namespace string, epS {Protocol: "TCP", Port: 1}}}} _, err = kubeClient.Core().Services(namespace).Create(service) if err != nil && errors.IsAlreadyExists(err) { - glog.V(1).Infof("glusterfs: service [%s] already exist in namespace [%s]", service, namespace) + glog.V(1).Infof("service [%s] already exist in namespace [%s]", service, namespace) err = nil } if err != nil { - glog.Errorf("glusterfs: failed to create service: %v", err) + glog.Errorf("failed to create service: %v", err) return nil, nil, fmt.Errorf("error creating service: %v", err) } return endpoint, service, nil @@ -851,14 +842,14 @@ func (p *glusterfsVolumeProvisioner) createEndpointService(namespace string, epS func (d *glusterfsVolumeDeleter) deleteEndpointService(namespace string, epServiceName string) (err error) { kubeClient := d.plugin.host.GetKubeClient() if kubeClient == nil { - return fmt.Errorf("glusterfs: failed to get kube client when deleting endpoint service") + return fmt.Errorf("failed to get kube client when deleting endpoint service") } err = kubeClient.Core().Services(namespace).Delete(epServiceName, nil) if err != nil { - glog.Errorf("glusterfs: error deleting service %s/%s: %v", namespace, epServiceName, err) + glog.Errorf("error deleting service %s/%s: %v", namespace, epServiceName, err) return fmt.Errorf("error deleting service %s/%s: %v", namespace, epServiceName, err) } - glog.V(1).Infof("glusterfs: service/endpoint %s/%s deleted successfully", namespace, epServiceName) + glog.V(1).Infof("service/endpoint %s/%s deleted successfully", namespace, epServiceName) return nil } @@ -888,7 +879,7 @@ func parseSecret(namespace, secretName string, kubeClient clientset.Interface) ( func getClusterNodes(cli *gcli.Client, cluster string) (dynamicHostIps []string, err error) { clusterinfo, err := cli.ClusterInfo(cluster) if err != nil { - glog.Errorf("glusterfs: failed to get cluster details: %v", err) + glog.Errorf("failed to get cluster details: %v", err) return nil, fmt.Errorf("failed to get cluster details: %v", err) } @@ -898,15 +889,15 @@ func getClusterNodes(cli *gcli.Client, cluster string) (dynamicHostIps []string, for _, node := range clusterinfo.Nodes { nodei, err := cli.NodeInfo(string(node)) if err != nil { - glog.Errorf("glusterfs: failed to get hostip: %v", err) + glog.Errorf(" failed to get hostip: %v", err) return nil, fmt.Errorf("failed to get hostip: %v", err) } ipaddr := dstrings.Join(nodei.NodeAddRequest.Hostnames.Storage, "") dynamicHostIps = append(dynamicHostIps, ipaddr) } - glog.V(3).Infof("glusterfs: hostlist :%v", dynamicHostIps) + glog.V(3).Infof("hostlist :%v", dynamicHostIps) if len(dynamicHostIps) == 0 { - glog.Errorf("glusterfs: no hosts found: %v", err) + glog.Errorf("no hosts found: %v", err) return nil, fmt.Errorf("no hosts found: %v", err) } return dynamicHostIps, nil @@ -944,25 +935,25 @@ func parseClassParameters(params map[string]string, kubeClient clientset.Interfa case "gidmin": parseGidMin, err := convertGid(v) if err != nil { - return nil, fmt.Errorf("glusterfs: invalid value %q for volume plugin %s", k, glusterfsPluginName) + return nil, fmt.Errorf("invalid value %q for volume plugin %s", k, glusterfsPluginName) } if parseGidMin < absoluteGidMin { - return nil, fmt.Errorf("glusterfs: gidMin must be >= %v", absoluteGidMin) + return nil, fmt.Errorf("gidMin must be >= %v", absoluteGidMin) } if parseGidMin > absoluteGidMax { - return nil, fmt.Errorf("glusterfs: gidMin must be <= %v", absoluteGidMax) + return nil, fmt.Errorf("gidMin must be <= %v", absoluteGidMax) } cfg.gidMin = parseGidMin case "gidmax": parseGidMax, err := convertGid(v) if err != nil { - return nil, fmt.Errorf("glusterfs: invalid value %q for volume plugin %s", k, glusterfsPluginName) + return nil, fmt.Errorf("invalid value %q for volume plugin %s", k, glusterfsPluginName) } if parseGidMax < absoluteGidMin { - return nil, fmt.Errorf("glusterfs: gidMax must be >= %v", absoluteGidMin) + return nil, fmt.Errorf("gidMax must be >= %v", absoluteGidMin) } if parseGidMax > absoluteGidMax { - return nil, fmt.Errorf("glusterfs: gidMax must be <= %v", absoluteGidMax) + return nil, fmt.Errorf("gidMax must be <= %v", absoluteGidMax) } cfg.gidMax = parseGidMax case "volumetype": @@ -974,7 +965,7 @@ func parseClassParameters(params map[string]string, kubeClient clientset.Interfa } default: - return nil, fmt.Errorf("glusterfs: invalid option %q for volume plugin %s", k, glusterfsPluginName) + return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, glusterfsPluginName) } } diff --git a/pkg/volume/glusterfs/glusterfs_test.go b/pkg/volume/glusterfs/glusterfs_test.go index 687259ec171d1..ba8a2fb37268f 100644 --- a/pkg/volume/glusterfs/glusterfs_test.go +++ b/pkg/volume/glusterfs/glusterfs_test.go @@ -33,8 +33,6 @@ import ( "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" ) func TestCanSupport(t *testing.T) { @@ -45,7 +43,7 @@ func TestCanSupport(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/glusterfs") if err != nil { t.Errorf("Can't find the plugin by name") @@ -69,7 +67,7 @@ func TestGetAccessModes(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/glusterfs") if err != nil { @@ -97,29 +95,15 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/glusterfs") if err != nil { t.Errorf("Can't find the plugin by name") } ep := &v1.Endpoints{ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Subsets: []v1.EndpointSubset{{ Addresses: []v1.EndpointAddress{{IP: "127.0.0.1"}}}}} - var fcmd fakeexec.FakeCmd - fcmd = fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ - // mount - func() ([]byte, error) { - return []byte{}, nil - }, - }, - } - fake := fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - } pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} - mounter, err := plug.(*glusterfsPlugin).newMounterInternal(spec, ep, pod, &mount.FakeMounter{}, &fake) + mounter, err := plug.(*glusterfsPlugin).newMounterInternal(spec, ep, pod, &mount.FakeMounter{}) volumePath := mounter.GetPath() if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) @@ -229,7 +213,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { client := fake.NewSimpleClientset(pv, claim, ep) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, client, nil)) plug, _ := plugMgr.FindPluginByName(glusterfsPluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes diff --git a/pkg/volume/host_path/BUILD b/pkg/volume/host_path/BUILD index d55dabee5ce7b..c209bd636968e 100644 --- a/pkg/volume/host_path/BUILD +++ b/pkg/volume/host_path/BUILD @@ -11,7 +11,20 @@ go_library( srcs = [ "doc.go", "host_path.go", - ], + "nsenter_unsupported.go", + ] + select({ + "@io_bazel_rules_go//go/platform:darwin_amd64": [ + "host_path_unix.go", + ], + "@io_bazel_rules_go//go/platform:linux_amd64": [ + "host_path_unix.go", + "nsenter.go", + ], + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "host_path_windows.go", + ], + "//conditions:default": [], + }), deps = [ "//pkg/volume:go_default_library", "//pkg/volume/util/volumehelper:go_default_library", @@ -20,7 +33,12 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:linux_amd64": [ + "//vendor/k8s.io/utils/exec:go_default_library", + ], + "//conditions:default": [], + }), ) go_test( diff --git a/pkg/volume/host_path/host_path.go b/pkg/volume/host_path/host_path.go index 13767047bd349..0b5f4b906a665 100644 --- a/pkg/volume/host_path/host_path.go +++ b/pkg/volume/host_path/host_path.go @@ -99,14 +99,15 @@ func (plugin *hostPathPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { } } -func (plugin *hostPathPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { +func (plugin *hostPathPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volume.VolumeOptions) (volume.Mounter, error) { hostPathVolumeSource, readOnly, err := getVolumeSource(spec) if err != nil { return nil, err } + path := hostPathVolumeSource.Path return &hostPathMounter{ - hostPath: &hostPath{path: hostPathVolumeSource.Path}, + hostPath: &hostPath{path: path, pathType: hostPathVolumeSource.Type, containerized: opts.Containerized}, readOnly: readOnly, }, nil } @@ -175,7 +176,9 @@ func newProvisioner(options volume.VolumeOptions, host volume.VolumeHost, plugin // HostPath volumes represent a bare host file or directory mount. // The direct at the specified path will be directly exposed to the container. type hostPath struct { - path string + path string + pathType *v1.HostPathType + containerized bool volume.MetricsNil } @@ -211,7 +214,11 @@ func (b *hostPathMounter) SetUp(fsGroup *int64) error { if err != nil { return fmt.Errorf("invalid HostPath `%s`: %v", b.GetPath(), err) } - return nil + + if *b.pathType == v1.HostPathUnset { + return nil + } + return checkType(b.GetPath(), b.pathType, b.containerized) } // SetUpAt does not make sense for host paths - probably programmer error. @@ -314,3 +321,182 @@ func getVolumeSource(spec *volume.Spec) (*v1.HostPathVolumeSource, bool, error) return nil, false, fmt.Errorf("Spec does not reference an HostPath volume type") } + +type hostPathTypeChecker interface { + Exists() bool + IsFile() bool + MakeFile() error + IsDir() bool + MakeDir() error + IsBlock() bool + IsChar() bool + IsSocket() bool + GetPath() string +} + +type fileTypeChecker interface { + getFileType(fileInfo os.FileInfo) (v1.HostPathType, error) +} + +// this is implemented in per-OS files +type defaultFileTypeChecker struct{} + +type osFileTypeChecker struct { + path string + exists bool + info os.FileInfo + checker fileTypeChecker +} + +func (ftc *osFileTypeChecker) Exists() bool { + return ftc.exists +} + +func (ftc *osFileTypeChecker) IsFile() bool { + if !ftc.Exists() { + return false + } + return !ftc.info.IsDir() +} + +func (ftc *osFileTypeChecker) MakeFile() error { + f, err := os.OpenFile(ftc.path, os.O_CREATE, os.FileMode(0644)) + defer f.Close() + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +func (ftc *osFileTypeChecker) IsDir() bool { + if !ftc.Exists() { + return false + } + return ftc.info.IsDir() +} + +func (ftc *osFileTypeChecker) MakeDir() error { + err := os.MkdirAll(ftc.path, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +func (ftc *osFileTypeChecker) IsBlock() bool { + if !ftc.Exists() { + return false + } + + blkDevType, err := ftc.checker.getFileType(ftc.info) + if err != nil { + return false + } + return blkDevType == v1.HostPathBlockDev +} + +func (ftc *osFileTypeChecker) IsChar() bool { + if !ftc.Exists() { + return false + } + + charDevType, err := ftc.checker.getFileType(ftc.info) + if err != nil { + return false + } + return charDevType == v1.HostPathCharDev +} + +func (ftc *osFileTypeChecker) IsSocket() bool { + if !ftc.Exists() { + return false + } + + socketType, err := ftc.checker.getFileType(ftc.info) + if err != nil { + return false + } + return socketType == v1.HostPathSocket +} + +func (ftc *osFileTypeChecker) GetPath() string { + return ftc.path +} + +func newOSFileTypeChecker(path string, checker fileTypeChecker) (hostPathTypeChecker, error) { + ftc := osFileTypeChecker{path: path, checker: checker} + info, err := os.Stat(path) + if err != nil { + ftc.exists = false + if !os.IsNotExist(err) { + return nil, err + } + } else { + ftc.info = info + ftc.exists = true + } + return &ftc, nil +} + +func checkType(path string, pathType *v1.HostPathType, containerized bool) error { + var ftc hostPathTypeChecker + var err error + if containerized { + // For a containerized kubelet, use nsenter to run commands in + // the host's mount namespace. + // TODO(dixudx): setns into docker's mount namespace, and then run the exact same go code for checks/setup + ftc, err = newNsenterFileTypeChecker(path) + if err != nil { + return err + } + } else { + ftc, err = newOSFileTypeChecker(path, &defaultFileTypeChecker{}) + if err != nil { + return err + } + } + return checkTypeInternal(ftc, pathType) +} + +func checkTypeInternal(ftc hostPathTypeChecker, pathType *v1.HostPathType) error { + switch *pathType { + case v1.HostPathDirectoryOrCreate: + if !ftc.Exists() { + return ftc.MakeDir() + } + fallthrough + case v1.HostPathDirectory: + if !ftc.IsDir() { + return fmt.Errorf("hostPath type check failed: %s is not a directory", ftc.GetPath()) + } + case v1.HostPathFileOrCreate: + if !ftc.Exists() { + return ftc.MakeFile() + } + fallthrough + case v1.HostPathFile: + if !ftc.IsFile() { + return fmt.Errorf("hostPath type check failed: %s is not a file", ftc.GetPath()) + } + case v1.HostPathSocket: + if !ftc.IsSocket() { + return fmt.Errorf("hostPath type check failed: %s is not a socket file", ftc.GetPath()) + } + case v1.HostPathCharDev: + if !ftc.IsChar() { + return fmt.Errorf("hostPath type check failed: %s is not a character device", ftc.GetPath()) + } + case v1.HostPathBlockDev: + if !ftc.IsBlock() { + return fmt.Errorf("hostPath type check failed: %s is not a block device", ftc.GetPath()) + } + default: + return fmt.Errorf("%s is an invalid volume type", *pathType) + } + + return nil +} diff --git a/pkg/volume/host_path/host_path_test.go b/pkg/volume/host_path/host_path_test.go index a16b2ee89a630..1184f5d35ad13 100644 --- a/pkg/volume/host_path/host_path_test.go +++ b/pkg/volume/host_path/host_path_test.go @@ -34,9 +34,24 @@ import ( volumetest "k8s.io/kubernetes/pkg/volume/testing" ) +func newHostPathType(pathType string) *v1.HostPathType { + hostPathType := new(v1.HostPathType) + *hostPathType = v1.HostPathType(pathType) + return hostPathType +} + +func newHostPathTypeList(pathType ...string) []*v1.HostPathType { + typeList := []*v1.HostPathType{} + for _, ele := range pathType { + typeList = append(typeList, newHostPathType(ele)) + } + + return typeList +} + func TestCanSupport(t *testing.T) { plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost("fake", nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost("fake", nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/host-path") if err != nil { @@ -58,7 +73,7 @@ func TestCanSupport(t *testing.T) { func TestGetAccessModes(t *testing.T) { plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/host-path") if err != nil { @@ -72,7 +87,7 @@ func TestGetAccessModes(t *testing.T) { func TestRecycler(t *testing.T) { plugMgr := volume.VolumePluginMgr{} pluginHost := volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil) - plugMgr.InitPlugins([]volume.VolumePlugin{&hostPathPlugin{nil, volume.VolumeConfig{}}}, pluginHost) + plugMgr.InitPlugins([]volume.VolumePlugin{&hostPathPlugin{nil, volume.VolumeConfig{}}}, nil, pluginHost) spec := &volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/foo"}}}}} _, err := plugMgr.FindRecyclablePluginBySpec(spec) @@ -91,7 +106,7 @@ func TestDeleter(t *testing.T) { } plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil)) spec := &volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{Path: tempPath}}}}} plug, err := plugMgr.FindDeletablePluginBySpec(spec) @@ -108,7 +123,7 @@ func TestDeleter(t *testing.T) { if err := deleter.Delete(); err != nil { t.Errorf("Mock Recycler expected to return nil but got %s", err) } - if exists, _ := utilfile.FileExists("foo"); exists { + if exists, _ := utilfile.FileExists(tempPath); exists { t.Errorf("Temp path expected to be deleted, but was found at %s", tempPath) } } @@ -125,7 +140,7 @@ func TestDeleterTempDir(t *testing.T) { for name, test := range tests { plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil)) spec := &volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{Path: test.path}}}}} plug, _ := plugMgr.FindDeletablePluginBySpec(spec) deleter, _ := plug.NewDeleter(spec) @@ -146,6 +161,7 @@ func TestProvisioner(t *testing.T) { plugMgr := volume.VolumePluginMgr{} plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{ProvisioningEnabled: true}), + nil, volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil)) spec := &volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{Path: tempPath}}}}} plug, err := plugMgr.FindCreatablePluginBySpec(spec) @@ -184,7 +200,7 @@ func TestProvisioner(t *testing.T) { func TestInvalidHostPath(t *testing.T) { plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost("fake", nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost("fake", nil, nil)) plug, err := plugMgr.FindPluginByName(hostPathPluginName) if err != nil { @@ -209,27 +225,30 @@ func TestInvalidHostPath(t *testing.T) { func TestPlugin(t *testing.T) { plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost("fake", nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost("fake", nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/host-path") if err != nil { t.Errorf("Can't find the plugin by name") } + + volPath := "/tmp/vol1" spec := &v1.Volume{ Name: "vol1", - VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/vol1"}}, + VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: volPath, Type: newHostPathType(string(v1.HostPathDirectoryOrCreate))}}, } pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} + defer os.RemoveAll(volPath) mounter, err := plug.NewMounter(volume.NewSpecFromVolume(spec), pod, volume.VolumeOptions{}) if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) } if mounter == nil { - t.Errorf("Got a nil Mounter") + t.Fatalf("Got a nil Mounter") } path := mounter.GetPath() - if path != "/vol1" { + if path != volPath { t.Errorf("Got unexpected path: %s", path) } @@ -242,7 +261,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Failed to make a new Unmounter: %v", err) } if unmounter == nil { - t.Errorf("Got a nil Unmounter") + t.Fatalf("Got a nil Unmounter") } if err := unmounter.TearDown(); err != nil { @@ -257,13 +276,14 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { }, Spec: v1.PersistentVolumeSpec{ PersistentVolumeSource: v1.PersistentVolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "foo"}, + HostPath: &v1.HostPathVolumeSource{Path: "foo", Type: newHostPathType(string(v1.HostPathDirectoryOrCreate))}, }, ClaimRef: &v1.ObjectReference{ Name: "claimA", }, }, } + defer os.RemoveAll("foo") claim := &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -281,15 +301,312 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { client := fake.NewSimpleClientset(pv, claim) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost("/tmp/fake", client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost("/tmp/fake", client, nil)) plug, _ := plugMgr.FindPluginByName(hostPathPluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes spec := volume.NewSpecFromPersistentVolume(pv, true) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{}) + if mounter == nil { + t.Fatalf("Got a nil Mounter") + } if !mounter.GetAttributes().ReadOnly { t.Errorf("Expected true for mounter.IsReadOnly") } } + +type fakeFileTypeChecker struct { + desiredType string +} + +func (fftc *fakeFileTypeChecker) getFileType(_ os.FileInfo) (v1.HostPathType, error) { + return *newHostPathType(fftc.desiredType), nil +} + +func setUp() error { + err := os.MkdirAll("/tmp/ExistingFolder", os.FileMode(0755)) + if err != nil { + return err + } + + f, err := os.OpenFile("/tmp/ExistingFolder/foo", os.O_CREATE, os.FileMode(0644)) + defer f.Close() + if err != nil { + return err + } + + return nil +} + +func tearDown() { + os.RemoveAll("/tmp/ExistingFolder") +} + +func TestOSFileTypeChecker(t *testing.T) { + err := setUp() + if err != nil { + t.Error(err) + } + defer tearDown() + testCases := []struct { + name string + path string + desiredType string + isDir bool + isFile bool + isSocket bool + isBlock bool + isChar bool + }{ + { + name: "Existing Folder", + path: "/tmp/ExistingFolder", + isDir: true, + }, + { + name: "Existing File", + path: "/tmp/ExistingFolder/foo", + isFile: true, + }, + { + name: "Existing Socket File", + path: "/tmp/ExistingFolder/foo", + desiredType: string(v1.HostPathSocket), + isSocket: true, + }, + { + name: "Existing Character Device", + path: "/tmp/ExistingFolder/foo", + desiredType: string(v1.HostPathCharDev), + isChar: true, + }, + { + name: "Existing Block Device", + path: "/tmp/ExistingFolder/foo", + desiredType: string(v1.HostPathBlockDev), + isBlock: true, + }, + } + + for i, tc := range testCases { + oftc, err := newOSFileTypeChecker(tc.path, + &fakeFileTypeChecker{desiredType: tc.desiredType}) + if err != nil { + t.Errorf("[%d: %q] expect nil, but got %v", i, tc.name, err) + } + + path := oftc.GetPath() + if path != tc.path { + t.Errorf("[%d: %q] got unexpected path: %s", i, tc.name, path) + } + + exist := oftc.Exists() + if !exist { + t.Errorf("[%d: %q] path: %s does not exist", i, tc.name, path) + } + + if tc.isDir { + if !oftc.IsDir() { + t.Errorf("[%d: %q] expected folder, got unexpected: %s", i, tc.name, path) + } + if oftc.IsFile() { + t.Errorf("[%d: %q] expected folder, got unexpected file: %s", i, tc.name, path) + } + if oftc.IsSocket() { + t.Errorf("[%d: %q] expected folder, got unexpected socket file: %s", i, tc.name, path) + } + if oftc.IsBlock() { + t.Errorf("[%d: %q] expected folder, got unexpected block device: %s", i, tc.name, path) + } + if oftc.IsChar() { + t.Errorf("[%d: %q] expected folder, got unexpected character device: %s", i, tc.name, path) + } + } + + if tc.isFile { + if !oftc.IsFile() { + t.Errorf("[%d: %q] expected file, got unexpected: %s", i, tc.name, path) + } + if oftc.IsDir() { + t.Errorf("[%d: %q] expected file, got unexpected folder: %s", i, tc.name, path) + } + if oftc.IsSocket() { + t.Errorf("[%d: %q] expected file, got unexpected socket file: %s", i, tc.name, path) + } + if oftc.IsBlock() { + t.Errorf("[%d: %q] expected file, got unexpected block device: %s", i, tc.name, path) + } + if oftc.IsChar() { + t.Errorf("[%d: %q] expected file, got unexpected character device: %s", i, tc.name, path) + } + } + + if tc.isSocket { + if !oftc.IsSocket() { + t.Errorf("[%d: %q] expected socket file, got unexpected: %s", i, tc.name, path) + } + if oftc.IsDir() { + t.Errorf("[%d: %q] expected socket file, got unexpected folder: %s", i, tc.name, path) + } + if !oftc.IsFile() { + t.Errorf("[%d: %q] expected socket file, got unexpected file: %s", i, tc.name, path) + } + if oftc.IsBlock() { + t.Errorf("[%d: %q] expected socket file, got unexpected block device: %s", i, tc.name, path) + } + if oftc.IsChar() { + t.Errorf("[%d: %q] expected socket file, got unexpected character device: %s", i, tc.name, path) + } + } + + if tc.isChar { + if !oftc.IsChar() { + t.Errorf("[%d: %q] expected character device, got unexpected: %s", i, tc.name, path) + } + if oftc.IsDir() { + t.Errorf("[%d: %q] expected character device, got unexpected folder: %s", i, tc.name, path) + } + if !oftc.IsFile() { + t.Errorf("[%d: %q] expected character device, got unexpected file: %s", i, tc.name, path) + } + if oftc.IsSocket() { + t.Errorf("[%d: %q] expected character device, got unexpected socket file: %s", i, tc.name, path) + } + if oftc.IsBlock() { + t.Errorf("[%d: %q] expected character device, got unexpected block device: %s", i, tc.name, path) + } + } + + if tc.isBlock { + if !oftc.IsBlock() { + t.Errorf("[%d: %q] expected block device, got unexpected: %s", i, tc.name, path) + } + if oftc.IsDir() { + t.Errorf("[%d: %q] expected block device, got unexpected folder: %s", i, tc.name, path) + } + if !oftc.IsFile() { + t.Errorf("[%d: %q] expected block device, got unexpected file: %s", i, tc.name, path) + } + if oftc.IsSocket() { + t.Errorf("[%d: %q] expected block device, got unexpected socket file: %s", i, tc.name, path) + } + if oftc.IsChar() { + t.Errorf("[%d: %q] expected block device, got unexpected character device: %s", i, tc.name, path) + } + } + } + +} + +type fakeHostPathTypeChecker struct { + name string + path string + exists bool + isDir bool + isFile bool + isSocket bool + isBlock bool + isChar bool + validpathType []*v1.HostPathType + invalidpathType []*v1.HostPathType +} + +func (ftc *fakeHostPathTypeChecker) MakeFile() error { return nil } +func (ftc *fakeHostPathTypeChecker) MakeDir() error { return nil } +func (ftc *fakeHostPathTypeChecker) Exists() bool { return ftc.exists } +func (ftc *fakeHostPathTypeChecker) IsFile() bool { return ftc.isFile } +func (ftc *fakeHostPathTypeChecker) IsDir() bool { return ftc.isDir } +func (ftc *fakeHostPathTypeChecker) IsBlock() bool { return ftc.isBlock } +func (ftc *fakeHostPathTypeChecker) IsChar() bool { return ftc.isChar } +func (ftc *fakeHostPathTypeChecker) IsSocket() bool { return ftc.isSocket } +func (ftc *fakeHostPathTypeChecker) GetPath() string { return ftc.path } + +func TestHostPathTypeCheckerInternal(t *testing.T) { + testCases := []fakeHostPathTypeChecker{ + { + name: "Existing Folder", + path: "/existingFolder", + isDir: true, + exists: true, + validpathType: newHostPathTypeList(string(v1.HostPathDirectoryOrCreate), string(v1.HostPathDirectory)), + invalidpathType: newHostPathTypeList(string(v1.HostPathFileOrCreate), string(v1.HostPathFile), + string(v1.HostPathSocket), string(v1.HostPathCharDev), string(v1.HostPathBlockDev)), + }, + { + name: "New Folder", + path: "/newFolder", + isDir: false, + exists: false, + validpathType: newHostPathTypeList(string(v1.HostPathDirectoryOrCreate)), + invalidpathType: newHostPathTypeList(string(v1.HostPathDirectory), string(v1.HostPathFile), + string(v1.HostPathSocket), string(v1.HostPathCharDev), string(v1.HostPathBlockDev)), + }, + { + name: "Existing File", + path: "/existingFile", + isFile: true, + exists: true, + validpathType: newHostPathTypeList(string(v1.HostPathFileOrCreate), string(v1.HostPathFile)), + invalidpathType: newHostPathTypeList(string(v1.HostPathDirectoryOrCreate), string(v1.HostPathDirectory), + string(v1.HostPathSocket), string(v1.HostPathCharDev), string(v1.HostPathBlockDev)), + }, + { + name: "New File", + path: "/newFile", + isFile: false, + exists: false, + validpathType: newHostPathTypeList(string(v1.HostPathFileOrCreate)), + invalidpathType: newHostPathTypeList(string(v1.HostPathDirectory), + string(v1.HostPathSocket), string(v1.HostPathCharDev), string(v1.HostPathBlockDev)), + }, + { + name: "Existing Socket", + path: "/existing.socket", + isSocket: true, + isFile: true, + exists: true, + validpathType: newHostPathTypeList(string(v1.HostPathSocket), string(v1.HostPathFileOrCreate), string(v1.HostPathFile)), + invalidpathType: newHostPathTypeList(string(v1.HostPathDirectoryOrCreate), string(v1.HostPathDirectory), + string(v1.HostPathCharDev), string(v1.HostPathBlockDev)), + }, + { + name: "Existing Character Device", + path: "/existing.char", + isChar: true, + isFile: true, + exists: true, + validpathType: newHostPathTypeList(string(v1.HostPathCharDev), string(v1.HostPathFileOrCreate), string(v1.HostPathFile)), + invalidpathType: newHostPathTypeList(string(v1.HostPathDirectoryOrCreate), string(v1.HostPathDirectory), + string(v1.HostPathSocket), string(v1.HostPathBlockDev)), + }, + { + name: "Existing Block Device", + path: "/existing.block", + isBlock: true, + isFile: true, + exists: true, + validpathType: newHostPathTypeList(string(v1.HostPathBlockDev), string(v1.HostPathFileOrCreate), string(v1.HostPathFile)), + invalidpathType: newHostPathTypeList(string(v1.HostPathDirectoryOrCreate), string(v1.HostPathDirectory), + string(v1.HostPathSocket), string(v1.HostPathCharDev)), + }, + } + + for i, tc := range testCases { + for _, pathType := range tc.validpathType { + err := checkTypeInternal(&tc, pathType) + if err != nil { + t.Errorf("[%d: %q] [%q] expected nil, got %v", i, tc.name, string(*pathType), err) + } + } + + for _, pathType := range tc.invalidpathType { + checkResult := checkTypeInternal(&tc, pathType) + if checkResult == nil { + t.Errorf("[%d: %q] [%q] expected error, got nil", i, tc.name, string(*pathType)) + } + } + } + +} diff --git a/pkg/volume/host_path/host_path_unix.go b/pkg/volume/host_path/host_path_unix.go new file mode 100644 index 0000000000000..05deaaed5b2ae --- /dev/null +++ b/pkg/volume/host_path/host_path_unix.go @@ -0,0 +1,40 @@ +// +build linux darwin + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package host_path + +import ( + "fmt" + "os" + "syscall" + + "k8s.io/api/core/v1" +) + +func (dftc *defaultFileTypeChecker) getFileType(info os.FileInfo) (v1.HostPathType, error) { + mode := info.Sys().(*syscall.Stat_t).Mode + switch mode & syscall.S_IFMT { + case syscall.S_IFSOCK: + return v1.HostPathSocket, nil + case syscall.S_IFBLK: + return v1.HostPathBlockDev, nil + case syscall.S_IFCHR: + return v1.HostPathCharDev, nil + } + return "", fmt.Errorf("only recognise socket, block device and character device") +} diff --git a/pkg/volume/host_path/host_path_windows.go b/pkg/volume/host_path/host_path_windows.go new file mode 100644 index 0000000000000..ff27e49371f58 --- /dev/null +++ b/pkg/volume/host_path/host_path_windows.go @@ -0,0 +1,38 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package host_path + +import ( + "fmt" + "os" + "syscall" + + "k8s.io/api/core/v1" +) + +func (dftc *defaultFileTypeChecker) getFileType(info os.FileInfo) (v1.HostPathType, error) { + mode := info.Sys().(*syscall.Win32FileAttributeData).FileAttributes + switch mode & syscall.S_IFMT { + case syscall.S_IFSOCK: + return v1.HostPathSocket, nil + case syscall.S_IFBLK: + return v1.HostPathBlockDev, nil + case syscall.S_IFCHR: + return v1.HostPathCharDev, nil + } + return "", fmt.Errorf("only recognise socket, block device and character device") +} diff --git a/pkg/volume/host_path/nsenter.go b/pkg/volume/host_path/nsenter.go new file mode 100644 index 0000000000000..c3c3cfece5770 --- /dev/null +++ b/pkg/volume/host_path/nsenter.go @@ -0,0 +1,150 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package host_path + +import ( + "fmt" + + "k8s.io/utils/exec" +) + +const ( + hostProcMountsNamespace = "/rootfs/proc/1/ns/mnt" + nsenterCmd = "nsenter" + statCmd = "stat" + touchCmd = "touch" + mkdirCmd = "mkdir" +) + +// nsenterFileTypeChecker is part of experimental support for running the kubelet +// in a container. nsenterFileTypeChecker works by executing "nsenter" to run commands in +// the host's mount namespace. +// +// nsenterFileTypeChecker requires: +// +// 1. The host's root filesystem must be available at "/rootfs"; +// 2. The "nsenter" binary must be on the Kubelet process' PATH in the container's +// filesystem; +// 3. The Kubelet process must have CAP_SYS_ADMIN (required by "nsenter"); at +// the present, this effectively means that the kubelet is running in a +// privileged container; +// 4. The host image must have "stat", "touch", "mkdir" binaries in "/bin", "/usr/sbin", or "/usr/bin"; + +type nsenterFileTypeChecker struct { + path string + exists bool +} + +func newNsenterFileTypeChecker(path string) (hostPathTypeChecker, error) { + ftc := &nsenterFileTypeChecker{path: path} + ftc.Exists() + return ftc, nil +} + +func (ftc *nsenterFileTypeChecker) Exists() bool { + args := []string{ + fmt.Sprintf("--mount=%s", hostProcMountsNamespace), + "--", + "ls", + ftc.path, + } + exec := exec.New() + _, err := exec.Command(nsenterCmd, args...).CombinedOutput() + if err == nil { + ftc.exists = true + } + return ftc.exists +} + +func (ftc *nsenterFileTypeChecker) IsFile() bool { + if !ftc.Exists() { + return false + } + return !ftc.IsDir() +} + +func (ftc *nsenterFileTypeChecker) MakeFile() error { + args := []string{ + fmt.Sprintf("--mount=%s", hostProcMountsNamespace), + "--", + touchCmd, + ftc.path, + } + exec := exec.New() + if _, err := exec.Command(nsenterCmd, args...).CombinedOutput(); err != nil { + return err + } + return nil +} + +func (ftc *nsenterFileTypeChecker) IsDir() bool { + return ftc.checkMimetype("directory") +} + +func (ftc *nsenterFileTypeChecker) MakeDir() error { + args := []string{ + fmt.Sprintf("--mount=%s", hostProcMountsNamespace), + "--", + mkdirCmd, + "-p", + ftc.path, + } + exec := exec.New() + if _, err := exec.Command(nsenterCmd, args...).CombinedOutput(); err != nil { + return err + } + return nil +} + +func (ftc *nsenterFileTypeChecker) IsBlock() bool { + return ftc.checkMimetype("block special file") +} + +func (ftc *nsenterFileTypeChecker) IsChar() bool { + return ftc.checkMimetype("character special file") +} + +func (ftc *nsenterFileTypeChecker) IsSocket() bool { + return ftc.checkMimetype("socket") +} + +func (ftc *nsenterFileTypeChecker) GetPath() string { + return ftc.path +} + +func (ftc *nsenterFileTypeChecker) checkMimetype(checkedType string) bool { + if !ftc.Exists() { + return false + } + + args := []string{ + fmt.Sprintf("--mount=%s", hostProcMountsNamespace), + "--", + statCmd, + "-L", + `--printf "%F"`, + ftc.path, + } + exec := exec.New() + outputBytes, err := exec.Command(nsenterCmd, args...).CombinedOutput() + if err != nil { + return false + } + return string(outputBytes) == checkedType +} diff --git a/pkg/volume/host_path/nsenter_unsupported.go b/pkg/volume/host_path/nsenter_unsupported.go new file mode 100644 index 0000000000000..166ef9f0a3911 --- /dev/null +++ b/pkg/volume/host_path/nsenter_unsupported.go @@ -0,0 +1,66 @@ +// +build !linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package host_path + +type nsenterFileTypeChecker struct { + path string + exists bool +} + +func newNsenterFileTypeChecker(path string) (hostPathTypeChecker, error) { + ftc := &nsenterFileTypeChecker{path: path} + ftc.Exists() + return ftc, nil +} + +func (ftc *nsenterFileTypeChecker) Exists() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) IsFile() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) MakeFile() error { + return nil +} + +func (ftc *nsenterFileTypeChecker) IsDir() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) MakeDir() error { + return nil +} + +func (ftc *nsenterFileTypeChecker) IsBlock() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) IsChar() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) IsSocket() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) GetPath() string { + return ftc.path +} diff --git a/pkg/volume/iscsi/BUILD b/pkg/volume/iscsi/BUILD index 823e394c84730..4ed90f8886988 100644 --- a/pkg/volume/iscsi/BUILD +++ b/pkg/volume/iscsi/BUILD @@ -9,6 +9,7 @@ load( go_library( name = "go_default_library", srcs = [ + "attacher.go", "disk_manager.go", "doc.go", "iscsi.go", @@ -22,7 +23,6 @@ go_library( "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/iscsi/attacher.go b/pkg/volume/iscsi/attacher.go new file mode 100644 index 0000000000000..4ff499cc408d8 --- /dev/null +++ b/pkg/volume/iscsi/attacher.go @@ -0,0 +1,213 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iscsi + +import ( + "fmt" + "os" + "strconv" + "time" + + "github.com/golang/glog" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" +) + +type iscsiAttacher struct { + host volume.VolumeHost + manager diskManager +} + +var _ volume.Attacher = &iscsiAttacher{} + +var _ volume.AttachableVolumePlugin = &iscsiPlugin{} + +func (plugin *iscsiPlugin) NewAttacher() (volume.Attacher, error) { + return &iscsiAttacher{ + host: plugin.host, + manager: &ISCSIUtil{}, + }, nil +} + +func (plugin *iscsiPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { + mounter := plugin.host.GetMounter(iscsiPluginName) + return mount.GetMountRefs(mounter, deviceMountPath) +} + +func (attacher *iscsiAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) { + return "", nil +} + +func (attacher *iscsiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.NodeName) (map[*volume.Spec]bool, error) { + volumesAttachedCheck := make(map[*volume.Spec]bool) + for _, spec := range specs { + volumesAttachedCheck[spec] = true + } + + return volumesAttachedCheck, nil +} + +func (attacher *iscsiAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) { + mounter, err := attacher.volumeSpecToMounter(spec, attacher.host, pod) + if err != nil { + glog.Warningf("failed to get iscsi mounter: %v", err) + return "", err + } + return attacher.manager.AttachDisk(*mounter) +} + +func (attacher *iscsiAttacher) GetDeviceMountPath( + spec *volume.Spec) (string, error) { + mounter, err := attacher.volumeSpecToMounter(spec, attacher.host, nil) + if err != nil { + glog.Warningf("failed to get iscsi mounter: %v", err) + return "", err + } + if mounter.InitiatorName != "" { + // new iface name is : + mounter.Iface = mounter.Portals[0] + ":" + mounter.VolName + } + return attacher.manager.MakeGlobalPDName(*mounter.iscsiDisk), nil +} + +func (attacher *iscsiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { + mounter := attacher.host.GetMounter(iscsiPluginName) + notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) + if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(deviceMountPath, 0750); err != nil { + return err + } + notMnt = true + } else { + return err + } + } + volumeSource, readOnly, err := getVolumeSource(spec) + if err != nil { + return err + } + + options := []string{} + if readOnly { + options = append(options, "ro") + } + if notMnt { + diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Exec: attacher.host.GetExec(iscsiPluginName)} + mountOptions := volume.MountOptionFromSpec(spec, options...) + err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions) + if err != nil { + os.Remove(deviceMountPath) + return err + } + } + return nil +} + +type iscsiDetacher struct { + host volume.VolumeHost + mounter mount.Interface + manager diskManager +} + +var _ volume.Detacher = &iscsiDetacher{} + +func (plugin *iscsiPlugin) NewDetacher() (volume.Detacher, error) { + return &iscsiDetacher{ + host: plugin.host, + mounter: plugin.host.GetMounter(iscsiPluginName), + manager: &ISCSIUtil{}, + }, nil +} + +func (detacher *iscsiDetacher) Detach(deviceMountPath string, nodeName types.NodeName) error { + return nil +} + +func (detacher *iscsiDetacher) UnmountDevice(deviceMountPath string) error { + unMounter := detacher.volumeSpecToUnmounter(detacher.mounter) + err := detacher.manager.DetachDisk(*unMounter, deviceMountPath) + if err != nil { + return fmt.Errorf("iscsi: failed to detach disk: %s\nError: %v", deviceMountPath, err) + } + glog.V(4).Infof("iscsi: successfully detached disk: %s", deviceMountPath) + return nil +} + +func (attacher *iscsiAttacher) volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, pod *v1.Pod) (*iscsiDiskMounter, error) { + var secret map[string]string + var bkportal []string + iscsi, readOnly, err := getVolumeSource(spec) + if err != nil { + return nil, err + } + // Obtain secret for AttachDisk + if iscsi.SecretRef != nil && pod != nil { + if secret, err = volumeutil.GetSecretForPod(pod, iscsi.SecretRef.Name, host.GetKubeClient()); err != nil { + glog.Errorf("Couldn't get secret from %v/%v", pod.Namespace, iscsi.SecretRef) + return nil, err + } + } + lun := strconv.Itoa(int(iscsi.Lun)) + portal := portalMounter(iscsi.TargetPortal) + bkportal = append(bkportal, portal) + for _, tp := range iscsi.Portals { + bkportal = append(bkportal, portalMounter(string(tp))) + } + iface := iscsi.ISCSIInterface + exec := attacher.host.GetExec(iscsiPluginName) + var initiatorName string + if iscsi.InitiatorName != nil { + initiatorName = *iscsi.InitiatorName + } + + return &iscsiDiskMounter{ + iscsiDisk: &iscsiDisk{ + plugin: &iscsiPlugin{ + host: host, + }, + VolName: spec.Name(), + Portals: bkportal, + Iqn: iscsi.IQN, + lun: lun, + Iface: iface, + chap_discovery: iscsi.DiscoveryCHAPAuth, + chap_session: iscsi.SessionCHAPAuth, + secret: secret, + InitiatorName: initiatorName, + manager: &ISCSIUtil{}}, + fsType: iscsi.FSType, + readOnly: readOnly, + mounter: &mount.SafeFormatAndMount{Interface: host.GetMounter(iscsiPluginName), Exec: exec}, + exec: exec, + deviceUtil: volumeutil.NewDeviceHandler(volumeutil.NewIOHandler()), + }, nil +} + +func (detacher *iscsiDetacher) volumeSpecToUnmounter(mounter mount.Interface) *iscsiDiskUnmounter { + exec := detacher.host.GetExec(iscsiPluginName) + return &iscsiDiskUnmounter{ + iscsiDisk: &iscsiDisk{ + plugin: &iscsiPlugin{}, + }, + mounter: mounter, + exec: exec, + } +} diff --git a/pkg/volume/iscsi/disk_manager.go b/pkg/volume/iscsi/disk_manager.go index 2c470b9b1ba1a..38c1b79850c8c 100644 --- a/pkg/volume/iscsi/disk_manager.go +++ b/pkg/volume/iscsi/disk_manager.go @@ -28,17 +28,15 @@ import ( type diskManager interface { MakeGlobalPDName(disk iscsiDisk) string // Attaches the disk to the kubelet's host machine. - AttachDisk(b iscsiDiskMounter) error + AttachDisk(b iscsiDiskMounter) (string, error) // Detaches the disk from the kubelet's host machine. DetachDisk(disk iscsiDiskUnmounter, mntPath string) error } // utility to mount a disk based filesystem func diskSetUp(manager diskManager, b iscsiDiskMounter, volPath string, mounter mount.Interface, fsGroup *int64) error { - globalPDPath := manager.MakeGlobalPDName(*b.iscsiDisk) // TODO: handle failed mounts here. notMnt, err := mounter.IsLikelyNotMountPoint(volPath) - if err != nil && !os.IsNotExist(err) { glog.Errorf("cannot validate mountpoint: %s", volPath) return err @@ -46,10 +44,6 @@ func diskSetUp(manager diskManager, b iscsiDiskMounter, volPath string, mounter if !notMnt { return nil } - if err := manager.AttachDisk(b); err != nil { - glog.Errorf("failed to attach disk") - return err - } if err := os.MkdirAll(volPath, 0750); err != nil { glog.Errorf("failed to mkdir:%s", volPath) @@ -60,6 +54,11 @@ func diskSetUp(manager diskManager, b iscsiDiskMounter, volPath string, mounter if b.readOnly { options = append(options, "ro") } + if b.iscsiDisk.InitiatorName != "" { + // new iface name is : + b.iscsiDisk.Iface = b.iscsiDisk.Portals[0] + ":" + b.iscsiDisk.VolName + } + globalPDPath := manager.MakeGlobalPDName(*b.iscsiDisk) mountOptions := volume.JoinMountOptions(b.mountOptions, options) err = mounter.Mount(globalPDPath, volPath, "", mountOptions) if err != nil { @@ -84,8 +83,7 @@ func diskTearDown(manager diskManager, c iscsiDiskUnmounter, volPath string, mou if notMnt { return os.Remove(volPath) } - - refs, err := mount.GetMountRefs(mounter, volPath) + _, err = mount.GetMountRefs(mounter, volPath) if err != nil { glog.Errorf("failed to get reference count %s", volPath) return err @@ -94,16 +92,6 @@ func diskTearDown(manager diskManager, c iscsiDiskUnmounter, volPath string, mou glog.Errorf("failed to unmount %s", volPath) return err } - // If len(refs) is 1, then all bind mounts have been removed, and the - // remaining reference is the global mount. It is safe to detach. - if len(refs) == 1 { - mntPath := refs[0] - if err := manager.DetachDisk(c, mntPath); err != nil { - glog.Errorf("failed to detach disk from %s", mntPath) - return err - } - } - notMnt, mntErr := mounter.IsLikelyNotMountPoint(volPath) if mntErr != nil { glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) diff --git a/pkg/volume/iscsi/iscsi.go b/pkg/volume/iscsi/iscsi.go index 3e47cc02a986b..6c70256692899 100644 --- a/pkg/volume/iscsi/iscsi.go +++ b/pkg/volume/iscsi/iscsi.go @@ -28,17 +28,15 @@ import ( utilstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" ioutil "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" ) // This is the primary entrypoint for volume plugins. func ProbeVolumePlugins() []volume.VolumePlugin { - return []volume.VolumePlugin{&iscsiPlugin{nil, exec.New()}} + return []volume.VolumePlugin{&iscsiPlugin{nil}} } type iscsiPlugin struct { host volume.VolumeHost - exe exec.Interface } var _ volume.VolumePlugin = &iscsiPlugin{} @@ -112,10 +110,10 @@ func (plugin *iscsiPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.V } } - return plugin.newMounterInternal(spec, pod.UID, &ISCSIUtil{}, plugin.host.GetMounter(), secret) + return plugin.newMounterInternal(spec, pod.UID, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret) } -func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, secret map[string]string) (volume.Mounter, error) { +func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec, secret map[string]string) (volume.Mounter, error) { // iscsi volumes used directly in a pod have a ReadOnly flag set by the pod author. // iscsi volumes used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV iscsi, readOnly, err := getVolumeSource(spec) @@ -132,10 +130,15 @@ func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UI } iface := iscsi.ISCSIInterface + var initiatorName string + if iscsi.InitiatorName != nil { + initiatorName = *iscsi.InitiatorName + } + return &iscsiDiskMounter{ iscsiDisk: &iscsiDisk{ podUID: podUID, - volName: spec.Name(), + VolName: spec.Name(), Portals: bkportal, Iqn: iscsi.IQN, lun: lun, @@ -143,11 +146,13 @@ func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UI chap_discovery: iscsi.DiscoveryCHAPAuth, chap_session: iscsi.SessionCHAPAuth, secret: secret, + InitiatorName: initiatorName, manager: manager, plugin: plugin}, fsType: iscsi.FSType, readOnly: readOnly, - mounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}, + mounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}, + exec: exec, deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()), mountOptions: volume.MountOptionFromSpec(spec), }, nil @@ -155,26 +160,22 @@ func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UI func (plugin *iscsiPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newUnmounterInternal(volName, podUID, &ISCSIUtil{}, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) } -func (plugin *iscsiPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface) (volume.Unmounter, error) { +func (plugin *iscsiPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec) (volume.Unmounter, error) { return &iscsiDiskUnmounter{ iscsiDisk: &iscsiDisk{ podUID: podUID, - volName: volName, + VolName: volName, manager: manager, plugin: plugin, }, mounter: mounter, + exec: exec, }, nil } -func (plugin *iscsiPlugin) execCommand(command string, args []string) ([]byte, error) { - cmd := plugin.exe.Command(command, args...) - return cmd.CombinedOutput() -} - func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { iscsiVolume := &v1.Volume{ Name: volumeName, @@ -189,7 +190,7 @@ func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*v } type iscsiDisk struct { - volName string + VolName string podUID types.UID Portals []string Iqn string @@ -198,6 +199,7 @@ type iscsiDisk struct { chap_discovery bool chap_session bool secret map[string]string + InitiatorName string plugin *iscsiPlugin // Utility interface that provides API calls to the provider to attach/detach disks. manager diskManager @@ -207,7 +209,7 @@ type iscsiDisk struct { func (iscsi *iscsiDisk) GetPath() string { name := iscsiPluginName // safe to use PodVolumeDir now: volume teardown occurs before pod is cleaned up - return iscsi.plugin.host.GetPodVolumeDir(iscsi.podUID, utilstrings.EscapeQualifiedNameForDisk(name), iscsi.volName) + return iscsi.plugin.host.GetPodVolumeDir(iscsi.podUID, utilstrings.EscapeQualifiedNameForDisk(name), iscsi.VolName) } type iscsiDiskMounter struct { @@ -215,6 +217,7 @@ type iscsiDiskMounter struct { readOnly bool fsType string mounter *mount.SafeFormatAndMount + exec mount.Exec deviceUtil ioutil.DeviceUtil mountOptions []string } @@ -252,6 +255,7 @@ func (b *iscsiDiskMounter) SetUpAt(dir string, fsGroup *int64) error { type iscsiDiskUnmounter struct { *iscsiDisk mounter mount.Interface + exec mount.Exec } var _ volume.Unmounter = &iscsiDiskUnmounter{} diff --git a/pkg/volume/iscsi/iscsi_test.go b/pkg/volume/iscsi/iscsi_test.go index 348b1994b3fb8..dd034657939ad 100644 --- a/pkg/volume/iscsi/iscsi_test.go +++ b/pkg/volume/iscsi/iscsi_test.go @@ -39,7 +39,7 @@ func TestCanSupport(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/iscsi") if err != nil { @@ -61,7 +61,7 @@ func TestGetAccessModes(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/iscsi") if err != nil { @@ -100,18 +100,17 @@ func (fake *fakeDiskManager) Cleanup() { func (fake *fakeDiskManager) MakeGlobalPDName(disk iscsiDisk) string { return fake.tmpDir } -func (fake *fakeDiskManager) AttachDisk(b iscsiDiskMounter) error { +func (fake *fakeDiskManager) AttachDisk(b iscsiDiskMounter) (string, error) { globalPath := b.manager.MakeGlobalPDName(*b.iscsiDisk) err := os.MkdirAll(globalPath, 0750) if err != nil { - return err + return "", err } // Simulate the global mount so that the fakeMounter returns the // expected number of mounts for the attached disk. b.mounter.Mount(globalPath, globalPath, b.fsType, nil) - fake.attachCalled = true - return nil + return "/dev/sdb", nil } func (fake *fakeDiskManager) DetachDisk(c iscsiDiskUnmounter, mntPath string) error { @@ -120,7 +119,6 @@ func (fake *fakeDiskManager) DetachDisk(c iscsiDiskUnmounter, mntPath string) er if err != nil { return err } - fake.detachCalled = true return nil } @@ -132,7 +130,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/iscsi") if err != nil { @@ -141,7 +139,8 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { fakeManager := NewFakeDiskManager() defer fakeManager.Cleanup() fakeMounter := &mount.FakeMounter{} - mounter, err := plug.(*iscsiPlugin).newMounterInternal(spec, types.UID("poduid"), fakeManager, fakeMounter, nil) + fakeExec := mount.NewFakeExec(nil) + mounter, err := plug.(*iscsiPlugin).newMounterInternal(spec, types.UID("poduid"), fakeManager, fakeMounter, fakeExec, nil) if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) } @@ -172,13 +171,10 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { t.Errorf("SetUp() failed: %v", err) } } - if !fakeManager.attachCalled { - t.Errorf("Attach was not called") - } fakeManager2 := NewFakeDiskManager() defer fakeManager2.Cleanup() - unmounter, err := plug.(*iscsiPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fakeManager2, fakeMounter) + unmounter, err := plug.(*iscsiPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fakeManager2, fakeMounter, fakeExec) if err != nil { t.Errorf("Failed to make a new Unmounter: %v", err) } @@ -194,9 +190,6 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { } else if !os.IsNotExist(err) { t.Errorf("SetUp() failed: %v", err) } - if !fakeManager2.detachCalled { - t.Errorf("Detach was not called") - } } func TestPluginVolume(t *testing.T) { @@ -275,13 +268,16 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { client := fake.NewSimpleClientset(pv, claim) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, client, nil)) plug, _ := plugMgr.FindPluginByName(iscsiPluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes spec := volume.NewSpecFromPersistentVolume(pv, true) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{}) + if mounter == nil { + t.Fatalf("Got a nil Mounter") + } if !mounter.GetAttributes().ReadOnly { t.Errorf("Expected true for mounter.IsReadOnly") diff --git a/pkg/volume/iscsi/iscsi_util.go b/pkg/volume/iscsi/iscsi_util.go old mode 100755 new mode 100644 index 9b5d3cb7a5972..6053d1633a0bc --- a/pkg/volume/iscsi/iscsi_util.go +++ b/pkg/volume/iscsi/iscsi_util.go @@ -29,6 +29,7 @@ import ( "github.com/golang/glog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" ) var ( @@ -45,19 +46,20 @@ var ( ) func updateISCSIDiscoverydb(b iscsiDiskMounter, tp string) error { - if b.chap_discovery { - out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", "discovery.sendtargets.auth.authmethod", "-v", "CHAP"}) - if err != nil { - return fmt.Errorf("iscsi: failed to update discoverydb with CHAP, output: %v", string(out)) - } + if !b.chap_discovery { + return nil + } + out, err := b.exec.Run("iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", "discovery.sendtargets.auth.authmethod", "-v", "CHAP") + if err != nil { + return fmt.Errorf("iscsi: failed to update discoverydb with CHAP, output: %v", string(out)) + } - for _, k := range chap_st { - v := b.secret[k] - if len(v) > 0 { - out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", k, "-v", v}) - if err != nil { - return fmt.Errorf("iscsi: failed to update discoverydb key %q with value %q error: %v", k, v, string(out)) - } + for _, k := range chap_st { + v := b.secret[k] + if len(v) > 0 { + out, err := b.exec.Run("iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "update", "-n", k, "-v", v) + if err != nil { + return fmt.Errorf("iscsi: failed to update discoverydb key %q with value %q error: %v", k, v, string(out)) } } } @@ -65,19 +67,21 @@ func updateISCSIDiscoverydb(b iscsiDiskMounter, tp string) error { } func updateISCSINode(b iscsiDiskMounter, tp string) error { - if b.chap_session { - out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", "node.session.auth.authmethod", "-v", "CHAP"}) - if err != nil { - return fmt.Errorf("iscsi: failed to update node with CHAP, output: %v", string(out)) - } + if !b.chap_session { + return nil + } - for _, k := range chap_sess { - v := b.secret[k] - if len(v) > 0 { - out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", k, "-v", v}) - if err != nil { - return fmt.Errorf("iscsi: failed to update node session key %q with value %q error: %v", k, v, string(out)) - } + out, err := b.exec.Run("iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", "node.session.auth.authmethod", "-v", "CHAP") + if err != nil { + return fmt.Errorf("iscsi: failed to update node with CHAP, output: %v", string(out)) + } + + for _, k := range chap_sess { + v := b.secret[k] + if len(v) > 0 { + out, err := b.exec.Run("iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "-o", "update", "-n", k, "-v", v) + if err != nil { + return fmt.Errorf("iscsi: failed to update node session key %q with value %q error: %v", k, v, string(out)) } } } @@ -95,33 +99,35 @@ func waitForPathToExist(devicePath *string, maxRetries int, deviceTransport stri } func waitForPathToExistInternal(devicePath *string, maxRetries int, deviceTransport string, osStat StatFunc, filepathGlob GlobFunc) bool { - if devicePath != nil { - for i := 0; i < maxRetries; i++ { - var err error - if deviceTransport == "tcp" { - _, err = osStat(*devicePath) + if devicePath == nil { + return false + } + + for i := 0; i < maxRetries; i++ { + var err error + if deviceTransport == "tcp" { + _, err = osStat(*devicePath) + } else { + fpath, _ := filepathGlob(*devicePath) + if fpath == nil { + err = os.ErrNotExist } else { - fpath, _ := filepathGlob(*devicePath) - if fpath == nil { - err = os.ErrNotExist - } else { - // There might be a case that fpath contains multiple device paths if - // multiple PCI devices connect to same iscsi target. We handle this - // case at subsequent logic. Pick up only first path here. - *devicePath = fpath[0] - } - } - if err == nil { - return true + // There might be a case that fpath contains multiple device paths if + // multiple PCI devices connect to same iscsi target. We handle this + // case at subsequent logic. Pick up only first path here. + *devicePath = fpath[0] } - if !os.IsNotExist(err) { - return false - } - if i == maxRetries-1 { - break - } - time.Sleep(time.Second) } + if err == nil { + return true + } + if !os.IsNotExist(err) { + return false + } + if i == maxRetries-1 { + break + } + time.Sleep(time.Second) } return false } @@ -190,86 +196,102 @@ func (util *ISCSIUtil) loadISCSI(conf *iscsiDisk, mnt string) error { return nil } -func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error { +func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) { var devicePath string var devicePaths []string var iscsiTransport string var lastErr error - out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "iface", "-I", b.Iface, "-o", "show"}) + out, err := b.exec.Run("iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "show") if err != nil { glog.Errorf("iscsi: could not read iface %s error: %s", b.Iface, string(out)) - return err + return "", err } iscsiTransport = extractTransportname(string(out)) bkpPortal := b.Portals + + // create new iface and copy parameters from pre-configured iface to the created iface + if b.InitiatorName != "" { + // new iface name is : + newIface := bkpPortal[0] + ":" + b.VolName + err = cloneIface(b, newIface) + if err != nil { + glog.Errorf("iscsi: failed to clone iface: %s error: %v", b.Iface, err) + return "", err + } + // update iface name + b.Iface = newIface + } + for _, tp := range bkpPortal { // Rescan sessions to discover newly mapped LUNs. Do not specify the interface when rescanning // to avoid establishing additional sessions to the same target. - out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.Iqn, "-R"}) + out, err := b.exec.Run("iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-R") if err != nil { glog.Errorf("iscsi: failed to rescan session with error: %s (%v)", string(out), err) } if iscsiTransport == "" { glog.Errorf("iscsi: could not find transport name in iface %s", b.Iface) - return fmt.Errorf("Could not parse iface file for %s", b.Iface) - } else if iscsiTransport == "tcp" { + return "", fmt.Errorf("Could not parse iface file for %s", b.Iface) + } + if iscsiTransport == "tcp" { devicePath = strings.Join([]string{"/dev/disk/by-path/ip", tp, "iscsi", b.Iqn, "lun", b.lun}, "-") } else { devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", tp, "iscsi", b.Iqn, "lun", b.lun}, "-") } - exist := waitForPathToExist(&devicePath, 1, iscsiTransport) - if exist == false { - // build discoverydb and discover iscsi target - b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "new"}) - // update discoverydb with CHAP secret - err = updateISCSIDiscoverydb(b, tp) - if err != nil { - lastErr = fmt.Errorf("iscsi: failed to update discoverydb to portal %s error: %v", tp, err) - continue - } - out, err := b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "--discover"}) - if err != nil { - // delete discoverydb record - b.plugin.execCommand("iscsiadm", []string{"-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "delete"}) - lastErr = fmt.Errorf("iscsi: failed to sendtargets to portal %s output: %s, err %v", tp, string(out), err) - continue - } - err = updateISCSINode(b, tp) - if err != nil { - // failure to update node db is rare. But deleting record will likely impact those who already start using it. - lastErr = fmt.Errorf("iscsi: failed to update iscsi node to portal %s error: %v", tp, err) - continue - } - // login to iscsi target - out, err = b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "--login"}) - if err != nil { - // delete the node record from database - b.plugin.execCommand("iscsiadm", []string{"-m", "node", "-p", tp, "-I", b.Iface, "-T", b.Iqn, "-o", "delete"}) - lastErr = fmt.Errorf("iscsi: failed to attach disk: Error: %s (%v)", string(out), err) - continue - } - exist = waitForPathToExist(&devicePath, 10, iscsiTransport) - if !exist { - glog.Errorf("Could not attach disk: Timeout after 10s") - // update last error - lastErr = fmt.Errorf("Could not attach disk: Timeout after 10s") - continue - } else { - devicePaths = append(devicePaths, devicePath) - } - } else { + + if exist := waitForPathToExist(&devicePath, 1, iscsiTransport); exist { glog.V(4).Infof("iscsi: devicepath (%s) exists", devicePath) devicePaths = append(devicePaths, devicePath) + continue + } + // build discoverydb and discover iscsi target + b.exec.Run("iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "new") + // update discoverydb with CHAP secret + err = updateISCSIDiscoverydb(b, tp) + if err != nil { + lastErr = fmt.Errorf("iscsi: failed to update discoverydb to portal %s error: %v", tp, err) + continue + } + out, err = b.exec.Run("iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "--discover") + if err != nil { + // delete discoverydb record + b.exec.Run("iscsiadm", "-m", "discoverydb", "-t", "sendtargets", "-p", tp, "-I", b.Iface, "-o", "delete") + lastErr = fmt.Errorf("iscsi: failed to sendtargets to portal %s output: %s, err %v", tp, string(out), err) + continue + } + err = updateISCSINode(b, tp) + if err != nil { + // failure to update node db is rare. But deleting record will likely impact those who already start using it. + lastErr = fmt.Errorf("iscsi: failed to update iscsi node to portal %s error: %v", tp, err) + continue + } + // login to iscsi target + out, err = b.exec.Run("iscsiadm", "-m", "node", "-p", tp, "-T", b.Iqn, "-I", b.Iface, "--login") + if err != nil { + // delete the node record from database + b.exec.Run("iscsiadm", "-m", "node", "-p", tp, "-I", b.Iface, "-T", b.Iqn, "-o", "delete") + lastErr = fmt.Errorf("iscsi: failed to attach disk: Error: %s (%v)", string(out), err) + continue + } + if exist := waitForPathToExist(&devicePath, 10, iscsiTransport); !exist { + glog.Errorf("Could not attach disk: Timeout after 10s") + // update last error + lastErr = fmt.Errorf("Could not attach disk: Timeout after 10s") + continue + } else { + devicePaths = append(devicePaths, devicePath) } } if len(devicePaths) == 0 { + // delete cloned iface + b.exec.Run("iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "delete") glog.Errorf("iscsi: failed to get any path for iscsi disk, last err seen:\n%v", lastErr) - return fmt.Errorf("failed to get any path for iscsi disk, last err seen:\n%v", lastErr) + return "", fmt.Errorf("failed to get any path for iscsi disk, last err seen:\n%v", lastErr) } //Make sure we use a valid devicepath to find mpio device. @@ -278,14 +300,17 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error { // mount it globalPDPath := b.manager.MakeGlobalPDName(*b.iscsiDisk) notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath) + if err != nil { + return "", fmt.Errorf("Heuristic determination of mount point failed:%v", err) + } if !notMnt { glog.Infof("iscsi: %s already mounted", globalPDPath) - return nil + return "", nil } if err := os.MkdirAll(globalPDPath, 0750); err != nil { glog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath) - return err + return "", err } // Persist iscsi disk config to json file for DetachDisk path @@ -308,7 +333,7 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) error { glog.Errorf("iscsi: failed to mount iscsi volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err) } - return err + return devicePath, err } func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error { @@ -317,73 +342,97 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error { glog.Errorf("iscsi detach disk: failed to get device from mnt: %s\nError: %v", mntPath, err) return err } + if pathExists, pathErr := volumeutil.PathExists(mntPath); pathErr != nil { + return fmt.Errorf("Error checking if path exists: %v", pathErr) + } else if !pathExists { + glog.Warningf("Warning: Unmount skipped because path does not exist: %v", mntPath) + return nil + } if err = c.mounter.Unmount(mntPath); err != nil { glog.Errorf("iscsi detach disk: failed to unmount: %s\nError: %v", mntPath, err) return err } cnt-- + if cnt != 0 { + return nil + } // if device is no longer used, see if need to logout the target - if cnt == 0 { - device, prefix, err := extractDeviceAndPrefix(mntPath) + device, prefix, err := extractDeviceAndPrefix(mntPath) + if err != nil { + return err + } + refCount, err := getDevicePrefixRefCount(c.mounter, prefix) + if err != nil || refCount != 0 { + return nil + } + + var bkpPortal []string + var volName, iqn, iface, initiatorName string + found := true + + // load iscsi disk config from json file + if err := util.loadISCSI(c.iscsiDisk, mntPath); err == nil { + bkpPortal, iqn, iface, volName = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Iface, c.iscsiDisk.VolName + initiatorName = c.iscsiDisk.InitiatorName + } else { + // If the iscsi disk config is not found, fall back to the original behavior. + // This portal/iqn/iface is no longer referenced, log out. + // Extract the portal and iqn from device path. + bkpPortal = make([]string, 1) + bkpPortal[0], iqn, err = extractPortalAndIqn(device) if err != nil { return err } - refCount, err := getDevicePrefixRefCount(c.mounter, prefix) - if err == nil && refCount == 0 { - var bkpPortal []string - var iqn, iface string - found := true - - // load iscsi disk config from json file - if err := util.loadISCSI(c.iscsiDisk, mntPath); err == nil { - bkpPortal, iqn, iface = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Iface - } else { - // If the iscsi disk config is not found, fall back to the original behavior. - // This portal/iqn/iface is no longer referenced, log out. - // Extract the portal and iqn from device path. - bkpPortal = make([]string, 1) - bkpPortal[0], iqn, err = extractPortalAndIqn(device) - if err != nil { - return err - } - // Extract the iface from the mountPath and use it to log out. If the iface - // is not found, maintain the previous behavior to facilitate kubelet upgrade. - // Logout may fail as no session may exist for the portal/IQN on the specified interface. - iface, found = extractIface(mntPath) - } - for _, portal := range removeDuplicate(bkpPortal) { - logout := []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"} - delete := []string{"-m", "node", "-p", portal, "-T", iqn, "-o", "delete"} - if found { - logout = append(logout, []string{"-I", iface}...) - delete = append(delete, []string{"-I", iface}...) - } - glog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface) - out, err := c.plugin.execCommand("iscsiadm", logout) - if err != nil { - glog.Errorf("iscsi: failed to detach disk Error: %s", string(out)) - } - // Delete the node record - glog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn) - out, err = c.plugin.execCommand("iscsiadm", delete) - if err != nil { - glog.Errorf("iscsi: failed to delete node record Error: %s", string(out)) - } - } + // Extract the iface from the mountPath and use it to log out. If the iface + // is not found, maintain the previous behavior to facilitate kubelet upgrade. + // Logout may fail as no session may exist for the portal/IQN on the specified interface. + iface, found = extractIface(mntPath) + } + portals := removeDuplicate(bkpPortal) + if len(portals) == 0 { + return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations.") + } + + for _, portal := range portals { + logoutArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"} + deleteArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "-o", "delete"} + if found { + logoutArgs = append(logoutArgs, []string{"-I", iface}...) + deleteArgs = append(deleteArgs, []string{"-I", iface}...) + } + glog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface) + out, err := c.exec.Run("iscsiadm", logoutArgs...) + if err != nil { + glog.Errorf("iscsi: failed to detach disk Error: %s", string(out)) + } + // Delete the node record + glog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn) + out, err = c.exec.Run("iscsiadm", deleteArgs...) + if err != nil { + glog.Errorf("iscsi: failed to delete node record Error: %s", string(out)) } } + // Delete the iface after all sessions have logged out + // If the iface is not created via iscsi plugin, skip to delete + if initiatorName != "" && found && iface == (portals[0]+":"+volName) { + deleteArgs := []string{"-m", "iface", "-I", iface, "-o", "delete"} + out, err := c.exec.Run("iscsiadm", deleteArgs...) + if err != nil { + glog.Errorf("iscsi: failed to delete iface Error: %s", string(out)) + } + } + return nil } func extractTransportname(ifaceOutput string) (iscsiTransport string) { re := regexp.MustCompile(`iface.transport_name = (.*)\n`) - rex_output := re.FindStringSubmatch(ifaceOutput) - if rex_output != nil { - iscsiTransport = rex_output[1] - } else { + rexOutput := re.FindStringSubmatch(ifaceOutput) + if rexOutput == nil { return "" } + iscsiTransport = rexOutput[1] // While iface.transport_name is a required parameter, handle it being unspecified anyways if iscsiTransport == "" { @@ -410,9 +459,9 @@ func extractDeviceAndPrefix(mntPath string) (string, string, error) { func extractIface(mntPath string) (string, bool) { re := regexp.MustCompile(`.+/iface-([^/]+)/.+`) - re_output := re.FindStringSubmatch(mntPath) - if re_output != nil { - return re_output[1], true + reOutput := re.FindStringSubmatch(mntPath) + if reOutput != nil { + return reOutput[1], true } return "", false @@ -448,3 +497,57 @@ func removeDuplicate(s []string) []string { s = s[:len(m)] return s } + +func parseIscsiadmShow(output string) (map[string]string, error) { + params := make(map[string]string) + slice := strings.Split(output, "\n") + for _, line := range slice { + if !strings.HasPrefix(line, "iface.") || strings.Contains(line, "") { + continue + } + iface := strings.Fields(line) + if len(iface) != 3 || iface[1] != "=" { + return nil, fmt.Errorf("Error: invalid iface setting: %v", iface) + } + // iscsi_ifacename is immutable once the iface is created + if iface[0] == "iface.iscsi_ifacename" { + continue + } + params[iface[0]] = iface[2] + } + return params, nil +} + +func cloneIface(b iscsiDiskMounter, newIface string) error { + var lastErr error + // get pre-configured iface records + out, err := b.exec.Run("iscsiadm", "-m", "iface", "-I", b.Iface, "-o", "show") + if err != nil { + lastErr = fmt.Errorf("iscsi: failed to show iface records: %s (%v)", string(out), err) + return lastErr + } + // parse obtained records + params, err := parseIscsiadmShow(string(out)) + if err != nil { + lastErr = fmt.Errorf("iscsi: failed to parse iface records: %s (%v)", string(out), err) + return lastErr + } + // update initiatorname + params["iface.initiatorname"] = b.InitiatorName + // create new iface + out, err = b.exec.Run("iscsiadm", "-m", "iface", "-I", newIface, "-o", "new") + if err != nil { + lastErr = fmt.Errorf("iscsi: failed to create new iface: %s (%v)", string(out), err) + return lastErr + } + // update new iface records + for key, val := range params { + _, err = b.exec.Run("iscsiadm", "-m", "iface", "-I", newIface, "-o", "update", "-n", key, "-v", val) + if err != nil { + b.exec.Run("iscsiadm", "-m", "iface", "-I", newIface, "-o", "delete") + lastErr = fmt.Errorf("iscsi: failed to update iface records: %s (%v). iface(%s) will be used", string(out), err, b.Iface) + break + } + } + return lastErr +} diff --git a/pkg/volume/iscsi/iscsi_util_test.go b/pkg/volume/iscsi/iscsi_util_test.go old mode 100755 new mode 100644 index 16f79a0664b1a..30eb7f9063f20 --- a/pkg/volume/iscsi/iscsi_util_test.go +++ b/pkg/volume/iscsi/iscsi_util_test.go @@ -17,12 +17,15 @@ limitations under the License. package iscsi import ( + "errors" + "fmt" "os" "path/filepath" "reflect" "testing" "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume" ) func TestGetDevicePrefixRefCount(t *testing.T) { @@ -183,3 +186,177 @@ func TestWaitForPathToExist(t *testing.T) { t.Errorf("waitForPathToExist: wrong code path called for %s", devicePath[1]) } } + +func TestParseIscsiadmShow(t *testing.T) { + fakeIscsiadmOutput1 := "# BEGIN RECORD 2.0-873\n" + + "iface.iscsi_ifacename = default\n" + + "iface.transport_name = tcp\n" + + "iface.initiatorname = \n" + + "iface.mtu = 0\n" + + "# END RECORD" + + fakeIscsiadmOutput2 := "# BEGIN RECORD 2.0-873\n" + + "iface.iscsi_ifacename = default\n" + + "iface.transport_name = cxgb4i\n" + + "iface.initiatorname = \n" + + "iface.mtu = 0\n" + + "# END RECORD" + + fakeIscsiadmOutput3 := "# BEGIN RECORD 2.0-873\n" + + "iface.iscsi_ifacename = custom\n" + + "iface.transport_name = \n" + + "iface.initiatorname = \n" + + "iface.mtu = 0\n" + + "# END RECORD" + + fakeIscsiadmOutput4 := "iface.iscsi_ifacename=error" + fakeIscsiadmOutput5 := "iface.iscsi_ifacename + error" + + expectedIscsiadmOutput1 := map[string]string{ + "iface.transport_name": "tcp", + "iface.mtu": "0"} + + expectedIscsiadmOutput2 := map[string]string{ + "iface.transport_name": "cxgb4i", + "iface.mtu": "0"} + + expectedIscsiadmOutput3 := map[string]string{ + "iface.mtu": "0"} + + params, _ := parseIscsiadmShow(fakeIscsiadmOutput1) + if !reflect.DeepEqual(params, expectedIscsiadmOutput1) { + t.Errorf("parseIscsiadmShow: Fail to parse iface record: %s", params) + } + params, _ = parseIscsiadmShow(fakeIscsiadmOutput2) + if !reflect.DeepEqual(params, expectedIscsiadmOutput2) { + t.Errorf("parseIscsiadmShow: Fail to parse iface record: %s", params) + } + params, _ = parseIscsiadmShow(fakeIscsiadmOutput3) + if !reflect.DeepEqual(params, expectedIscsiadmOutput3) { + t.Errorf("parseIscsiadmShow: Fail to parse iface record: %s", params) + } + _, err := parseIscsiadmShow(fakeIscsiadmOutput4) + if err == nil { + t.Errorf("parseIscsiadmShow: Fail to handle invalid record: iface %s", fakeIscsiadmOutput4) + } + _, err = parseIscsiadmShow(fakeIscsiadmOutput5) + if err == nil { + t.Errorf("parseIscsiadmShow: Fail to handle invalid record: iface %s", fakeIscsiadmOutput5) + } +} + +func TestClonedIface(t *testing.T) { + cmdCount := 0 + fakeExec := mount.NewFakeExec(func(cmd string, args ...string) ([]byte, error) { + cmdCount++ + if cmd != "iscsiadm" { + t.Errorf("iscsiadm command expected, got %q", cmd) + } + switch cmdCount { + case 1: + // iscsiadm -m iface -I -o show + return []byte("iface.ipaddress = \niface.transport_name = tcp\niface.initiatorname = \n"), nil + + case 2: + // iscsiadm -m iface -I -o new + return []byte("New interface 192.168.1.10:pv0001 added"), nil + case 3: + // iscsiadm -m iface -I -o update -n -v + return []byte(""), nil + case 4: + return []byte(""), nil + } + return nil, fmt.Errorf("Unexpected exec call nr %d: %s", cmdCount, cmd) + }) + plugins := []volume.VolumePlugin{ + &iscsiPlugin{ + host: nil, + }, + } + plugin := plugins[0] + fakeMounter := iscsiDiskMounter{ + iscsiDisk: &iscsiDisk{ + plugin: plugin.(*iscsiPlugin)}, + exec: fakeExec, + } + newIface := "192.168.1.10:pv0001" + cloneIface(fakeMounter, newIface) + if cmdCount != 4 { + t.Errorf("expected 4 CombinedOutput() calls, got %d", cmdCount) + } + +} + +func TestClonedIfaceShowError(t *testing.T) { + cmdCount := 0 + fakeExec := mount.NewFakeExec(func(cmd string, args ...string) ([]byte, error) { + cmdCount++ + if cmd != "iscsiadm" { + t.Errorf("iscsiadm command expected, got %q", cmd) + } + // iscsiadm -m iface -I -o show, return test error + return []byte(""), errors.New("test error") + }) + plugins := []volume.VolumePlugin{ + &iscsiPlugin{ + host: nil, + }, + } + plugin := plugins[0] + fakeMounter := iscsiDiskMounter{ + iscsiDisk: &iscsiDisk{ + plugin: plugin.(*iscsiPlugin)}, + exec: fakeExec, + } + newIface := "192.168.1.10:pv0001" + cloneIface(fakeMounter, newIface) + if cmdCount != 1 { + t.Errorf("expected 1 CombinedOutput() calls, got %d", cmdCount) + } + +} + +func TestClonedIfaceUpdateError(t *testing.T) { + cmdCount := 0 + fakeExec := mount.NewFakeExec(func(cmd string, args ...string) ([]byte, error) { + cmdCount++ + if cmd != "iscsiadm" { + t.Errorf("iscsiadm command expected, got %q", cmd) + } + switch cmdCount { + case 1: + // iscsiadm -m iface -I -o show + return []byte("iface.ipaddress = \niface.transport_name = tcp\niface.initiatorname = \n"), nil + + case 2: + // iscsiadm -m iface -I -o new + return []byte("New interface 192.168.1.10:pv0001 added"), nil + case 3: + // iscsiadm -m iface -I -o update -n -v + return []byte(""), nil + case 4: + return []byte(""), errors.New("test error") + case 5: + // iscsiadm -m iface -I -o delete + return []byte(""), nil + } + return nil, fmt.Errorf("Unexpected exec call nr %d: %s", cmdCount, cmd) + }) + plugins := []volume.VolumePlugin{ + &iscsiPlugin{ + host: nil, + }, + } + plugin := plugins[0] + fakeMounter := iscsiDiskMounter{ + iscsiDisk: &iscsiDisk{ + plugin: plugin.(*iscsiPlugin)}, + exec: fakeExec, + } + newIface := "192.168.1.10:pv0001" + cloneIface(fakeMounter, newIface) + if cmdCount != 5 { + t.Errorf("expected 5 CombinedOutput() calls, got %d", cmdCount) + } + +} diff --git a/pkg/volume/local/BUILD b/pkg/volume/local/BUILD index beab00e289c39..931fba880c018 100644 --- a/pkg/volume/local/BUILD +++ b/pkg/volume/local/BUILD @@ -13,6 +13,8 @@ go_library( "local.go", ], deps = [ + "//pkg/kubelet/events:go_default_library", + "//pkg/util/keymutex:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/util/strings:go_default_library", "//pkg/volume:go_default_library", @@ -22,6 +24,8 @@ go_library( "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", ], ) diff --git a/pkg/volume/local/local.go b/pkg/volume/local/local.go index 33c1acbb6839b..350d7ceb2925b 100644 --- a/pkg/volume/local/local.go +++ b/pkg/volume/local/local.go @@ -18,13 +18,16 @@ package local import ( "fmt" - "os" - "github.com/golang/glog" + "os" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "k8s.io/kubernetes/pkg/kubelet/events" + "k8s.io/kubernetes/pkg/util/keymutex" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" @@ -38,7 +41,9 @@ func ProbeVolumePlugins() []volume.VolumePlugin { } type localVolumePlugin struct { - host volume.VolumeHost + host volume.VolumeHost + volumeLocks keymutex.KeyMutex + recorder record.EventRecorder } var _ volume.VolumePlugin = &localVolumePlugin{} @@ -50,6 +55,11 @@ const ( func (plugin *localVolumePlugin) Init(host volume.VolumeHost) error { plugin.host = host + plugin.volumeLocks = keymutex.NewKeyMutex() + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(glog.Infof) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "localvolume"}) + plugin.recorder = recorder return nil } @@ -102,9 +112,10 @@ func (plugin *localVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ vo return &localVolumeMounter{ localVolume: &localVolume{ + pod: pod, podUID: pod.UID, volName: spec.Name(), - mounter: plugin.host.GetMounter(), + mounter: plugin.host.GetMounter(plugin.GetPluginName()), plugin: plugin, globalPath: volumeSource.Path, MetricsProvider: volume.NewMetricsStatFS(volumeSource.Path), @@ -119,7 +130,7 @@ func (plugin *localVolumePlugin) NewUnmounter(volName string, podUID types.UID) localVolume: &localVolume{ podUID: podUID, volName: volName, - mounter: plugin.host.GetMounter(), + mounter: plugin.host.GetMounter(plugin.GetPluginName()), plugin: plugin, }, }, nil @@ -146,6 +157,7 @@ func (plugin *localVolumePlugin) ConstructVolumeSpec(volumeName, mountPath strin // The directory at the globalPath will be bind-mounted to the pod's directory type localVolume struct { volName string + pod *v1.Pod podUID types.UID // Global path to the volume globalPath string @@ -188,6 +200,9 @@ func (m *localVolumeMounter) SetUp(fsGroup *int64) error { // SetUpAt bind mounts the directory to the volume path and sets up volume ownership func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { + m.plugin.volumeLocks.LockKey(m.globalPath) + defer m.plugin.volumeLocks.UnlockKey(m.globalPath) + if m.globalPath == "" { err := fmt.Errorf("LocalVolume volume %q path is empty", m.volName) return err @@ -204,9 +219,30 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { glog.Errorf("cannot validate mount point: %s %v", dir, err) return err } + if !notMnt { return nil } + refs, err := mount.GetMountRefsByDev(m.mounter, m.globalPath) + if fsGroup != nil { + if err != nil { + glog.Errorf("cannot collect mounting information: %s %v", m.globalPath, err) + return err + } + + if len(refs) > 0 { + fsGroupNew := int64(*fsGroup) + fsGroupSame, fsGroupOld, err := volume.IsSameFSGroup(m.globalPath, fsGroupNew) + if err != nil { + err = fmt.Errorf("failed to check fsGroup for %s (%v)", m.globalPath, err) + return err + } + if !fsGroupSame { + m.plugin.recorder.Eventf(m.pod, v1.EventTypeWarning, events.WarnAlreadyMountedVolume, "The requested fsGroup is %d, but the volume %s has GID %d. The volume may not be shareable.", fsGroupNew, m.volName, fsGroupOld) + } + } + + } if err := os.MkdirAll(dir, 0750); err != nil { glog.Errorf("mkdir failed on disk %s (%v)", dir, err) @@ -247,10 +283,11 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { os.Remove(dir) return err } - if !m.readOnly { - // TODO: how to prevent multiple mounts with conflicting fsGroup? - return volume.SetVolumeOwnership(m, fsGroup) + // Volume owner will be written only once on the first volume mount + if len(refs) == 0 { + return volume.SetVolumeOwnership(m, fsGroup) + } } return nil } diff --git a/pkg/volume/local/local_test.go b/pkg/volume/local/local_test.go index bf8b3a3aad522..87fb829c493b3 100644 --- a/pkg/volume/local/local_test.go +++ b/pkg/volume/local/local_test.go @@ -17,8 +17,10 @@ limitations under the License. package local import ( + "fmt" "os" "path" + "syscall" "testing" "k8s.io/api/core/v1" @@ -42,7 +44,7 @@ func getPlugin(t *testing.T) (string, volume.VolumePlugin) { } plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName(localVolumePluginName) if err != nil { @@ -62,7 +64,7 @@ func getPersistentPlugin(t *testing.T) (string, volume.PersistentVolumePlugin) { } plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName(localVolumePluginName) if err != nil { @@ -104,7 +106,7 @@ func TestCanSupport(t *testing.T) { tmpDir, plug := getPlugin(t) defer os.RemoveAll(tmpDir) - if !plug.CanSupport(getTestVolume(false, "/test-vol")) { + if !plug.CanSupport(getTestVolume(false, tmpDir)) { t.Errorf("Expected true") } } @@ -130,7 +132,7 @@ func TestGetVolumeName(t *testing.T) { tmpDir, plug := getPersistentPlugin(t) defer os.RemoveAll(tmpDir) - volName, err := plug.GetVolumeName(getTestVolume(false, "/test-vol")) + volName, err := plug.GetVolumeName(getTestVolume(false, tmpDir)) if err != nil { t.Errorf("Failed to get volume name: %v", err) } @@ -161,7 +163,7 @@ func TestMountUnmount(t *testing.T) { defer os.RemoveAll(tmpDir) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} - mounter, err := plug.NewMounter(getTestVolume(false, "/test-vol"), pod, volume.VolumeOptions{}) + mounter, err := plug.NewMounter(getTestVolume(false, tmpDir), pod, volume.VolumeOptions{}) if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) } @@ -204,6 +206,66 @@ func TestMountUnmount(t *testing.T) { } } +func testFSGroupMount(plug volume.VolumePlugin, pod *v1.Pod, tmpDir string, fsGroup int64) error { + mounter, err := plug.NewMounter(getTestVolume(false, tmpDir), pod, volume.VolumeOptions{}) + if err != nil { + return err + } + if mounter == nil { + return fmt.Errorf("Got a nil Mounter") + } + + volPath := path.Join(tmpDir, testMountPath) + path := mounter.GetPath() + if path != volPath { + return fmt.Errorf("Got unexpected path: %s", path) + } + + if err := mounter.SetUp(&fsGroup); err != nil { + return err + } + return nil +} + +func TestFSGroupMount(t *testing.T) { + tmpDir, plug := getPlugin(t) + defer os.RemoveAll(tmpDir) + info, err := os.Stat(tmpDir) + if err != nil { + t.Errorf("Error getting stats for %s (%v)", tmpDir, err) + } + s := info.Sys().(*syscall.Stat_t) + if s == nil { + t.Errorf("Error getting stats for %s (%v)", tmpDir, err) + } + fsGroup1 := int64(s.Gid) + fsGroup2 := fsGroup1 + 1 + pod1 := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} + pod1.Spec.SecurityContext = &v1.PodSecurityContext{ + FSGroup: &fsGroup1, + } + pod2 := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} + pod2.Spec.SecurityContext = &v1.PodSecurityContext{ + FSGroup: &fsGroup2, + } + err = testFSGroupMount(plug, pod1, tmpDir, fsGroup1) + if err != nil { + t.Errorf("Failed to make a new Mounter: %v", err) + } + err = testFSGroupMount(plug, pod2, tmpDir, fsGroup2) + if err != nil { + t.Errorf("Failed to make a new Mounter: %v", err) + } + //Checking if GID of tmpDir has not been changed by mounting it by second pod + s = info.Sys().(*syscall.Stat_t) + if s == nil { + t.Errorf("Error getting stats for %s (%v)", tmpDir, err) + } + if fsGroup1 != int64(s.Gid) { + t.Errorf("Old Gid %d for volume %s got overwritten by new Gid %d", fsGroup1, tmpDir, int64(s.Gid)) + } +} + func TestConstructVolumeSpec(t *testing.T) { tmpDir, plug := getPlugin(t) defer os.RemoveAll(tmpDir) @@ -243,7 +305,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { // Read only == true pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} - mounter, err := plug.NewMounter(getTestVolume(true, "/test-vol"), pod, volume.VolumeOptions{}) + mounter, err := plug.NewMounter(getTestVolume(true, tmpDir), pod, volume.VolumeOptions{}) if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) } @@ -255,7 +317,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { } // Read only == false - mounter, err = plug.NewMounter(getTestVolume(false, "/test-vol"), pod, volume.VolumeOptions{}) + mounter, err = plug.NewMounter(getTestVolume(false, tmpDir), pod, volume.VolumeOptions{}) if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) } @@ -275,8 +337,8 @@ func TestUnsupportedPlugins(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) - spec := getTestVolume(false, "/test-vol") + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + spec := getTestVolume(false, tmpDir) recyclePlug, err := plugMgr.FindRecyclablePluginBySpec(spec) if err == nil && recyclePlug != nil { diff --git a/pkg/volume/nfs/BUILD b/pkg/volume/nfs/BUILD index 40a22756059a2..14937d80d96e9 100644 --- a/pkg/volume/nfs/BUILD +++ b/pkg/volume/nfs/BUILD @@ -21,7 +21,6 @@ go_library( "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/nfs/nfs.go b/pkg/volume/nfs/nfs.go index 52169caf346e4..1435db92815c2 100644 --- a/pkg/volume/nfs/nfs.go +++ b/pkg/volume/nfs/nfs.go @@ -29,7 +29,6 @@ import ( "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" ) // This is the primary entrypoint for volume plugins. @@ -105,7 +104,7 @@ func (plugin *nfsPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { } func (plugin *nfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *nfsPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, mounter mount.Interface) (volume.Mounter, error) { @@ -129,7 +128,7 @@ func (plugin *nfsPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, moun } func (plugin *nfsPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *nfsPlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) { @@ -192,18 +191,18 @@ func (nfsVolume *nfs) GetPath() string { // to mount the volume are available on the underlying node. // If not, it returns an error func (nfsMounter *nfsMounter) CanMount() error { - exe := exec.New() + exec := nfsMounter.plugin.host.GetExec(nfsMounter.plugin.GetPluginName()) switch runtime.GOOS { case "linux": - if _, err := exe.Command("/bin/ls", "/sbin/mount.nfs").CombinedOutput(); err != nil { + if _, err := exec.Run("/bin/ls", "/sbin/mount.nfs"); err != nil { return fmt.Errorf("Required binary /sbin/mount.nfs is missing") } - if _, err := exe.Command("/bin/ls", "/sbin/mount.nfs4").CombinedOutput(); err != nil { + if _, err := exec.Run("/bin/ls", "/sbin/mount.nfs4"); err != nil { return fmt.Errorf("Required binary /sbin/mount.nfs4 is missing") } return nil case "darwin": - if _, err := exe.Command("/bin/ls", "/sbin/mount_nfs").CombinedOutput(); err != nil { + if _, err := exec.Run("/bin/ls", "/sbin/mount_nfs"); err != nil { return fmt.Errorf("Required binary /sbin/mount_nfs is missing") } } diff --git a/pkg/volume/nfs/nfs_test.go b/pkg/volume/nfs/nfs_test.go index e96ef55caefb4..2cb83235b59c8 100644 --- a/pkg/volume/nfs/nfs_test.go +++ b/pkg/volume/nfs/nfs_test.go @@ -39,7 +39,7 @@ func TestCanSupport(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/nfs") if err != nil { t.Errorf("Can't find the plugin by name") @@ -67,7 +67,7 @@ func TestGetAccessModes(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/nfs") if err != nil { @@ -86,7 +86,7 @@ func TestRecycler(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins([]volume.VolumePlugin{&nfsPlugin{nil, volume.VolumeConfig{}}}, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins([]volume.VolumePlugin{&nfsPlugin{nil, volume.VolumeConfig{}}}, nil, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) spec := &volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{NFS: &v1.NFSVolumeSource{Path: "/foo"}}}}} _, plugin_err := plugMgr.FindRecyclablePluginBySpec(spec) @@ -112,7 +112,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/nfs") if err != nil { t.Errorf("Can't find the plugin by name") @@ -240,13 +240,16 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { client := fake.NewSimpleClientset(pv, claim) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), volumetest.NewFakeVolumeHost(tmpDir, client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(volume.VolumeConfig{}), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, client, nil)) plug, _ := plugMgr.FindPluginByName(nfsPluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes spec := volume.NewSpecFromPersistentVolume(pv, true) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{}) + if mounter == nil { + t.Fatalf("Got a nil Mounter") + } if !mounter.GetAttributes().ReadOnly { t.Errorf("Expected true for mounter.IsReadOnly") diff --git a/pkg/volume/photon_pd/BUILD b/pkg/volume/photon_pd/BUILD index d35face84e1b6..a57d2b6cefda9 100644 --- a/pkg/volume/photon_pd/BUILD +++ b/pkg/volume/photon_pd/BUILD @@ -26,7 +26,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/photon_pd/attacher.go b/pkg/volume/photon_pd/attacher.go index b68fcc26534b2..4af726716e69f 100644 --- a/pkg/volume/photon_pd/attacher.go +++ b/pkg/volume/photon_pd/attacher.go @@ -25,12 +25,13 @@ import ( "time" "github.com/golang/glog" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider/providers/photon" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" + "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) type photonPersistentDiskAttacher struct { @@ -121,7 +122,7 @@ func (attacher *photonPersistentDiskAttacher) VolumesAreAttached(specs []*volume return volumesAttachedCheck, nil } -func (attacher *photonPersistentDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { +func (attacher *photonPersistentDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { volumeSource, _, err := getVolumeSource(spec) if err != nil { glog.Errorf("Photon Controller attacher: WaitForAttach failed to get volume source") @@ -180,13 +181,13 @@ func (attacher *photonPersistentDiskAttacher) GetDeviceMountPath(spec *volume.Sp // GetMountDeviceRefs finds all other references to the device referenced // by deviceMountPath; returns a list of paths. func (plugin *photonPersistentDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mount.GetMountRefs(mounter, deviceMountPath) } // MountDevice mounts device to global mount point. func (attacher *photonPersistentDiskAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { - mounter := attacher.host.GetMounter() + mounter := attacher.host.GetMounter(photonPersistentDiskPluginName) notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { if os.IsNotExist(err) { @@ -209,7 +210,7 @@ func (attacher *photonPersistentDiskAttacher) MountDevice(spec *volume.Spec, dev options := []string{} if notMnt { - diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()} + diskMounter := volumehelper.NewSafeFormatAndMountFromHost(photonPersistentDiskPluginName, attacher.host) mountOptions := volume.MountOptionFromSpec(spec) err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions) if err != nil { @@ -236,7 +237,7 @@ func (plugin *photonPersistentDiskPlugin) NewDetacher() (volume.Detacher, error) } return &photonPersistentDiskDetacher{ - mounter: plugin.host.GetMounter(), + mounter: plugin.host.GetMounter(plugin.GetPluginName()), photonDisks: photonCloud, }, nil } diff --git a/pkg/volume/photon_pd/photon_pd.go b/pkg/volume/photon_pd/photon_pd.go index 446cd9aa06707..daa0470893e70 100644 --- a/pkg/volume/photon_pd/photon_pd.go +++ b/pkg/volume/photon_pd/photon_pd.go @@ -31,7 +31,6 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) // This is the primary entrypoint for volume plugins. @@ -89,11 +88,11 @@ func (plugin *photonPersistentDiskPlugin) SupportsBulkVolumeVerification() bool } func (plugin *photonPersistentDiskPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - return plugin.newMounterInternal(spec, pod.UID, &PhotonDiskUtil{}, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod.UID, &PhotonDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *photonPersistentDiskPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, &PhotonDiskUtil{}, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, &PhotonDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *photonPersistentDiskPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Mounter, error) { @@ -116,7 +115,7 @@ func (plugin *photonPersistentDiskPlugin) newMounterInternal(spec *volume.Spec, plugin: plugin, }, fsType: fsType, - diskMounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}}, nil + diskMounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host)}, nil } func (plugin *photonPersistentDiskPlugin) newUnmounterInternal(volName string, podUID types.UID, manager pdManager, mounter mount.Interface) (volume.Unmounter, error) { @@ -131,7 +130,7 @@ func (plugin *photonPersistentDiskPlugin) newUnmounterInternal(volName string, p } func (plugin *photonPersistentDiskPlugin) ConstructVolumeSpec(volumeSpecName, mountPath string) (*volume.Spec, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName()) pdID, err := mounter.GetDeviceNameFromMount(mountPath, pluginDir) if err != nil { diff --git a/pkg/volume/photon_pd/photon_pd_test.go b/pkg/volume/photon_pd/photon_pd_test.go index b874c5b0da624..719727307e534 100644 --- a/pkg/volume/photon_pd/photon_pd_test.go +++ b/pkg/volume/photon_pd/photon_pd_test.go @@ -37,7 +37,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/photon-pd") if err != nil { @@ -61,7 +61,7 @@ func TestGetAccessModes(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/photon-pd") if err != nil { @@ -106,7 +106,7 @@ func TestPlugin(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/photon-pd") if err != nil { @@ -179,6 +179,9 @@ func TestPlugin(t *testing.T) { PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimDelete, } provisioner, err := plug.(*photonPersistentDiskPlugin).newProvisionerInternal(options, &fakePDManager{}) + if err != nil { + t.Fatalf("Error creating new provisioner:%v", err) + } persistentSpec, err := provisioner.Provision() if err != nil { t.Errorf("Provision() failed: %v", err) @@ -198,6 +201,9 @@ func TestPlugin(t *testing.T) { PersistentVolume: persistentSpec, } deleter, err := plug.(*photonPersistentDiskPlugin).newDeleterInternal(volSpec, &fakePDManager{}) + if err != nil { + t.Fatalf("Error creating new deleter:%v", err) + } err = deleter.Delete() if err != nil { t.Errorf("Deleter() failed: %v", err) @@ -211,7 +217,7 @@ func TestMounterAndUnmounterTypeAssert(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/photon-pd") if err != nil { @@ -228,11 +234,17 @@ func TestMounterAndUnmounterTypeAssert(t *testing.T) { } mounter, err := plug.(*photonPersistentDiskPlugin).newMounterInternal(volume.NewSpecFromVolume(spec), types.UID("poduid"), &fakePDManager{}, &mount.FakeMounter{}) + if err != nil { + t.Fatalf("Error creating new mounter:%v", err) + } if _, ok := mounter.(volume.Unmounter); ok { t.Errorf("Volume Mounter can be type-assert to Unmounter") } unmounter, err := plug.(*photonPersistentDiskPlugin).newUnmounterInternal("vol1", types.UID("poduid"), &fakePDManager{}, &mount.FakeMounter{}) + if err != nil { + t.Fatalf("Error creating new unmounter:%v", err) + } if _, ok := unmounter.(volume.Mounter); ok { t.Errorf("Volume Unmounter can be type-assert to Mounter") } diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index 39c67366ffc60..dccb99fce7d79 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -66,6 +66,19 @@ type VolumeOptions struct { CloudTags *map[string]string // Volume provisioning parameters from StorageClass Parameters map[string]string + // This flag helps identify whether kubelet is running in a container + Containerized bool +} + +type DynamicPluginProber interface { + Init() error + + // If an update has occurred since the last probe, updated = true + // and the list of probed plugins is returned. + // Otherwise, update = false and probedPlugins = nil. + // + // If an error occurs, updated and probedPlugins are undefined. + Probe() (updated bool, probedPlugins []VolumePlugin, err error) } // VolumePlugin is an interface to volume plugins that can be used on a @@ -224,7 +237,7 @@ type VolumeHost interface { GetCloudProvider() cloudprovider.Interface // Get mounter interface. - GetMounter() mount.Interface + GetMounter(pluginName string) mount.Interface // Get writer interface for writing data to disk. GetWriter() io.Writer @@ -244,15 +257,20 @@ type VolumeHost interface { // Returns a function that returns a configmap. GetConfigMapFunc() func(namespace, name string) (*v1.ConfigMap, error) + // Returns an interface that should be used to execute any utilities in volume plugins + GetExec(pluginName string) mount.Exec + // Returns the labels on the node GetNodeLabels() (map[string]string, error) } // VolumePluginMgr tracks registered plugins. type VolumePluginMgr struct { - mutex sync.Mutex - plugins map[string]VolumePlugin - Host VolumeHost + mutex sync.Mutex + plugins map[string]VolumePlugin + prober DynamicPluginProber + probedPlugins []VolumePlugin + Host VolumeHost } // Spec is an internal representation of a volume. All API volume types translate to Spec. @@ -347,11 +365,24 @@ func NewSpecFromPersistentVolume(pv *v1.PersistentVolume, readOnly bool) *Spec { // InitPlugins initializes each plugin. All plugins must have unique names. // This must be called exactly once before any New* methods are called on any // plugins. -func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, host VolumeHost) error { +func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, prober DynamicPluginProber, host VolumeHost) error { pm.mutex.Lock() defer pm.mutex.Unlock() pm.Host = host + + if prober == nil { + // Use a dummy prober to prevent nil deference. + pm.prober = &dummyPluginProber{} + } else { + pm.prober = prober + } + if err := pm.prober.Init(); err != nil { + // Prober init failure should not affect the initialization of other plugins. + glog.Errorf("Error initializing dynamic plugin prober: %s", err) + pm.prober = &dummyPluginProber{} + } + if pm.plugins == nil { pm.plugins = map[string]VolumePlugin{} } @@ -380,6 +411,21 @@ func (pm *VolumePluginMgr) InitPlugins(plugins []VolumePlugin, host VolumeHost) return utilerrors.NewAggregate(allErrs) } +func (pm *VolumePluginMgr) initProbedPlugin(probedPlugin VolumePlugin) error { + name := probedPlugin.GetPluginName() + if errs := validation.IsQualifiedName(name); len(errs) != 0 { + return fmt.Errorf("volume plugin has invalid name: %q: %s", name, strings.Join(errs, ";")) + } + + err := probedPlugin.Init(pm.Host) + if err != nil { + return fmt.Errorf("Failed to load volume plugin %s, error: %s", name, err.Error()) + } + + glog.V(1).Infof("Loaded volume plugin %q", name) + return nil +} + // FindPluginBySpec looks for a plugin that can support a given volume // specification. If no plugins can support or more than one plugin can // support it, return error. @@ -391,19 +437,30 @@ func (pm *VolumePluginMgr) FindPluginBySpec(spec *Spec) (VolumePlugin, error) { return nil, fmt.Errorf("Could not find plugin because volume spec is nil") } - matches := []string{} + matchedPluginNames := []string{} + matches := []VolumePlugin{} for k, v := range pm.plugins { if v.CanSupport(spec) { - matches = append(matches, k) + matchedPluginNames = append(matchedPluginNames, k) + matches = append(matches, v) } } + + pm.refreshProbedPlugins() + for _, plugin := range pm.probedPlugins { + if plugin.CanSupport(spec) { + matchedPluginNames = append(matchedPluginNames, plugin.GetPluginName()) + matches = append(matches, plugin) + } + } + if len(matches) == 0 { return nil, fmt.Errorf("no volume plugin matched") } if len(matches) > 1 { - return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matches, ",")) + return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ",")) } - return pm.plugins[matches[0]], nil + return matches[0], nil } // FindPluginByName fetches a plugin by name or by legacy name. If no plugin @@ -413,19 +470,52 @@ func (pm *VolumePluginMgr) FindPluginByName(name string) (VolumePlugin, error) { defer pm.mutex.Unlock() // Once we can get rid of legacy names we can reduce this to a map lookup. - matches := []string{} + matchedPluginNames := []string{} + matches := []VolumePlugin{} for k, v := range pm.plugins { if v.GetPluginName() == name { - matches = append(matches, k) + matchedPluginNames = append(matchedPluginNames, k) + matches = append(matches, v) } } + + pm.refreshProbedPlugins() + for _, plugin := range pm.probedPlugins { + if plugin.GetPluginName() == name { + matchedPluginNames = append(matchedPluginNames, plugin.GetPluginName()) + matches = append(matches, plugin) + } + } + if len(matches) == 0 { return nil, fmt.Errorf("no volume plugin matched") } if len(matches) > 1 { - return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matches, ",")) + return nil, fmt.Errorf("multiple volume plugins matched: %s", strings.Join(matchedPluginNames, ",")) + } + return matches[0], nil +} + +// Check if probedPlugin cache update is required. +// If it is, initialize all probed plugins and replace the cache with them. +func (pm *VolumePluginMgr) refreshProbedPlugins() { + updated, plugins, err := pm.prober.Probe() + if err != nil { + glog.Errorf("Error dynamically probing plugins: %s", err) + return // Use cached plugins upon failure. + } + + if updated { + pm.probedPlugins = []VolumePlugin{} + for _, plugin := range plugins { + if err := pm.initProbedPlugin(plugin); err != nil { + glog.Errorf("Error initializing dynamically probed plugin %s; error: %s", + plugin.GetPluginName(), err) + continue + } + pm.probedPlugins = append(pm.probedPlugins, plugin) + } } - return pm.plugins[matches[0]], nil } // FindPersistentPluginBySpec looks for a persistent volume plugin that can @@ -613,3 +703,8 @@ func ValidateRecyclerPodTemplate(pod *v1.Pod) error { } return nil } + +type dummyPluginProber struct{} + +func (*dummyPluginProber) Init() error { return nil } +func (*dummyPluginProber) Probe() (bool, []VolumePlugin, error) { return false, nil, nil } diff --git a/pkg/volume/plugins_test.go b/pkg/volume/plugins_test.go index fc319e1027159..b8d74a26fa876 100644 --- a/pkg/volume/plugins_test.go +++ b/pkg/volume/plugins_test.go @@ -103,7 +103,8 @@ func newTestPlugin() []VolumePlugin { func TestVolumePluginMgrFunc(t *testing.T) { vpm := VolumePluginMgr{} - vpm.InitPlugins(newTestPlugin(), nil) + var prober DynamicPluginProber = nil // TODO (#51147) inject mock + vpm.InitPlugins(newTestPlugin(), prober, nil) plug, err := vpm.FindPluginByName("testPlugin") if err != nil { diff --git a/pkg/volume/portworx/BUILD b/pkg/volume/portworx/BUILD index 5f7d8abe3cee0..155eddc4ed3bb 100644 --- a/pkg/volume/portworx/BUILD +++ b/pkg/volume/portworx/BUILD @@ -43,7 +43,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/portworx/portworx.go b/pkg/volume/portworx/portworx.go index e65d4687f404f..165e9206e243e 100644 --- a/pkg/volume/portworx/portworx.go +++ b/pkg/volume/portworx/portworx.go @@ -29,7 +29,6 @@ import ( kstrings "k8s.io/kubernetes/pkg/util/strings" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) // This is the primary entrypoint for volume plugins. @@ -91,7 +90,7 @@ func (plugin *portworxVolumePlugin) GetAccessModes() []v1.PersistentVolumeAccess } func (plugin *portworxVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - return plugin.newMounterInternal(spec, pod.UID, plugin.util, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod.UID, plugin.util, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *portworxVolumePlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager portworxManager, mounter mount.Interface) (volume.Mounter, error) { @@ -115,11 +114,11 @@ func (plugin *portworxVolumePlugin) newMounterInternal(spec *volume.Spec, podUID }, fsType: fsType, readOnly: readOnly, - diskMounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(), Runner: exec.New()}}, nil + diskMounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host)}, nil } func (plugin *portworxVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, plugin.util, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, plugin.util, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *portworxVolumePlugin) newUnmounterInternal(volName string, podUID types.UID, manager portworxManager, diff --git a/pkg/volume/portworx/portworx_test.go b/pkg/volume/portworx/portworx_test.go index 195e6c784721f..e14ef825020a4 100644 --- a/pkg/volume/portworx/portworx_test.go +++ b/pkg/volume/portworx/portworx_test.go @@ -41,7 +41,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/portworx-volume") if err != nil { @@ -65,7 +65,7 @@ func TestGetAccessModes(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/portworx-volume") if err != nil { @@ -135,7 +135,7 @@ func TestPlugin(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/portworx-volume") if err != nil { diff --git a/pkg/volume/projected/projected_test.go b/pkg/volume/projected/projected_test.go index 3c65ce1107ac7..6b8258652393b 100644 --- a/pkg/volume/projected/projected_test.go +++ b/pkg/volume/projected/projected_test.go @@ -670,7 +670,7 @@ func TestCanSupport(t *testing.T) { pluginMgr := volume.VolumePluginMgr{} tempDir, host := newTestHost(t, nil) defer os.RemoveAll(tempDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(projectedPluginName) if err != nil { @@ -701,7 +701,7 @@ func TestPlugin(t *testing.T) { rootDir, host = newTestHost(t, client) ) defer os.RemoveAll(rootDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(projectedPluginName) if err != nil { @@ -765,7 +765,7 @@ func TestPluginReboot(t *testing.T) { rootDir, host = newTestHost(t, client) ) defer os.RemoveAll(rootDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(projectedPluginName) if err != nil { @@ -819,7 +819,7 @@ func TestPluginOptional(t *testing.T) { ) volumeSpec.VolumeSource.Projected.Sources[0].Secret.Optional = &trueVal defer os.RemoveAll(rootDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(projectedPluginName) if err != nil { @@ -896,7 +896,7 @@ func TestPluginOptionalKeys(t *testing.T) { } volumeSpec.VolumeSource.Projected.Sources[0].Secret.Optional = &trueVal defer os.RemoveAll(rootDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(projectedPluginName) if err != nil { diff --git a/pkg/volume/quobyte/BUILD b/pkg/volume/quobyte/BUILD index d72c52334e878..af4351eb8d093 100644 --- a/pkg/volume/quobyte/BUILD +++ b/pkg/volume/quobyte/BUILD @@ -26,7 +26,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/quobyte/quobyte.go b/pkg/volume/quobyte/quobyte.go index a63d952aa28d9..c2cff8b7fdee8 100644 --- a/pkg/volume/quobyte/quobyte.go +++ b/pkg/volume/quobyte/quobyte.go @@ -33,7 +33,6 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) // ProbeVolumePlugins is the primary entrypoint for volume plugins. @@ -92,7 +91,7 @@ func (plugin *quobytePlugin) CanSupport(spec *volume.Spec) bool { } // If Quobyte is already mounted we don't need to check if the binary is installed - if mounter, err := plugin.newMounterInternal(spec, nil, plugin.host.GetMounter()); err == nil { + if mounter, err := plugin.newMounterInternal(spec, nil, plugin.host.GetMounter(plugin.GetPluginName())); err == nil { qm, _ := mounter.(*quobyteMounter) pluginDir := plugin.host.GetPluginDir(strings.EscapeQualifiedNameForDisk(quobytePluginName)) if mounted, err := qm.pluginDirIsMounted(pluginDir); mounted && err == nil { @@ -103,7 +102,8 @@ func (plugin *quobytePlugin) CanSupport(spec *volume.Spec) bool { glog.V(4).Infof("quobyte: Error: %v", err) } - if out, err := exec.New().Command("ls", "/sbin/mount.quobyte").CombinedOutput(); err == nil { + exec := plugin.host.GetExec(plugin.GetPluginName()) + if out, err := exec.Run("ls", "/sbin/mount.quobyte"); err == nil { glog.V(4).Infof("quobyte: can support: %s", string(out)) return true } @@ -155,7 +155,7 @@ func (plugin *quobytePlugin) ConstructVolumeSpec(volumeName, mountPath string) ( } func (plugin *quobytePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *quobytePlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, mounter mount.Interface) (volume.Mounter, error) { @@ -181,7 +181,7 @@ func (plugin *quobytePlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, } func (plugin *quobytePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *quobytePlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) { @@ -395,7 +395,7 @@ func (provisioner *quobyteVolumeProvisioner) Provision() (*v1.PersistentVolume, } if !validateRegistry(provisioner.registry) { - return nil, fmt.Errorf("Quoybte registry missing or malformed: must be a host:port pair or multiple pairs separated by commas") + return nil, fmt.Errorf("Quobyte registry missing or malformed: must be a host:port pair or multiple pairs separated by commas") } // create random image name @@ -465,7 +465,7 @@ func parseAPIConfig(plugin *quobytePlugin, params map[string]string) (*quobyteAP } if len(apiServer) == 0 { - return nil, fmt.Errorf("Quoybte API server missing or malformed: must be a http(s)://host:port pair or multiple pairs separated by commas") + return nil, fmt.Errorf("Quobyte API server missing or malformed: must be a http(s)://host:port pair or multiple pairs separated by commas") } secretMap, err := util.GetSecretForPV(secretNamespace, secretName, quobytePluginName, plugin.host.GetKubeClient()) diff --git a/pkg/volume/quobyte/quobyte_test.go b/pkg/volume/quobyte/quobyte_test.go index 6dcc5616b4d18..1ae60e4078b93 100644 --- a/pkg/volume/quobyte/quobyte_test.go +++ b/pkg/volume/quobyte/quobyte_test.go @@ -39,7 +39,7 @@ func TestCanSupport(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/quobyte") if err != nil { t.Errorf("Can't find the plugin by name") @@ -63,7 +63,7 @@ func TestGetAccessModes(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/quobyte") if err != nil { @@ -91,7 +91,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/quobyte") if err != nil { t.Errorf("Can't find the plugin by name") @@ -187,13 +187,16 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { client := fake.NewSimpleClientset(pv, claim) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, client, nil)) plug, _ := plugMgr.FindPluginByName(quobytePluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes spec := volume.NewSpecFromPersistentVolume(pv, true) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{}) + if mounter == nil { + t.Fatalf("Got a nil Mounter") + } if !mounter.GetAttributes().ReadOnly { t.Errorf("Expected true for mounter.IsReadOnly") diff --git a/pkg/volume/rbd/BUILD b/pkg/volume/rbd/BUILD index 32b6efcfab8e4..f906a9dea4dea 100644 --- a/pkg/volume/rbd/BUILD +++ b/pkg/volume/rbd/BUILD @@ -29,7 +29,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/rbd/disk_manager.go b/pkg/volume/rbd/disk_manager.go index 482151c075dde..d219d62ab1b1d 100644 --- a/pkg/volume/rbd/disk_manager.go +++ b/pkg/volume/rbd/disk_manager.go @@ -77,6 +77,7 @@ func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount. glog.Errorf("failed to bind mount:%s", globalPDPath) return err } + glog.V(3).Infof("rbd: successfully bind mount %s to %s with options %v", globalPDPath, volPath, mountOptions) if !b.ReadOnly { volume.SetVolumeOwnership(&b, fsGroup) diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index ba06953e265cd..4aa87d9db89d4 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -33,7 +33,6 @@ import ( "k8s.io/kubernetes/pkg/volume" volutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) var ( @@ -42,12 +41,11 @@ var ( // This is the primary entrypoint for volume plugins. func ProbeVolumePlugins() []volume.VolumePlugin { - return []volume.VolumePlugin{&rbdPlugin{nil, exec.New()}} + return []volume.VolumePlugin{&rbdPlugin{nil}} } type rbdPlugin struct { host volume.VolumeHost - exe exec.Interface } var _ volume.VolumePlugin = &rbdPlugin{} @@ -131,7 +129,7 @@ func (plugin *rbdPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.Vol } // Inject real implementations here, test through the internal function. - return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, plugin.host.GetMounter(), secret) + return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret) } func (plugin *rbdPlugin) getRBDVolumeSource(spec *volume.Spec) (*v1.RBDVolumeSource, bool) { @@ -144,7 +142,7 @@ func (plugin *rbdPlugin) getRBDVolumeSource(spec *volume.Spec) (*v1.RBDVolumeSou } } -func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, secret string) (volume.Mounter, error) { +func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec, secret string) (volume.Mounter, error) { source, readOnly := plugin.getRBDVolumeSource(spec) pool := source.RBDPool id := source.RadosUser @@ -158,7 +156,8 @@ func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, Pool: pool, ReadOnly: readOnly, manager: manager, - mounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}, + mounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}, + exec: exec, plugin: plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)), }, @@ -173,17 +172,18 @@ func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, func (plugin *rbdPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { // Inject real implementations here, test through the internal function. - return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{}, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) } -func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface) (volume.Unmounter, error) { +func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec) (volume.Unmounter, error) { return &rbdUnmounter{ rbdMounter: &rbdMounter{ rbd: &rbd{ podUID: podUID, volName: volName, manager: manager, - mounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}, + mounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}, + exec: exec, plugin: plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)), }, @@ -246,6 +246,8 @@ func (plugin *rbdPlugin) newDeleterInternal(spec *volume.Spec, admin, secret str Pool: spec.PersistentVolume.Spec.RBD.RBDPool, manager: manager, plugin: plugin, + mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())}, + exec: plugin.host.GetExec(plugin.GetPluginName()), }, Mon: spec.PersistentVolume.Spec.RBD.CephMonitors, adminId: admin, @@ -263,6 +265,8 @@ func (plugin *rbdPlugin) newProvisionerInternal(options volume.VolumeOptions, ma rbd: &rbd{ manager: manager, plugin: plugin, + mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())}, + exec: plugin.host.GetExec(plugin.GetPluginName()), }, }, options: options, @@ -402,6 +406,7 @@ type rbd struct { ReadOnly bool plugin *rbdPlugin mounter *mount.SafeFormatAndMount + exec mount.Exec // Utility interface that provides API calls to the provider to attach/detach disks. manager diskManager volume.MetricsProvider @@ -480,11 +485,6 @@ func (c *rbdUnmounter) TearDownAt(dir string) error { return diskTearDown(c.manager, *c, dir, c.mounter) } -func (plugin *rbdPlugin) execCommand(command string, args []string) ([]byte, error) { - cmd := plugin.exe.Command(command, args...) - return cmd.CombinedOutput() -} - func getVolumeSource( spec *volume.Spec) (*v1.RBDVolumeSource, bool, error) { if spec.Volume != nil && spec.Volume.RBD != nil { diff --git a/pkg/volume/rbd/rbd_test.go b/pkg/volume/rbd/rbd_test.go index 6ef38de89ecef..ba27d570c3c72 100644 --- a/pkg/volume/rbd/rbd_test.go +++ b/pkg/volume/rbd/rbd_test.go @@ -39,7 +39,7 @@ func TestCanSupport(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/rbd") if err != nil { @@ -104,7 +104,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/rbd") if err != nil { @@ -112,7 +112,8 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { } fdm := NewFakeDiskManager() defer fdm.Cleanup() - mounter, err := plug.(*rbdPlugin).newMounterInternal(spec, types.UID("poduid"), fdm, &mount.FakeMounter{}, "secrets") + exec := mount.NewFakeExec(nil) + mounter, err := plug.(*rbdPlugin).newMounterInternal(spec, types.UID("poduid"), fdm, &mount.FakeMounter{}, exec, "secrets") if err != nil { t.Errorf("Failed to make a new Mounter: %v", err) } @@ -137,7 +138,7 @@ func doTestPlugin(t *testing.T, spec *volume.Spec) { } } - unmounter, err := plug.(*rbdPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fdm, &mount.FakeMounter{}) + unmounter, err := plug.(*rbdPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fdm, &mount.FakeMounter{}, exec) if err != nil { t.Errorf("Failed to make a new Unmounter: %v", err) } @@ -228,13 +229,16 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { client := fake.NewSimpleClientset(pv, claim) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, client, nil)) plug, _ := plugMgr.FindPluginByName(rbdPluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes spec := volume.NewSpecFromPersistentVolume(pv, true) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}} mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{}) + if mounter == nil { + t.Fatalf("Got a nil Mounter") + } if !mounter.GetAttributes().ReadOnly { t.Errorf("Expected true for mounter.IsReadOnly") diff --git a/pkg/volume/rbd/rbd_util.go b/pkg/volume/rbd/rbd_util.go index 001054e903a16..1f06f16a4e8df 100644 --- a/pkg/volume/rbd/rbd_util.go +++ b/pkg/volume/rbd/rbd_util.go @@ -38,7 +38,6 @@ import ( "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/volume" - "k8s.io/utils/exec" ) const ( @@ -58,15 +57,24 @@ func getDevFromImageAndPool(pool, image string) (string, bool) { // https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c name := f.Name() // first match pool, then match name - po := path.Join(sys_path, name, "pool") - img := path.Join(sys_path, name, "name") - exe := exec.New() - out, err := exe.Command("cat", po, img).CombinedOutput() + poolFile := path.Join(sys_path, name, "pool") + poolBytes, err := ioutil.ReadFile(poolFile) if err != nil { + glog.V(4).Infof("Error reading %s: %v", poolFile, err) continue } - matched, err := regexp.MatchString("^"+pool+"\n"+image+"\n$", string(out)) - if err != nil || !matched { + if strings.TrimSpace(string(poolBytes)) != pool { + glog.V(4).Infof("Device %s is not %q: %q", name, pool, string(poolBytes)) + continue + } + imgFile := path.Join(sys_path, name, "name") + imgBytes, err := ioutil.ReadFile(imgFile) + if err != nil { + glog.V(4).Infof("Error reading %s: %v", imgFile, err) + continue + } + if strings.TrimSpace(string(imgBytes)) != image { + glog.V(4).Infof("Device %s is not %q: %q", name, image, string(imgBytes)) continue } // found a match, check if device exists @@ -142,8 +150,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { // for fencing, check if lock already held for this host // this edge case happens if host crashes in the middle of acquiring lock and mounting rbd // for defencing, get the locker name, something like "client.1234" - cmd, err = b.plugin.execCommand("rbd", - append([]string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) + args := []string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon} + args = append(args, secret_opt...) + cmd, err = b.exec.Run("rbd", args...) output = string(cmd) glog.Infof("lock list output %q", output) if err != nil { @@ -166,8 +175,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { if len(v) > 0 { lockInfo := strings.Split(v[0], " ") if len(lockInfo) > 2 { - cmd, err = b.plugin.execCommand("rbd", - append([]string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) + args := []string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon} + args = append(args, secret_opt...) + cmd, err = b.exec.Run("rbd", args...) glog.Infof("remove orphaned locker %s from client %s: err %v, output: %s", lockInfo[1], lockInfo[0], err, string(cmd)) } } @@ -175,8 +185,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { } // hold a lock: rbd lock add - cmd, err = b.plugin.execCommand("rbd", - append([]string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) + args := []string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon} + args = append(args, secret_opt...) + cmd, err = b.exec.Run("rbd", args...) } else { // defencing, find locker name ind := strings.LastIndex(output, lock_id) - 1 @@ -187,8 +198,9 @@ func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error { } } // remove a lock: rbd lock remove - cmd, err = b.plugin.execCommand("rbd", - append([]string{"lock", "remove", b.Image, lock_id, locker, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...)) + args := []string{"lock", "remove", b.Image, lock_id, locker, "--pool", b.Pool, "--id", b.Id, "-m", mon} + args = append(args, secret_opt...) + cmd, err = b.exec.Run("rbd", args...) } if err == nil { @@ -268,8 +280,7 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error { devicePath, found := waitForPath(b.Pool, b.Image, 1) if !found { - // modprobe - _, err = b.plugin.execCommand("modprobe", []string{"rbd"}) + _, err = b.exec.Run("modprobe", "rbd") if err != nil { glog.Warningf("rbd: failed to load rbd kernel module:%v", err) } @@ -294,11 +305,11 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error { mon := b.Mon[i%l] glog.V(1).Infof("rbd: map mon %s", mon) if b.Secret != "" { - output, err = b.plugin.execCommand("rbd", - []string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key=" + b.Secret}) + output, err = b.exec.Run("rbd", + "map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key="+b.Secret) } else { - output, err = b.plugin.execCommand("rbd", - []string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring}) + output, err = b.exec.Run("rbd", + "map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring) } if err == nil { break @@ -312,12 +323,14 @@ func (util *RBDUtil) AttachDisk(b rbdMounter) error { if !found { return errors.New("Could not map image: Timeout after 10s") } + glog.V(3).Infof("rbd: successfully map image %s/%s to %s", b.Pool, b.Image, devicePath) } // mount it if err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil); err != nil { err = fmt.Errorf("rbd: failed to mount rbd volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err) } + glog.V(3).Infof("rbd: successfully mount image %s/%s at %s", b.Pool, b.Image, globalPDPath) return err } @@ -329,10 +342,11 @@ func (util *RBDUtil) DetachDisk(c rbdUnmounter, mntPath string) error { if err = c.mounter.Unmount(mntPath); err != nil { return fmt.Errorf("rbd detach disk: failed to umount: %s\nError: %v", mntPath, err) } + glog.V(3).Infof("rbd: successfully umount mountpoint %s", mntPath) // if device is no longer used, see if can unmap if cnt <= 1 { // rbd unmap - _, err = c.plugin.execCommand("rbd", []string{"unmap", device}) + _, err = c.exec.Run("rbd", "unmap", device) if err != nil { return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s:Error: %v", device, err)) } @@ -343,7 +357,7 @@ func (util *RBDUtil) DetachDisk(c rbdUnmounter, mntPath string) error { util.defencing(c) } - glog.Infof("rbd: successfully unmap device %s", device) + glog.V(3).Infof("rbd: successfully unmap device %s", device) } return nil } @@ -374,7 +388,7 @@ func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource features := strings.Join(p.rbdMounter.imageFeatures, ",") args = append(args, "--image-feature", features) } - output, err = p.rbdMounter.plugin.execCommand("rbd", args) + output, err = p.exec.Run("rbd", args...) if err == nil { break } else { @@ -411,8 +425,8 @@ func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error { for i := start; i < start+l; i++ { mon := p.rbdMounter.Mon[i%l] glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret) - output, err = p.plugin.execCommand("rbd", - []string{"rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret}) + output, err = p.exec.Run("rbd", + "rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key="+p.rbdMounter.adminSecret) if err == nil { return nil } else { @@ -437,8 +451,8 @@ func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, error) { // Watchers: // watcher=10.16.153.105:0/710245699 client.14163 cookie=1 glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, b.adminId, b.adminSecret) - cmd, err = b.plugin.execCommand("rbd", - []string{"status", b.Image, "--pool", b.Pool, "-m", mon, "--id", b.adminId, "--key=" + b.adminSecret}) + cmd, err = b.exec.Run("rbd", + "status", b.Image, "--pool", b.Pool, "-m", mon, "--id", b.adminId, "--key="+b.adminSecret) output = string(cmd) if err != nil { diff --git a/pkg/volume/scaleio/BUILD b/pkg/volume/scaleio/BUILD index cdd168fd922dc..6374c05205c0d 100644 --- a/pkg/volume/scaleio/BUILD +++ b/pkg/volume/scaleio/BUILD @@ -52,7 +52,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/scaleio/sio_client.go b/pkg/volume/scaleio/sio_client.go index 8742905ed09d0..ac26868d324f1 100644 --- a/pkg/volume/scaleio/sio_client.go +++ b/pkg/volume/scaleio/sio_client.go @@ -21,7 +21,6 @@ import ( "fmt" "io/ioutil" "os" - "os/exec" "path" "path/filepath" "regexp" @@ -30,6 +29,8 @@ import ( "sync" "time" + "k8s.io/kubernetes/pkg/util/mount" + sio "github.com/codedellemc/goscaleio" siotypes "github.com/codedellemc/goscaleio/types/v1" "github.com/golang/glog" @@ -77,13 +78,15 @@ type sioClient struct { inited bool diskRegex *regexp.Regexp mtx sync.Mutex + exec mount.Exec } -func newSioClient(gateway, username, password string, sslEnabled bool) (*sioClient, error) { +func newSioClient(gateway, username, password string, sslEnabled bool, exec mount.Exec) (*sioClient, error) { client := new(sioClient) client.gateway = gateway client.username = username client.password = password + client.exec = exec if sslEnabled { client.insecure = false client.certsEnabled = true @@ -296,7 +299,7 @@ func (c *sioClient) IID() (string, error) { if c.instanceID == "" { cmd := c.getSdcCmd() - output, err := exec.Command(cmd, "--query_guid").Output() + output, err := c.exec.Run(cmd, "--query_guid") if err != nil { glog.Error(log("drv_cfg --query_guid failed: %v", err)) return "", err @@ -355,7 +358,7 @@ func (c *sioClient) Devs() (map[string]string, error) { volumeMap := make(map[string]string) // grab the sdc tool output - out, err := exec.Command(c.getSdcCmd(), "--query_vols").Output() + out, err := c.exec.Run(c.getSdcCmd(), "--query_vols") if err != nil { glog.Error(log("sdc --query_vols failed: %v", err)) return nil, err diff --git a/pkg/volume/scaleio/sio_mgr.go b/pkg/volume/scaleio/sio_mgr.go index 83d5e498dc87e..ecde665a1ac3e 100644 --- a/pkg/volume/scaleio/sio_mgr.go +++ b/pkg/volume/scaleio/sio_mgr.go @@ -20,6 +20,8 @@ import ( "errors" "strconv" + "k8s.io/kubernetes/pkg/util/mount" + "github.com/golang/glog" siotypes "github.com/codedellemc/goscaleio/types/v1" @@ -36,9 +38,10 @@ type storageInterface interface { type sioMgr struct { client sioInterface configData map[string]string + exec mount.Exec } -func newSioMgr(configs map[string]string) (*sioMgr, error) { +func newSioMgr(configs map[string]string, exec mount.Exec) (*sioMgr, error) { if configs == nil { return nil, errors.New("missing configuration data") } @@ -47,7 +50,7 @@ func newSioMgr(configs map[string]string) (*sioMgr, error) { configs[confKey.sdcRootPath] = defaultString(configs[confKey.sdcRootPath], sdcRootPath) configs[confKey.storageMode] = defaultString(configs[confKey.storageMode], "ThinProvisioned") - mgr := &sioMgr{configData: configs} + mgr := &sioMgr{configData: configs, exec: exec} return mgr, nil } @@ -67,7 +70,7 @@ func (m *sioMgr) getClient() (sioInterface, error) { certsEnabled := b glog.V(4).Info(log("creating new client for gateway %s", gateway)) - client, err := newSioClient(gateway, username, password, certsEnabled) + client, err := newSioClient(gateway, username, password, certsEnabled, m.exec) if err != nil { glog.Error(log("failed to create scaleio client: %v", err)) return nil, err diff --git a/pkg/volume/scaleio/sio_mgr_test.go b/pkg/volume/scaleio/sio_mgr_test.go index 3d580b6b99bd3..e2fe5c2002be7 100644 --- a/pkg/volume/scaleio/sio_mgr_test.go +++ b/pkg/volume/scaleio/sio_mgr_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "k8s.io/kubernetes/pkg/util/mount" + siotypes "github.com/codedellemc/goscaleio/types/v1" ) @@ -42,7 +44,7 @@ var ( ) func newTestMgr(t *testing.T) *sioMgr { - mgr, err := newSioMgr(fakeConfig) + mgr, err := newSioMgr(fakeConfig, mount.NewFakeExec(nil)) if err != nil { t.Error(err) } @@ -51,7 +53,7 @@ func newTestMgr(t *testing.T) *sioMgr { } func TestMgrNew(t *testing.T) { - mgr, err := newSioMgr(fakeConfig) + mgr, err := newSioMgr(fakeConfig, mount.NewFakeExec(nil)) if err != nil { t.Fatal(err) } diff --git a/pkg/volume/scaleio/sio_plugin.go b/pkg/volume/scaleio/sio_plugin.go index 3b3a620aaf411..d775875621628 100644 --- a/pkg/volume/scaleio/sio_plugin.go +++ b/pkg/volume/scaleio/sio_plugin.go @@ -23,7 +23,6 @@ import ( api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/util/keymutex" - "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" ) @@ -35,7 +34,6 @@ const ( type sioPlugin struct { host volume.VolumeHost - mounter mount.Interface volumeMtx keymutex.KeyMutex } @@ -53,7 +51,6 @@ var _ volume.VolumePlugin = &sioPlugin{} func (p *sioPlugin) Init(host volume.VolumeHost) error { p.host = host - p.mounter = host.GetMounter() p.volumeMtx = keymutex.NewKeyMutex() return nil } diff --git a/pkg/volume/scaleio/sio_util_test.go b/pkg/volume/scaleio/sio_util_test.go index 352d6abb4d307..360b67e3fac2b 100644 --- a/pkg/volume/scaleio/sio_util_test.go +++ b/pkg/volume/scaleio/sio_util_test.go @@ -212,10 +212,13 @@ func TestUtilLoadConfig(t *testing.T) { configFile := path.Join(tmpDir, sioConfigFileName) if err := saveConfig(configFile, config); err != nil { - t.Fatal("failed while saving data", err) + t.Fatalf("failed to save configFile %s error:%v", configFile, err) } dataRcvd, err := loadConfig(configFile) + if err != nil { + t.Fatalf("failed to load configFile %s error:%v", configFile, err) + } if dataRcvd[confKey.gateway] != config[confKey.gateway] || dataRcvd[confKey.system] != config[confKey.system] { t.Fatal("loaded config data not matching saved config data") diff --git a/pkg/volume/scaleio/sio_volume.go b/pkg/volume/scaleio/sio_volume.go index 1897ea449e90d..079df8f67f92e 100644 --- a/pkg/volume/scaleio/sio_volume.go +++ b/pkg/volume/scaleio/sio_volume.go @@ -34,7 +34,6 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) type sioVolume struct { @@ -94,7 +93,8 @@ func (v *sioVolume) SetUpAt(dir string, fsGroup *int64) error { return err } - notDevMnt, err := v.plugin.mounter.IsLikelyNotMountPoint(dir) + mounter := v.plugin.host.GetMounter(v.plugin.GetPluginName()) + notDevMnt, err := mounter.IsLikelyNotMountPoint(dir) if err != nil && !os.IsNotExist(err) { glog.Error(log("IsLikelyNotMountPoint test failed for dir %v", dir)) return err @@ -143,10 +143,7 @@ func (v *sioVolume) SetUpAt(dir string, fsGroup *int64) error { } glog.V(4).Info(log("setup created mount point directory %s", dir)) - diskMounter := &mount.SafeFormatAndMount{ - Interface: v.plugin.mounter, - Runner: exec.New(), - } + diskMounter := volumehelper.NewSafeFormatAndMountFromHost(v.plugin.GetPluginName(), v.plugin.host) err = diskMounter.FormatAndMount(devicePath, dir, v.fsType, options) if err != nil { @@ -190,21 +187,22 @@ func (v *sioVolume) TearDownAt(dir string) error { v.plugin.volumeMtx.LockKey(v.volSpecName) defer v.plugin.volumeMtx.UnlockKey(v.volSpecName) - dev, _, err := mount.GetDeviceNameFromMount(v.plugin.mounter, dir) + mounter := v.plugin.host.GetMounter(v.plugin.GetPluginName()) + dev, _, err := mount.GetDeviceNameFromMount(mounter, dir) if err != nil { glog.Errorf(log("failed to get reference count for volume: %s", dir)) return err } glog.V(4).Info(log("attempting to unmount %s", dir)) - if err := util.UnmountPath(dir, v.plugin.mounter); err != nil { + if err := util.UnmountPath(dir, mounter); err != nil { glog.Error(log("teardown failed while unmounting dir %s: %v ", dir, err)) return err } glog.V(4).Info(log("dir %s unmounted successfully", dir)) // detach/unmap - deviceBusy, err := v.plugin.mounter.DeviceOpened(dev) + deviceBusy, err := mounter.DeviceOpened(dev) if err != nil { glog.Error(log("teardown unable to get status for device %s: %v", dev, err)) return err @@ -388,7 +386,7 @@ func (v *sioVolume) setSioMgr() error { return err } - mgr, err := newSioMgr(configData) + mgr, err := newSioMgr(configData, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { glog.Error(log("failed to reset sio manager: %v", err)) return err @@ -420,7 +418,7 @@ func (v *sioVolume) resetSioMgr() error { return err } - mgr, err := newSioMgr(configData) + mgr, err := newSioMgr(configData, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { glog.Error(log("failed to reset scaleio mgr: %v", err)) return err @@ -454,7 +452,7 @@ func (v *sioVolume) setSioMgrFromConfig() error { return err } - mgr, err := newSioMgr(data) + mgr, err := newSioMgr(data, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { glog.Error(log("failed while setting scaleio mgr from config: %v", err)) return err @@ -483,7 +481,7 @@ func (v *sioVolume) setSioMgrFromSpec() error { return err } - mgr, err := newSioMgr(configData) + mgr, err := newSioMgr(configData, v.plugin.host.GetExec(v.plugin.GetPluginName())) if err != nil { glog.Error(log("failed to reset sio manager: %v", err)) return err diff --git a/pkg/volume/scaleio/sio_volume_test.go b/pkg/volume/scaleio/sio_volume_test.go index 9d495d3e67524..b83be41828e02 100644 --- a/pkg/volume/scaleio/sio_volume_test.go +++ b/pkg/volume/scaleio/sio_volume_test.go @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/types" fakeclient "k8s.io/client-go/kubernetes/fake" utiltesting "k8s.io/client-go/util/testing" - "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" ) @@ -64,7 +63,7 @@ func newPluginMgr(t *testing.T) (*volume.VolumePluginMgr, string) { fakeClient := fakeclient.NewSimpleClientset(config) host := volumetest.NewFakeVolumeHost(tmpDir, fakeClient, nil) plugMgr := &volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), host) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) return plugMgr, tmpDir } @@ -138,8 +137,6 @@ func TestVolumeMounterUnmounter(t *testing.T) { t.Errorf("Cannot assert plugin to be type sioPlugin") } - sioPlug.mounter = &mount.FakeMounter{} - vol := &api.Volume{ Name: testSioVolName, VolumeSource: api.VolumeSource{ diff --git a/pkg/volume/secret/secret.go b/pkg/volume/secret/secret.go index ae551e822d777..67f4556e3f010 100644 --- a/pkg/volume/secret/secret.go +++ b/pkg/volume/secret/secret.go @@ -99,7 +99,7 @@ func (plugin *secretPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volu spec.Name(), pod.UID, plugin, - plugin.host.GetMounter(), + plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetWriter(), volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))), }, @@ -116,7 +116,7 @@ func (plugin *secretPlugin) NewUnmounter(volName string, podUID types.UID) (volu volName, podUID, plugin, - plugin.host.GetMounter(), + plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetWriter(), volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))), }, diff --git a/pkg/volume/secret/secret_test.go b/pkg/volume/secret/secret_test.go index 702b0999b31ce..6ba6a10d4b02c 100644 --- a/pkg/volume/secret/secret_test.go +++ b/pkg/volume/secret/secret_test.go @@ -275,7 +275,7 @@ func TestCanSupport(t *testing.T) { pluginMgr := volume.VolumePluginMgr{} tempDir, host := newTestHost(t, nil) defer os.RemoveAll(tempDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(secretPluginName) if err != nil { @@ -306,7 +306,7 @@ func TestPlugin(t *testing.T) { rootDir, host = newTestHost(t, client) ) defer os.RemoveAll(rootDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(secretPluginName) if err != nil { @@ -319,7 +319,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Failed to make a new Mounter: %v", err) } if mounter == nil { - t.Errorf("Got a nil Mounter") + t.Fatalf("Got a nil Mounter") } volumePath := mounter.GetPath() @@ -379,7 +379,7 @@ func TestPluginReboot(t *testing.T) { rootDir, host = newTestHost(t, client) ) defer os.RemoveAll(rootDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(secretPluginName) if err != nil { @@ -392,7 +392,7 @@ func TestPluginReboot(t *testing.T) { t.Errorf("Failed to make a new Mounter: %v", err) } if mounter == nil { - t.Errorf("Got a nil Mounter") + t.Fatalf("Got a nil Mounter") } podMetadataDir := fmt.Sprintf("%v/pods/test_pod_uid3/plugins/kubernetes.io~secret/test_volume_name", rootDir) @@ -433,7 +433,7 @@ func TestPluginOptional(t *testing.T) { ) volumeSpec.Secret.Optional = &trueVal defer os.RemoveAll(rootDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(secretPluginName) if err != nil { @@ -510,7 +510,7 @@ func TestPluginOptionalKeys(t *testing.T) { } volumeSpec.Secret.Optional = &trueVal defer os.RemoveAll(rootDir) - pluginMgr.InitPlugins(ProbeVolumePlugins(), host) + pluginMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) plugin, err := pluginMgr.FindPluginByName(secretPluginName) if err != nil { @@ -617,7 +617,7 @@ func doTestCleanAndTeardown(plugin volume.VolumePlugin, podUID types.UID, testVo t.Errorf("Failed to make a new Unmounter: %v", err) } if unmounter == nil { - t.Errorf("Got a nil Unmounter") + t.Fatalf("Got a nil Unmounter") } if err := unmounter.TearDown(); err != nil { diff --git a/pkg/volume/storageos/BUILD b/pkg/volume/storageos/BUILD index 14e5a73d0165e..fa29f8d24a1ca 100644 --- a/pkg/volume/storageos/BUILD +++ b/pkg/volume/storageos/BUILD @@ -27,7 +27,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/storageos/storageos.go b/pkg/volume/storageos/storageos.go index 6541f2eeb3fa9..532ccd2898788 100644 --- a/pkg/volume/storageos/storageos.go +++ b/pkg/volume/storageos/storageos.go @@ -36,7 +36,6 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) // ProbeVolumePlugins is the primary entrypoint for volume plugins. @@ -111,10 +110,10 @@ func (plugin *storageosPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volu return nil, err } - return plugin.newMounterInternal(spec, pod, apiCfg, &storageosUtil{}, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod, apiCfg, &storageosUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) } -func (plugin *storageosPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, apiCfg *storageosAPIConfig, manager storageosManager, mounter mount.Interface) (volume.Mounter, error) { +func (plugin *storageosPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod, apiCfg *storageosAPIConfig, manager storageosManager, mounter mount.Interface, exec mount.Exec) (volume.Mounter, error) { volName, volNamespace, fsType, readOnly, err := getVolumeInfoFromSpec(spec) if err != nil { @@ -133,19 +132,20 @@ func (plugin *storageosPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod apiCfg: apiCfg, manager: manager, mounter: mounter, + exec: exec, plugin: plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(pod.UID, volNamespace, volName, spec.Name(), plugin.host)), }, devicePath: storageosDevicePath, - diskMounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}, + diskMounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}, }, nil } func (plugin *storageosPlugin) NewUnmounter(pvName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(pvName, podUID, &storageosUtil{}, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(pvName, podUID, &storageosUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName())) } -func (plugin *storageosPlugin) newUnmounterInternal(pvName string, podUID types.UID, manager storageosManager, mounter mount.Interface) (volume.Unmounter, error) { +func (plugin *storageosPlugin) newUnmounterInternal(pvName string, podUID types.UID, manager storageosManager, mounter mount.Interface, exec mount.Exec) (volume.Unmounter, error) { // Parse volume namespace & name from mountpoint if mounted volNamespace, volName, err := getVolumeInfo(pvName, podUID, plugin.host) @@ -161,6 +161,7 @@ func (plugin *storageosPlugin) newUnmounterInternal(pvName string, podUID types. volNamespace: volNamespace, manager: manager, mounter: mounter, + exec: exec, plugin: plugin, MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volNamespace, volName, pvName, plugin.host)), }, @@ -304,6 +305,7 @@ type storageos struct { apiCfg *storageosAPIConfig manager storageosManager mounter mount.Interface + exec mount.Exec plugin *storageosPlugin volume.MetricsProvider } diff --git a/pkg/volume/storageos/storageos_test.go b/pkg/volume/storageos/storageos_test.go index 84120b319df5c..8f1f29887b783 100644 --- a/pkg/volume/storageos/storageos_test.go +++ b/pkg/volume/storageos/storageos_test.go @@ -39,7 +39,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/storageos") if err != nil { @@ -63,7 +63,7 @@ func TestGetAccessModes(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPersistentPluginByName("kubernetes.io/storageos") if err != nil { @@ -138,7 +138,7 @@ func TestPlugin(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/storageos") if err != nil { @@ -184,7 +184,7 @@ func TestPlugin(t *testing.T) { t.Errorf("Couldn't get secret from %v/%v", pod.Namespace, secretName) } - mounter, err := plug.(*storageosPlugin).newMounterInternal(volume.NewSpecFromVolume(spec), pod, apiCfg, fakeManager, &mount.FakeMounter{}) + mounter, err := plug.(*storageosPlugin).newMounterInternal(volume.NewSpecFromVolume(spec), pod, apiCfg, fakeManager, &mount.FakeMounter{}, mount.NewFakeExec(nil)) if err != nil { t.Fatalf("Failed to make a new Mounter: %v", err) } @@ -218,7 +218,7 @@ func TestPlugin(t *testing.T) { // Test Unmounter fakeManager = &fakePDManager{} - unmounter, err := plug.(*storageosPlugin).newUnmounterInternal("vol1-pvname", types.UID("poduid"), fakeManager, &mount.FakeMounter{}) + unmounter, err := plug.(*storageosPlugin).newUnmounterInternal("vol1-pvname", types.UID("poduid"), fakeManager, &mount.FakeMounter{}, mount.NewFakeExec(nil)) if err != nil { t.Errorf("Failed to make a new Unmounter: %v", err) } @@ -353,7 +353,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { client := fake.NewSimpleClientset(pv, claim) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, client, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, client, nil)) plug, _ := plugMgr.FindPluginByName(storageosPluginName) // readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes @@ -362,7 +362,7 @@ func TestPersistentClaimReadOnlyFlag(t *testing.T) { fakeManager := &fakePDManager{} fakeConfig := &fakeConfig{} apiCfg := fakeConfig.GetAPIConfig() - mounter, err := plug.(*storageosPlugin).newMounterInternal(spec, pod, apiCfg, fakeManager, &mount.FakeMounter{}) + mounter, err := plug.(*storageosPlugin).newMounterInternal(spec, pod, apiCfg, fakeManager, &mount.FakeMounter{}, mount.NewFakeExec(nil)) if !mounter.GetAttributes().ReadOnly { t.Errorf("Expected true for mounter.IsReadOnly") diff --git a/pkg/volume/storageos/storageos_util.go b/pkg/volume/storageos/storageos_util.go index b2d1767abf76c..107ba85901346 100644 --- a/pkg/volume/storageos/storageos_util.go +++ b/pkg/volume/storageos/storageos_util.go @@ -23,7 +23,7 @@ import ( "path" "strings" - "k8s.io/utils/exec" + "k8s.io/kubernetes/pkg/util/mount" "github.com/golang/glog" storageosapi "github.com/storageos/go-api" @@ -177,7 +177,7 @@ func (u *storageosUtil) AttachVolume(b *storageosMounter) (string, error) { case modeBlock: return srcPath, nil case modeFile: - return attachFileDevice(srcPath) + return attachFileDevice(srcPath, b.exec) default: return "", fmt.Errorf(ErrDeviceNotSupported) } @@ -192,7 +192,7 @@ func (u *storageosUtil) DetachVolume(b *storageosUnmounter, devicePath string) e if _, err := os.Stat(devicePath); os.IsNotExist(err) { return nil } - return removeLoopDevice(devicePath) + return removeLoopDevice(devicePath, b.exec) } // Mount mounts the volume on the host. @@ -295,8 +295,8 @@ func pathDeviceType(path string) (deviceType, error) { // attachFileDevice takes a path to a regular file and makes it available as an // attached block device. -func attachFileDevice(path string) (string, error) { - blockDevicePath, err := getLoopDevice(path) +func attachFileDevice(path string, exec mount.Exec) (string, error) { + blockDevicePath, err := getLoopDevice(path, exec) if err != nil && err.Error() != ErrDeviceNotFound { return "", err } @@ -304,7 +304,7 @@ func attachFileDevice(path string) (string, error) { // If no existing loop device for the path, create one if blockDevicePath == "" { glog.V(4).Infof("Creating device for path: %s", path) - blockDevicePath, err = makeLoopDevice(path) + blockDevicePath, err = makeLoopDevice(path, exec) if err != nil { return "", err } @@ -313,7 +313,7 @@ func attachFileDevice(path string) (string, error) { } // Returns the full path to the loop device associated with the given path. -func getLoopDevice(path string) (string, error) { +func getLoopDevice(path string, exec mount.Exec) (string, error) { _, err := os.Stat(path) if os.IsNotExist(err) { return "", errors.New(ErrNotAvailable) @@ -322,9 +322,8 @@ func getLoopDevice(path string) (string, error) { return "", fmt.Errorf("not attachable: %v", err) } - exec := exec.New() args := []string{"-j", path} - out, err := exec.Command(losetupPath, args...).CombinedOutput() + out, err := exec.Run(losetupPath, args...) if err != nil { glog.V(2).Infof("Failed device discover command for path %s: %v", path, err) return "", err @@ -332,10 +331,9 @@ func getLoopDevice(path string) (string, error) { return parseLosetupOutputForDevice(out) } -func makeLoopDevice(path string) (string, error) { - exec := exec.New() +func makeLoopDevice(path string, exec mount.Exec) (string, error) { args := []string{"-f", "--show", path} - out, err := exec.Command(losetupPath, args...).CombinedOutput() + out, err := exec.Run(losetupPath, args...) if err != nil { glog.V(2).Infof("Failed device create command for path %s: %v", path, err) return "", err @@ -343,10 +341,9 @@ func makeLoopDevice(path string) (string, error) { return parseLosetupOutputForDevice(out) } -func removeLoopDevice(device string) error { - exec := exec.New() +func removeLoopDevice(device string, exec mount.Exec) error { args := []string{"-d", device} - out, err := exec.Command(losetupPath, args...).CombinedOutput() + out, err := exec.Run(losetupPath, args...) if err != nil { if !strings.Contains(string(out), "No such device or address") { return err diff --git a/pkg/volume/storageos/storageos_util_test.go b/pkg/volume/storageos/storageos_util_test.go index 99897e7a0309f..3eee51231e770 100644 --- a/pkg/volume/storageos/storageos_util_test.go +++ b/pkg/volume/storageos/storageos_util_test.go @@ -117,7 +117,7 @@ func TestCreateVolume(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, _ := plugMgr.FindPluginByName("kubernetes.io/storageos") // Use real util with stubbed api @@ -209,7 +209,7 @@ func TestAttachVolume(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, _ := plugMgr.FindPluginByName("kubernetes.io/storageos") // Use real util with stubbed api diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index eacc124e64a55..d2903f5f14407 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -49,6 +49,7 @@ type fakeVolumeHost struct { pluginMgr VolumePluginMgr cloud cloudprovider.Interface mounter mount.Interface + exec mount.Exec writer io.Writer nodeLabels map[string]string } @@ -71,7 +72,8 @@ func newFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins [ host := &fakeVolumeHost{rootDir: rootDir, kubeClient: kubeClient, cloud: cloud} host.mounter = &mount.FakeMounter{} host.writer = &io.StdWriter{} - host.pluginMgr.InitPlugins(plugins, host) + host.exec = mount.NewFakeExec(nil) + host.pluginMgr.InitPlugins(plugins, nil /* prober */, host) return host } @@ -95,7 +97,7 @@ func (f *fakeVolumeHost) GetCloudProvider() cloudprovider.Interface { return f.cloud } -func (f *fakeVolumeHost) GetMounter() mount.Interface { +func (f *fakeVolumeHost) GetMounter(pluginName string) mount.Interface { return f.mounter } @@ -149,6 +151,10 @@ func (f *fakeVolumeHost) GetSecretFunc() func(namespace, name string) (*v1.Secre } } +func (f *fakeVolumeHost) GetExec(pluginName string) mount.Exec { + return f.exec +} + func (f *fakeVolumeHost) GetConfigMapFunc() func(namespace, name string) (*v1.ConfigMap, error) { return func(namespace, name string) (*v1.ConfigMap, error) { return f.kubeClient.Core().ConfigMaps(namespace).Get(name, metav1.GetOptions{}) @@ -429,7 +435,7 @@ func (fv *FakeVolume) GetAttachCallCount() int { return fv.AttachCallCount } -func (fv *FakeVolume) WaitForAttach(spec *Spec, devicePath string, spectimeout time.Duration) (string, error) { +func (fv *FakeVolume) WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, spectimeout time.Duration) (string, error) { fv.Lock() defer fv.Unlock() fv.WaitForAttachCallCount++ @@ -762,7 +768,7 @@ func GetTestVolumePluginMgr( nil, /* plugins */ ) plugins := ProbeVolumePlugins(VolumeConfig{}) - if err := v.pluginMgr.InitPlugins(plugins, v); err != nil { + if err := v.pluginMgr.InitPlugins(plugins, nil /* prober */, v); err != nil { t.Fatal(err) } diff --git a/pkg/volume/util.go b/pkg/volume/util.go index 0d2ee3bed2c10..6999b91beca9c 100644 --- a/pkg/volume/util.go +++ b/pkg/volume/util.go @@ -389,12 +389,17 @@ func MountOptionFromSpec(spec *Spec, options ...string) []string { pv := spec.PersistentVolume if pv != nil { + // Use beta annotation first if mo, ok := pv.Annotations[v1.MountOptionAnnotation]; ok { moList := strings.Split(mo, ",") return JoinMountOptions(moList, options) } + if len(pv.Spec.MountOptions) > 0 { + return JoinMountOptions(pv.Spec.MountOptions, options) + } } + return options } diff --git a/pkg/volume/util/BUILD b/pkg/volume/util/BUILD index 85c3ee9423b6e..6c2c7ac2f1746 100644 --- a/pkg/volume/util/BUILD +++ b/pkg/volume/util/BUILD @@ -15,6 +15,7 @@ go_library( "doc.go", "fs_unsupported.go", "io_util.go", + "metrics.go", "util.go", ] + select({ "@io_bazel_rules_go//go/platform:darwin_amd64": [ @@ -31,6 +32,7 @@ go_library( "//pkg/api/v1/helper:go_default_library", "//pkg/util/mount:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/storage/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", diff --git a/pkg/volume/util/metrics.go b/pkg/volume/util/metrics.go new file mode 100644 index 0000000000000..087bbfff41691 --- /dev/null +++ b/pkg/volume/util/metrics.go @@ -0,0 +1,63 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +var storageOperationMetric = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "storage_operation_duration_seconds", + Help: "Storage operation duration", + }, + []string{"volume_plugin", "operation_name"}, +) + +var storageOperationErrorMetric = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "storage_operation_errors_total", + Help: "Storage operation errors", + }, + []string{"volume_plugin", "operation_name"}, +) + +func init() { + registerMetrics() +} + +func registerMetrics() { + prometheus.MustRegister(storageOperationMetric) + prometheus.MustRegister(storageOperationErrorMetric) +} + +// OperationCompleteHook returns a hook to call when an operation is completed +func OperationCompleteHook(plugin, operationName string) func(error) { + requestTime := time.Now() + opComplete := func(err error) { + timeTaken := time.Since(requestTime).Seconds() + // Create metric with operation name and plugin name + if err != nil { + storageOperationErrorMetric.WithLabelValues(plugin, operationName).Inc() + } else { + storageOperationMetric.WithLabelValues(plugin, operationName).Observe(timeTaken) + } + } + return opComplete +} diff --git a/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go b/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go index 38613ec946e0a..82462c1f2f6c7 100644 --- a/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go +++ b/pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go @@ -55,7 +55,7 @@ type NestedPendingOperations interface { // concatenation of volumeName and podName is removed from the list of // executing operations allowing a new operation to be started with the // volumeName without error. - Run(volumeName v1.UniqueVolumeName, podName types.UniquePodName, operationFunc func() error) error + Run(volumeName v1.UniqueVolumeName, podName types.UniquePodName, operationFunc func() error, operationCompleteFunc func(error)) error // Wait blocks until all operations are completed. This is typically // necessary during tests - the test should wait until all operations finish @@ -94,7 +94,8 @@ type operation struct { func (grm *nestedPendingOperations) Run( volumeName v1.UniqueVolumeName, podName types.UniquePodName, - operationFunc func() error) error { + operationFunc func() error, + operationCompleteFunc func(error)) error { grm.lock.Lock() defer grm.lock.Unlock() opExists, previousOpIndex := grm.isOperationExists(volumeName, podName) @@ -132,6 +133,7 @@ func (grm *nestedPendingOperations) Run( defer k8sRuntime.HandleCrash() // Handle completion of and error, if any, from operationFunc() defer grm.operationComplete(volumeName, podName, &err) + defer operationCompleteFunc(err) // Handle panic, if any, from operationFunc() defer k8sRuntime.RecoverFromPanic(&err) return operationFunc() diff --git a/pkg/volume/util/nestedpendingoperations/nestedpendingoperations_test.go b/pkg/volume/util/nestedpendingoperations/nestedpendingoperations_test.go index 5cc3b508ff7b7..8882303bde554 100644 --- a/pkg/volume/util/nestedpendingoperations/nestedpendingoperations_test.go +++ b/pkg/volume/util/nestedpendingoperations/nestedpendingoperations_test.go @@ -50,7 +50,7 @@ func Test_NewGoRoutineMap_Positive_SingleOp(t *testing.T) { operation := func() error { return nil } // Act - err := grm.Run(volumeName, "" /* operationSubName */, operation) + err := grm.Run(volumeName, "" /* operationSubName */, operation, func(error) {}) // Assert if err != nil { @@ -66,8 +66,8 @@ func Test_NewGoRoutineMap_Positive_TwoOps(t *testing.T) { operation := func() error { return nil } // Act - err1 := grm.Run(volume1Name, "" /* operationSubName */, operation) - err2 := grm.Run(volume2Name, "" /* operationSubName */, operation) + err1 := grm.Run(volume1Name, "" /* operationSubName */, operation, func(error) {}) + err2 := grm.Run(volume2Name, "" /* operationSubName */, operation, func(error) {}) // Assert if err1 != nil { @@ -88,8 +88,8 @@ func Test_NewGoRoutineMap_Positive_TwoSubOps(t *testing.T) { operation := func() error { return nil } // Act - err1 := grm.Run(volumeName, operation1PodName, operation) - err2 := grm.Run(volumeName, operation2PodName, operation) + err1 := grm.Run(volumeName, operation1PodName, operation, func(error) {}) + err2 := grm.Run(volumeName, operation2PodName, operation, func(error) {}) // Assert if err1 != nil { @@ -108,7 +108,7 @@ func Test_NewGoRoutineMap_Positive_SingleOpWithExpBackoff(t *testing.T) { operation := func() error { return nil } // Act - err := grm.Run(volumeName, "" /* operationSubName */, operation) + err := grm.Run(volumeName, "" /* operationSubName */, operation, func(error) {}) // Assert if err != nil { @@ -122,7 +122,7 @@ func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstCompletes(t *testing.T) { volumeName := v1.UniqueVolumeName("volume-name") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateCallbackFunc(operation1DoneCh) - err1 := grm.Run(volumeName, "" /* operationSubName */, operation1) + err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } @@ -133,7 +133,7 @@ func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstCompletes(t *testing.T) { err2 := retryWithExponentialBackOff( time.Duration(initialOperationWaitTimeShort), func() (bool, error) { - err := grm.Run(volumeName, "" /* operationSubName */, operation2) + err := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {}) if err != nil { t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err) return false, nil @@ -154,7 +154,7 @@ func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstCompletesWithExpBackoff(t * volumeName := v1.UniqueVolumeName("volume-name") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateCallbackFunc(operation1DoneCh) - err1 := grm.Run(volumeName, "" /* operationSubName */, operation1) + err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } @@ -165,7 +165,7 @@ func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstCompletesWithExpBackoff(t * err2 := retryWithExponentialBackOff( time.Duration(initialOperationWaitTimeShort), func() (bool, error) { - err := grm.Run(volumeName, "" /* operationSubName */, operation2) + err := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {}) if err != nil { t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err) return false, nil @@ -185,7 +185,7 @@ func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstPanics(t *testing.T) { grm := NewNestedPendingOperations(false /* exponentialBackOffOnError */) volumeName := v1.UniqueVolumeName("volume-name") operation1 := generatePanicFunc() - err1 := grm.Run(volumeName, "" /* operationSubName */, operation1) + err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } @@ -195,7 +195,7 @@ func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstPanics(t *testing.T) { err2 := retryWithExponentialBackOff( time.Duration(initialOperationWaitTimeShort), func() (bool, error) { - err := grm.Run(volumeName, "" /* operationSubName */, operation2) + err := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {}) if err != nil { t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err) return false, nil @@ -215,7 +215,7 @@ func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstPanicsWithExpBackoff(t *tes grm := NewNestedPendingOperations(true /* exponentialBackOffOnError */) volumeName := v1.UniqueVolumeName("volume-name") operation1 := generatePanicFunc() - err1 := grm.Run(volumeName, "" /* operationSubName */, operation1) + err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } @@ -225,7 +225,7 @@ func Test_NewGoRoutineMap_Positive_SecondOpAfterFirstPanicsWithExpBackoff(t *tes err2 := retryWithExponentialBackOff( time.Duration(initialOperationWaitTimeLong), // Longer duration to accommodate for backoff func() (bool, error) { - err := grm.Run(volumeName, "" /* operationSubName */, operation2) + err := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {}) if err != nil { t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err) return false, nil @@ -246,14 +246,14 @@ func Test_NewGoRoutineMap_Negative_SecondOpBeforeFirstCompletes(t *testing.T) { volumeName := v1.UniqueVolumeName("volume-name") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateWaitFunc(operation1DoneCh) - err1 := grm.Run(volumeName, "" /* operationSubName */, operation1) + err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } operation2 := generateNoopFunc() // Act - err2 := grm.Run(volumeName, "" /* operationSubName */, operation2) + err2 := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {}) // Assert if err2 == nil { @@ -271,14 +271,14 @@ func Test_NewGoRoutineMap_Negative_SecondSubOpBeforeFirstCompletes2(t *testing.T operationPodName := types.UniquePodName("operation-podname") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateWaitFunc(operation1DoneCh) - err1 := grm.Run(volumeName, operationPodName, operation1) + err1 := grm.Run(volumeName, operationPodName, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } operation2 := generateNoopFunc() // Act - err2 := grm.Run(volumeName, operationPodName, operation2) + err2 := grm.Run(volumeName, operationPodName, operation2, func(error) {}) // Assert if err2 == nil { @@ -296,14 +296,14 @@ func Test_NewGoRoutineMap_Negative_SecondSubOpBeforeFirstCompletes(t *testing.T) operationPodName := types.UniquePodName("operation-podname") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateWaitFunc(operation1DoneCh) - err1 := grm.Run(volumeName, operationPodName, operation1) + err1 := grm.Run(volumeName, operationPodName, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } operation2 := generateNoopFunc() // Act - err2 := grm.Run(volumeName, operationPodName, operation2) + err2 := grm.Run(volumeName, operationPodName, operation2, func(error) {}) // Assert if err2 == nil { @@ -320,14 +320,14 @@ func Test_NewGoRoutineMap_Negative_SecondOpBeforeFirstCompletesWithExpBackoff(t volumeName := v1.UniqueVolumeName("volume-name") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateWaitFunc(operation1DoneCh) - err1 := grm.Run(volumeName, "" /* operationSubName */, operation1) + err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } operation2 := generateNoopFunc() // Act - err2 := grm.Run(volumeName, "" /* operationSubName */, operation2) + err2 := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {}) // Assert if err2 == nil { @@ -344,7 +344,7 @@ func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletes(t *testing.T) { volumeName := v1.UniqueVolumeName("volume-name") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateWaitFunc(operation1DoneCh) - err1 := grm.Run(volumeName, "" /* operationSubName */, operation1) + err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } @@ -352,7 +352,7 @@ func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletes(t *testing.T) { operation3 := generateNoopFunc() // Act - err2 := grm.Run(volumeName, "" /* operationSubName */, operation2) + err2 := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {}) // Assert if err2 == nil { @@ -367,7 +367,7 @@ func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletes(t *testing.T) { err3 := retryWithExponentialBackOff( time.Duration(initialOperationWaitTimeShort), func() (bool, error) { - err := grm.Run(volumeName, "" /* operationSubName */, operation3) + err := grm.Run(volumeName, "" /* operationSubName */, operation3, func(error) {}) if err != nil { t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err) return false, nil @@ -388,7 +388,7 @@ func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletesWithExpBackoff(t *t volumeName := v1.UniqueVolumeName("volume-name") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateWaitFunc(operation1DoneCh) - err1 := grm.Run(volumeName, "" /* operationSubName */, operation1) + err1 := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err1 != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err1) } @@ -396,7 +396,7 @@ func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletesWithExpBackoff(t *t operation3 := generateNoopFunc() // Act - err2 := grm.Run(volumeName, "" /* operationSubName */, operation2) + err2 := grm.Run(volumeName, "" /* operationSubName */, operation2, func(error) {}) // Assert if err2 == nil { @@ -411,7 +411,7 @@ func Test_NewGoRoutineMap_Positive_ThirdOpAfterFirstCompletesWithExpBackoff(t *t err3 := retryWithExponentialBackOff( time.Duration(initialOperationWaitTimeShort), func() (bool, error) { - err := grm.Run(volumeName, "" /* operationSubName */, operation3) + err := grm.Run(volumeName, "" /* operationSubName */, operation3, func(error) {}) if err != nil { t.Logf("Warning: NewGoRoutine failed with %v. Will retry.", err) return false, nil @@ -471,7 +471,7 @@ func Test_NewGoRoutineMap_Positive_Wait(t *testing.T) { volumeName := v1.UniqueVolumeName("volume-name") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateWaitFunc(operation1DoneCh) - err := grm.Run(volumeName, "" /* operationSubName */, operation1) + err := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err) } @@ -500,7 +500,7 @@ func Test_NewGoRoutineMap_Positive_WaitWithExpBackoff(t *testing.T) { volumeName := v1.UniqueVolumeName("volume-name") operation1DoneCh := make(chan interface{}, 0 /* bufferSize */) operation1 := generateWaitFunc(operation1DoneCh) - err := grm.Run(volumeName, "" /* operationSubName */, operation1) + err := grm.Run(volumeName, "" /* operationSubName */, operation1, func(error) {}) if err != nil { t.Fatalf("NewGoRoutine failed. Expected: Actual: <%v>", err) } diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index 02d7ab2c6aebb..21e8eecbdc740 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations" volumetypes "k8s.io/kubernetes/pkg/volume/util/types" "k8s.io/kubernetes/pkg/volume/util/volumehelper" @@ -535,29 +536,32 @@ func (oe *operationExecutor) IsOperationPending(volumeName v1.UniqueVolumeName, func (oe *operationExecutor) AttachVolume( volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) error { - attachFunc, err := + attachFunc, plugin, err := oe.operationGenerator.GenerateAttachVolumeFunc(volumeToAttach, actualStateOfWorld) if err != nil { return err } + opCompleteFunc := util.OperationCompleteHook(plugin, "volume_attach") return oe.pendingOperations.Run( - volumeToAttach.VolumeName, "" /* podName */, attachFunc) + volumeToAttach.VolumeName, "" /* podName */, attachFunc, opCompleteFunc) } func (oe *operationExecutor) DetachVolume( volumeToDetach AttachedVolume, verifySafeToDetach bool, actualStateOfWorld ActualStateOfWorldAttacherUpdater) error { - detachFunc, err := + detachFunc, plugin, err := oe.operationGenerator.GenerateDetachVolumeFunc(volumeToDetach, verifySafeToDetach, actualStateOfWorld) if err != nil { return err } + opCompleteFunc := util.OperationCompleteHook(plugin, "volume_detach") return oe.pendingOperations.Run( - volumeToDetach.VolumeName, "" /* podName */, detachFunc) + volumeToDetach.VolumeName, "" /* podName */, detachFunc, opCompleteFunc) } + func (oe *operationExecutor) VerifyVolumesAreAttached( attachedVolumes map[types.NodeName][]AttachedVolume, actualStateOfWorld ActualStateOfWorldAttacherUpdater) { @@ -630,9 +634,11 @@ func (oe *operationExecutor) VerifyVolumesAreAttached( if err != nil { glog.Errorf("BulkVerifyVolumes.GenerateBulkVolumeVerifyFunc error bulk verifying volumes for plugin %q with %v", pluginName, err) } + + opCompleteFunc := util.OperationCompleteHook(pluginName, "verify_volumes_are_attached") // Ugly hack to ensure - we don't do parallel bulk polling of same volume plugin uniquePluginName := v1.UniqueVolumeName(pluginName) - err = oe.pendingOperations.Run(uniquePluginName, "" /* Pod Name */, bulkVerifyVolumeFunc) + err = oe.pendingOperations.Run(uniquePluginName, "" /* Pod Name */, bulkVerifyVolumeFunc, opCompleteFunc) if err != nil { glog.Errorf("BulkVerifyVolumes.Run Error bulk volume verification for plugin %q with %v", pluginName, err) } @@ -648,8 +654,10 @@ func (oe *operationExecutor) VerifyVolumesAreAttachedPerNode( if err != nil { return err } + + opCompleteFunc := util.OperationCompleteHook("", "verify_volumes_are_attached_per_node") // Give an empty UniqueVolumeName so that this operation could be executed concurrently. - return oe.pendingOperations.Run("" /* volumeName */, "" /* podName */, volumesAreAttachedFunc) + return oe.pendingOperations.Run("" /* volumeName */, "" /* podName */, volumesAreAttachedFunc, opCompleteFunc) } func (oe *operationExecutor) MountVolume( @@ -657,7 +665,7 @@ func (oe *operationExecutor) MountVolume( volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, isRemount bool) error { - mountFunc, err := oe.operationGenerator.GenerateMountVolumeFunc( + mountFunc, plugin, err := oe.operationGenerator.GenerateMountVolumeFunc( waitForAttachTimeout, volumeToMount, actualStateOfWorld, isRemount) if err != nil { return err @@ -671,15 +679,17 @@ func (oe *operationExecutor) MountVolume( podName = volumehelper.GetUniquePodName(volumeToMount.Pod) } + // TODO mount_device + opCompleteFunc := util.OperationCompleteHook(plugin, "volume_mount") return oe.pendingOperations.Run( - volumeToMount.VolumeName, podName, mountFunc) + volumeToMount.VolumeName, podName, mountFunc, opCompleteFunc) } func (oe *operationExecutor) UnmountVolume( volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error { - unmountFunc, err := + unmountFunc, plugin, err := oe.operationGenerator.GenerateUnmountVolumeFunc(volumeToUnmount, actualStateOfWorld) if err != nil { return err @@ -689,36 +699,39 @@ func (oe *operationExecutor) UnmountVolume( // same volume in parallel podName := volumetypes.UniquePodName(volumeToUnmount.PodUID) + opCompleteFunc := util.OperationCompleteHook(plugin, "volume_unmount") return oe.pendingOperations.Run( - volumeToUnmount.VolumeName, podName, unmountFunc) + volumeToUnmount.VolumeName, podName, unmountFunc, opCompleteFunc) } func (oe *operationExecutor) UnmountDevice( deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error { - unmountDeviceFunc, err := + unmountDeviceFunc, plugin, err := oe.operationGenerator.GenerateUnmountDeviceFunc(deviceToDetach, actualStateOfWorld, mounter) if err != nil { return err } + opCompleteFunc := util.OperationCompleteHook(plugin, "unmount_device") return oe.pendingOperations.Run( - deviceToDetach.VolumeName, "" /* podName */, unmountDeviceFunc) + deviceToDetach.VolumeName, "" /* podName */, unmountDeviceFunc, opCompleteFunc) } func (oe *operationExecutor) VerifyControllerAttachedVolume( volumeToMount VolumeToMount, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) error { - verifyControllerAttachedVolumeFunc, err := + verifyControllerAttachedVolumeFunc, plugin, err := oe.operationGenerator.GenerateVerifyControllerAttachedVolumeFunc(volumeToMount, nodeName, actualStateOfWorld) if err != nil { return err } + opCompleteFunc := util.OperationCompleteHook(plugin, "verify_controller_attached_volume") return oe.pendingOperations.Run( - volumeToMount.VolumeName, "" /* podName */, verifyControllerAttachedVolumeFunc) + volumeToMount.VolumeName, "" /* podName */, verifyControllerAttachedVolumeFunc, opCompleteFunc) } // TODO: this is a workaround for the unmount device issue caused by gci mounter. diff --git a/pkg/volume/util/operationexecutor/operation_executor_test.go b/pkg/volume/util/operationexecutor/operation_executor_test.go index 700b82b79db91..94118674ecc28 100644 --- a/pkg/volume/util/operationexecutor/operation_executor_test.go +++ b/pkg/volume/util/operationexecutor/operation_executor_test.go @@ -239,29 +239,29 @@ func newFakeOperationGenerator(ch chan interface{}, quit chan interface{}) Opera } } -func (fopg *fakeOperationGenerator) GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (func() error, error) { +func (fopg *fakeOperationGenerator) GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (func() error, string, error) { return func() error { startOperationAndBlock(fopg.ch, fopg.quit) return nil - }, nil + }, "", nil } -func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, error) { +func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) { return func() error { startOperationAndBlock(fopg.ch, fopg.quit) return nil - }, nil + }, "", nil } -func (fopg *fakeOperationGenerator) GenerateAttachVolumeFunc(volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) { +func (fopg *fakeOperationGenerator) GenerateAttachVolumeFunc(volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) { return func() error { startOperationAndBlock(fopg.ch, fopg.quit) return nil - }, nil + }, "", nil } -func (fopg *fakeOperationGenerator) GenerateDetachVolumeFunc(volumeToDetach AttachedVolume, verifySafeToDetach bool, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) { +func (fopg *fakeOperationGenerator) GenerateDetachVolumeFunc(volumeToDetach AttachedVolume, verifySafeToDetach bool, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) { return func() error { startOperationAndBlock(fopg.ch, fopg.quit) return nil - }, nil + }, "", nil } func (fopg *fakeOperationGenerator) GenerateVolumesAreAttachedFunc(attachedVolumes []AttachedVolume, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) { return func() error { @@ -269,17 +269,17 @@ func (fopg *fakeOperationGenerator) GenerateVolumesAreAttachedFunc(attachedVolum return nil }, nil } -func (fopg *fakeOperationGenerator) GenerateUnmountDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, error) { +func (fopg *fakeOperationGenerator) GenerateUnmountDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, string, error) { return func() error { startOperationAndBlock(fopg.ch, fopg.quit) return nil - }, nil + }, "", nil } -func (fopg *fakeOperationGenerator) GenerateVerifyControllerAttachedVolumeFunc(volumeToMount VolumeToMount, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) { +func (fopg *fakeOperationGenerator) GenerateVerifyControllerAttachedVolumeFunc(volumeToMount VolumeToMount, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) { return func() error { startOperationAndBlock(fopg.ch, fopg.quit) return nil - }, nil + }, "", nil } func (fopg *fakeOperationGenerator) GenerateBulkVolumeVerifyFunc( diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index cceabde48cff4..73445ac3a0f53 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -73,25 +73,25 @@ func NewOperationGenerator(kubeClient clientset.Interface, // OperationGenerator interface that extracts out the functions from operation_executor to make it dependency injectable type OperationGenerator interface { // Generates the MountVolume function needed to perform the mount of a volume plugin - GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (func() error, error) + GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (func() error, string, error) // Generates the UnmountVolume function needed to perform the unmount of a volume plugin - GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, error) + GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) // Generates the AttachVolume function needed to perform attach of a volume plugin - GenerateAttachVolumeFunc(volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) + GenerateAttachVolumeFunc(volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) // Generates the DetachVolume function needed to perform the detach of a volume plugin - GenerateDetachVolumeFunc(volumeToDetach AttachedVolume, verifySafeToDetach bool, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) + GenerateDetachVolumeFunc(volumeToDetach AttachedVolume, verifySafeToDetach bool, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) // Generates the VolumesAreAttached function needed to verify if volume plugins are attached GenerateVolumesAreAttachedFunc(attachedVolumes []AttachedVolume, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) // Generates the UnMountDevice function needed to perform the unmount of a device - GenerateUnmountDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, error) + GenerateUnmountDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, string, error) // Generates the function needed to check if the attach_detach controller has attached the volume plugin - GenerateVerifyControllerAttachedVolumeFunc(volumeToMount VolumeToMount, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) + GenerateVerifyControllerAttachedVolumeFunc(volumeToMount VolumeToMount, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) // GetVolumePluginMgr returns volume plugin manager GetVolumePluginMgr() *volume.VolumePluginMgr @@ -245,17 +245,17 @@ func (og *operationGenerator) GenerateBulkVolumeVerifyFunc( func (og *operationGenerator) GenerateAttachVolumeFunc( volumeToAttach VolumeToAttach, - actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) { + actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) { // Get attacher plugin attachableVolumePlugin, err := og.volumePluginMgr.FindAttachablePluginBySpec(volumeToAttach.VolumeSpec) if err != nil || attachableVolumePlugin == nil { - return nil, volumeToAttach.GenerateErrorDetailed("AttachVolume.FindAttachablePluginBySpec failed", err) + return nil, "", volumeToAttach.GenerateErrorDetailed("AttachVolume.FindAttachablePluginBySpec failed", err) } volumeAttacher, newAttacherErr := attachableVolumePlugin.NewAttacher() if newAttacherErr != nil { - return nil, volumeToAttach.GenerateErrorDetailed("AttachVolume.NewAttacher failed", newAttacherErr) + return nil, attachableVolumePlugin.GetPluginName(), volumeToAttach.GenerateErrorDetailed("AttachVolume.NewAttacher failed", newAttacherErr) } return func() error { @@ -283,7 +283,7 @@ func (og *operationGenerator) GenerateAttachVolumeFunc( } return nil - }, nil + }, attachableVolumePlugin.GetPluginName(), nil } func (og *operationGenerator) GetVolumePluginMgr() *volume.VolumePluginMgr { @@ -293,9 +293,10 @@ func (og *operationGenerator) GetVolumePluginMgr() *volume.VolumePluginMgr { func (og *operationGenerator) GenerateDetachVolumeFunc( volumeToDetach AttachedVolume, verifySafeToDetach bool, - actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) { + actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) { var volumeName string var attachableVolumePlugin volume.AttachableVolumePlugin + var pluginName string var err error if volumeToDetach.VolumeSpec != nil { @@ -303,31 +304,35 @@ func (og *operationGenerator) GenerateDetachVolumeFunc( attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginBySpec(volumeToDetach.VolumeSpec) if err != nil || attachableVolumePlugin == nil { - return nil, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginBySpec failed", err) + return nil, "", volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginBySpec failed", err) } volumeName, err = attachableVolumePlugin.GetVolumeName(volumeToDetach.VolumeSpec) if err != nil { - return nil, volumeToDetach.GenerateErrorDetailed("DetachVolume.GetVolumeName failed", err) + return nil, attachableVolumePlugin.GetPluginName(), volumeToDetach.GenerateErrorDetailed("DetachVolume.GetVolumeName failed", err) } } else { - var pluginName string // Get attacher plugin and the volumeName by splitting the volume unique name in case // there's no VolumeSpec: this happens only on attach/detach controller crash recovery // when a pod has been deleted during the controller downtime pluginName, volumeName, err = volumehelper.SplitUniqueName(volumeToDetach.VolumeName) if err != nil { - return nil, volumeToDetach.GenerateErrorDetailed("DetachVolume.SplitUniqueName failed", err) + return nil, pluginName, volumeToDetach.GenerateErrorDetailed("DetachVolume.SplitUniqueName failed", err) } attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(pluginName) if err != nil { - return nil, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginBySpec failed", err) + return nil, pluginName, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginBySpec failed", err) } } + + if pluginName == "" { + pluginName = attachableVolumePlugin.GetPluginName() + } + volumeDetacher, err := attachableVolumePlugin.NewDetacher() if err != nil { - return nil, volumeToDetach.GenerateErrorDetailed("DetachVolume.NewDetacher failed", err) + return nil, pluginName, volumeToDetach.GenerateErrorDetailed("DetachVolume.NewDetacher failed", err) } return func() error { @@ -352,24 +357,24 @@ func (og *operationGenerator) GenerateDetachVolumeFunc( volumeToDetach.VolumeName, volumeToDetach.NodeName) return nil - }, nil + }, pluginName, nil } func (og *operationGenerator) GenerateMountVolumeFunc( waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, - isRemount bool) (func() error, error) { + isRemount bool) (func() error, string, error) { // Get mounter plugin volumePlugin, err := og.volumePluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec) if err != nil || volumePlugin == nil { - return nil, volumeToMount.GenerateErrorDetailed("MountVolume.FindPluginBySpec failed", err) + return nil, "", volumeToMount.GenerateErrorDetailed("MountVolume.FindPluginBySpec failed", err) } affinityErr := checkNodeAffinity(og, volumeToMount, volumePlugin) if affinityErr != nil { - return nil, affinityErr + return nil, volumePlugin.GetPluginName(), affinityErr } volumeMounter, newMounterErr := volumePlugin.NewMounter( @@ -379,13 +384,13 @@ func (og *operationGenerator) GenerateMountVolumeFunc( if newMounterErr != nil { eventErr, detailedErr := volumeToMount.GenerateError("MountVolume.NewMounter initialization failed", newMounterErr) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMountVolume, eventErr.Error()) - return nil, detailedErr + return nil, volumePlugin.GetPluginName(), detailedErr } mountCheckError := checkMountOptionSupport(og, volumeToMount, volumePlugin) if mountCheckError != nil { - return nil, mountCheckError + return nil, volumePlugin.GetPluginName(), mountCheckError } // Get attacher, if possible @@ -408,7 +413,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) devicePath, err := volumeAttacher.WaitForAttach( - volumeToMount.VolumeSpec, volumeToMount.DevicePath, waitForAttachTimeout) + volumeToMount.VolumeSpec, volumeToMount.DevicePath, volumeToMount.Pod, waitForAttachTimeout) if err != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateErrorDetailed("MountVolume.WaitForAttach failed", err) @@ -489,23 +494,23 @@ func (og *operationGenerator) GenerateMountVolumeFunc( } return nil - }, nil + }, volumePlugin.GetPluginName(), nil } func (og *operationGenerator) GenerateUnmountVolumeFunc( volumeToUnmount MountedVolume, - actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, error) { + actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) { // Get mountable plugin volumePlugin, err := og.volumePluginMgr.FindPluginByName(volumeToUnmount.PluginName) if err != nil || volumePlugin == nil { - return nil, volumeToUnmount.GenerateErrorDetailed("UnmountVolume.FindPluginByName failed", err) + return nil, "", volumeToUnmount.GenerateErrorDetailed("UnmountVolume.FindPluginByName failed", err) } volumeUnmounter, newUnmounterErr := volumePlugin.NewUnmounter( volumeToUnmount.InnerVolumeSpecName, volumeToUnmount.PodUID) if newUnmounterErr != nil { - return nil, volumeToUnmount.GenerateErrorDetailed("UnmountVolume.NewUnmounter failed", newUnmounterErr) + return nil, volumePlugin.GetPluginName(), volumeToUnmount.GenerateErrorDetailed("UnmountVolume.NewUnmounter failed", newUnmounterErr) } return func() error { @@ -535,28 +540,28 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc( } return nil - }, nil + }, volumePlugin.GetPluginName(), nil } func (og *operationGenerator) GenerateUnmountDeviceFunc( deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, - mounter mount.Interface) (func() error, error) { + mounter mount.Interface) (func() error, string, error) { // Get attacher plugin attachableVolumePlugin, err := og.volumePluginMgr.FindAttachablePluginBySpec(deviceToDetach.VolumeSpec) if err != nil || attachableVolumePlugin == nil { - return nil, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindAttachablePluginBySpec failed", err) + return nil, "", deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindAttachablePluginBySpec failed", err) } volumeDetacher, err := attachableVolumePlugin.NewDetacher() if err != nil { - return nil, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDetacher failed", err) + return nil, attachableVolumePlugin.GetPluginName(), deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDetacher failed", err) } volumeAttacher, err := attachableVolumePlugin.NewAttacher() if err != nil { - return nil, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewAttacher failed", err) + return nil, attachableVolumePlugin.GetPluginName(), deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewAttacher failed", err) } return func() error { @@ -616,13 +621,19 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( } return nil - }, nil + }, attachableVolumePlugin.GetPluginName(), nil } func (og *operationGenerator) GenerateVerifyControllerAttachedVolumeFunc( volumeToMount VolumeToMount, nodeName types.NodeName, - actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) { + actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) { + volumePlugin, err := + og.volumePluginMgr.FindPluginBySpec(volumeToMount.VolumeSpec) + if err != nil || volumePlugin == nil { + return nil, "", volumeToMount.GenerateErrorDetailed("VerifyControllerAttachedVolume.FindPluginBySpec failed", err) + } + return func() error { if !volumeToMount.PluginIsAttachable { // If the volume does not implement the attacher interface, it is @@ -678,7 +689,7 @@ func (og *operationGenerator) GenerateVerifyControllerAttachedVolumeFunc( // Volume not attached, return error. Caller will log and retry. return volumeToMount.GenerateErrorDetailed("Volume not attached according to node status", nil) - }, nil + }, volumePlugin.GetPluginName(), nil } func (og *operationGenerator) verifyVolumeIsSafeToDetach( diff --git a/pkg/volume/util/volumehelper/BUILD b/pkg/volume/util/volumehelper/BUILD index 1880d21c05853..849d3ee4e2c14 100644 --- a/pkg/volume/util/volumehelper/BUILD +++ b/pkg/volume/util/volumehelper/BUILD @@ -9,6 +9,7 @@ go_library( name = "go_default_library", srcs = ["volumehelper.go"], deps = [ + "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util/types:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/volume/util/volumehelper/volumehelper.go b/pkg/volume/util/volumehelper/volumehelper.go index bcde93f71ed89..b0734601d0af7 100644 --- a/pkg/volume/util/volumehelper/volumehelper.go +++ b/pkg/volume/util/volumehelper/volumehelper.go @@ -23,6 +23,7 @@ import ( "strings" "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/types" ) @@ -127,3 +128,11 @@ func SplitUniqueName(uniqueName v1.UniqueVolumeName) (string, string, error) { pluginName := fmt.Sprintf("%s/%s", components[0], components[1]) return pluginName, components[2], nil } + +// NewSafeFormatAndMountFromHost creates a new SafeFormatAndMount with Mounter +// and Exec taken from given VolumeHost. +func NewSafeFormatAndMountFromHost(pluginName string, host volume.VolumeHost) *mount.SafeFormatAndMount { + mounter := host.GetMounter(pluginName) + exec := host.GetExec(pluginName) + return &mount.SafeFormatAndMount{Interface: mounter, Exec: exec} +} diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index 899cf8aac7fc5..78cb21a32a74e 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -173,7 +173,7 @@ type Attacher interface { // node. If it successfully attaches, the path to the device // is returned. Otherwise, if the device does not attach after // the given timeout period, an error will be returned. - WaitForAttach(spec *Spec, devicePath string, timeout time.Duration) (string, error) + WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) // GetDeviceMountPath returns a path where the device should // be mounted after it is attached. This is a global mount diff --git a/pkg/volume/volume_linux.go b/pkg/volume/volume_linux.go index ef1f45208c95f..d67ee4a95ac53 100644 --- a/pkg/volume/volume_linux.go +++ b/pkg/volume/volume_linux.go @@ -89,3 +89,17 @@ func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error { return nil }) } + +// IsSameFSGroup is called only for requests to mount an already mounted +// volume. It checks if fsGroup of new mount request is the same or not. +// It returns false if it not the same. It also returns current Gid of a path +// provided for dir variable. +func IsSameFSGroup(dir string, fsGroup int64) (bool, int, error) { + info, err := os.Stat(dir) + if err != nil { + glog.Errorf("Error getting stats for %s (%v)", dir, err) + return false, 0, err + } + s := info.Sys().(*syscall.Stat_t) + return int(s.Gid) == int(fsGroup), int(s.Gid), nil +} diff --git a/pkg/volume/volume_unsupported.go b/pkg/volume/volume_unsupported.go index 45a6cc5ca7faf..46a6aeaf0a938 100644 --- a/pkg/volume/volume_unsupported.go +++ b/pkg/volume/volume_unsupported.go @@ -21,3 +21,7 @@ package volume func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error { return nil } + +func IsSameFSGroup(dir string, fsGroup int64) (bool, int, error) { + return true, int(fsGroup), nil +} diff --git a/pkg/volume/vsphere_volume/BUILD b/pkg/volume/vsphere_volume/BUILD index e118362772b60..9f5a9ac63a6e9 100644 --- a/pkg/volume/vsphere_volume/BUILD +++ b/pkg/volume/vsphere_volume/BUILD @@ -28,7 +28,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/pkg/volume/vsphere_volume/attacher.go b/pkg/volume/vsphere_volume/attacher.go index f7a58c80d4e39..19edc58ca0429 100644 --- a/pkg/volume/vsphere_volume/attacher.go +++ b/pkg/volume/vsphere_volume/attacher.go @@ -23,13 +23,14 @@ import ( "time" "github.com/golang/glog" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere" "k8s.io/kubernetes/pkg/util/keymutex" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" - "k8s.io/utils/exec" + "k8s.io/kubernetes/pkg/volume/util/volumehelper" ) type vsphereVMDKAttacher struct { @@ -118,7 +119,7 @@ func (attacher *vsphereVMDKAttacher) VolumesAreAttached(specs []*volume.Spec, no return volumesAttachedCheck, nil } -func (attacher *vsphereVMDKAttacher) WaitForAttach(spec *volume.Spec, devicePath string, timeout time.Duration) (string, error) { +func (attacher *vsphereVMDKAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) { volumeSource, _, err := getVolumeSource(spec) if err != nil { return "", err @@ -167,13 +168,13 @@ func (attacher *vsphereVMDKAttacher) GetDeviceMountPath(spec *volume.Spec) (stri // GetMountDeviceRefs finds all other references to the device referenced // by deviceMountPath; returns a list of paths. func (plugin *vsphereVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mount.GetMountRefs(mounter, deviceMountPath) } // MountDevice mounts device to global mount point. func (attacher *vsphereVMDKAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { - mounter := attacher.host.GetMounter() + mounter := attacher.host.GetMounter(vsphereVolumePluginName) notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) if err != nil { if os.IsNotExist(err) { @@ -195,7 +196,7 @@ func (attacher *vsphereVMDKAttacher) MountDevice(spec *volume.Spec, devicePath s options := []string{} if notMnt { - diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()} + diskMounter := volumehelper.NewSafeFormatAndMountFromHost(vsphereVolumePluginName, attacher.host) mountOptions := volume.MountOptionFromSpec(spec, options...) err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions) if err != nil { @@ -221,7 +222,7 @@ func (plugin *vsphereVolumePlugin) NewDetacher() (volume.Detacher, error) { } return &vsphereVMDKDetacher{ - mounter: plugin.host.GetMounter(), + mounter: plugin.host.GetMounter(plugin.GetPluginName()), vsphereVolumes: vsphereCloud, }, nil } diff --git a/pkg/volume/vsphere_volume/vsphere_volume.go b/pkg/volume/vsphere_volume/vsphere_volume.go index 0871055d33b1d..92322fa2ace34 100644 --- a/pkg/volume/vsphere_volume/vsphere_volume.go +++ b/pkg/volume/vsphere_volume/vsphere_volume.go @@ -32,7 +32,6 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/volumehelper" - "k8s.io/utils/exec" ) // This is the primary entrypoint for volume plugins. @@ -90,11 +89,11 @@ func (plugin *vsphereVolumePlugin) SupportsBulkVolumeVerification() bool { } func (plugin *vsphereVolumePlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - return plugin.newMounterInternal(spec, pod.UID, &VsphereDiskUtil{}, plugin.host.GetMounter()) + return plugin.newMounterInternal(spec, pod.UID, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *vsphereVolumePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { - return plugin.newUnmounterInternal(volName, podUID, &VsphereDiskUtil{}, plugin.host.GetMounter()) + return plugin.newUnmounterInternal(volName, podUID, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) } func (plugin *vsphereVolumePlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.Mounter, error) { @@ -116,7 +115,7 @@ func (plugin *vsphereVolumePlugin) newMounterInternal(spec *volume.Spec, podUID plugin: plugin, }, fsType: fsType, - diskMounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()}}, nil + diskMounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host)}, nil } func (plugin *vsphereVolumePlugin) newUnmounterInternal(volName string, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.Unmounter, error) { @@ -131,7 +130,7 @@ func (plugin *vsphereVolumePlugin) newUnmounterInternal(volName string, podUID t } func (plugin *vsphereVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { - mounter := plugin.host.GetMounter() + mounter := plugin.host.GetMounter(plugin.GetPluginName()) pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName()) volumePath, err := mounter.GetDeviceNameFromMount(mountPath, pluginDir) if err != nil { diff --git a/pkg/volume/vsphere_volume/vsphere_volume_test.go b/pkg/volume/vsphere_volume/vsphere_volume_test.go index 992bc612750bf..282cb23d60271 100644 --- a/pkg/volume/vsphere_volume/vsphere_volume_test.go +++ b/pkg/volume/vsphere_volume/vsphere_volume_test.go @@ -37,7 +37,7 @@ func TestCanSupport(t *testing.T) { } defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/vsphere-volume") if err != nil { @@ -90,7 +90,7 @@ func TestPlugin(t *testing.T) { defer os.RemoveAll(tmpDir) plugMgr := volume.VolumePluginMgr{} - plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) + plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil)) plug, err := plugMgr.FindPluginByName("kubernetes.io/vsphere-volume") if err != nil { diff --git a/plugin/pkg/admission/initialization/BUILD b/plugin/pkg/admission/initialization/BUILD index 54529cc258b1a..05ecb5abaaaee 100644 --- a/plugin/pkg/admission/initialization/BUILD +++ b/plugin/pkg/admission/initialization/BUILD @@ -23,6 +23,8 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + "//vendor/k8s.io/apiserver/pkg/features:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", ], ) @@ -46,6 +48,12 @@ go_test( library = ":go_default_library", deps = [ "//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", ], ) diff --git a/plugin/pkg/admission/initialization/initialization.go b/plugin/pkg/admission/initialization/initialization.go index 447753c47b3bd..8c56f51e3b3f8 100644 --- a/plugin/pkg/admission/initialization/initialization.go +++ b/plugin/pkg/admission/initialization/initialization.go @@ -33,6 +33,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/kubeapiserver/admission/configuration" @@ -71,6 +73,18 @@ func (i *initializer) Validate() error { if i.config == nil { return fmt.Errorf("the Initializer admission plugin requires a Kubernetes client to be provided") } + if i.authorizer == nil { + return fmt.Errorf("the Initializer admission plugin requires an authorizer to be provided") + } + + if !utilfeature.DefaultFeatureGate.Enabled(features.Initializers) { + if err := utilfeature.DefaultFeatureGate.Set(string(features.Initializers) + "=true"); err != nil { + glog.Errorf("error enabling Initializers feature as part of admission plugin setup: %v", err) + } else { + glog.Infof("enabled Initializers feature as part of admission plugin setup") + } + } + i.config.Run(wait.NeverStop) return nil } @@ -208,6 +222,9 @@ func (i *initializer) Admit(a admission.Attributes) (err error) { glog.V(5).Infof("Modifying uninitialized resource %s", a.GetResource()) + if updated != nil && len(updated.Pending) == 0 && updated.Result == nil { + accessor.SetInitializers(nil) + } // because we are called before validation, we need to ensure the update transition is valid. if errs := validation.ValidateInitializersUpdate(updated, existing, initializerFieldPath); len(errs) > 0 { return errors.NewInvalid(a.GetKind().GroupKind(), a.GetName(), errs) @@ -225,11 +242,6 @@ func (i *initializer) Admit(a admission.Attributes) (err error) { } func (i *initializer) canInitialize(a admission.Attributes, message string) error { - // if no authorizer is present, the initializer plugin allows modification of uninitialized resources - if i.authorizer == nil { - glog.V(4).Infof("No authorizer provided to initialization admission control, unable to check permissions") - return nil - } // caller must have the ability to mutate un-initialized resources authorized, reason, err := i.authorizer.Authorize(authorizer.AttributesRecord{ Name: a.GetName(), diff --git a/plugin/pkg/admission/initialization/initialization_test.go b/plugin/pkg/admission/initialization/initialization_test.go index 8f871993919cf..cd8728379f25d 100644 --- a/plugin/pkg/admission/initialization/initialization_test.go +++ b/plugin/pkg/admission/initialization/initialization_test.go @@ -18,10 +18,17 @@ package initialization import ( "reflect" + "strings" "testing" "k8s.io/api/admissionregistration/v1alpha1" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/authorization/authorizer" ) func newInitializer(name string, rules ...v1alpha1.Rule) *v1alpha1.InitializerConfiguration { @@ -97,3 +104,75 @@ func TestFindInitializers(t *testing.T) { }) } } + +type fakeAuthorizer struct { + accept bool +} + +func (f *fakeAuthorizer) Authorize(a authorizer.Attributes) (bool, string, error) { + if f.accept { + return true, "", nil + } + return false, "denied", nil +} + +func TestAdmitUpdate(t *testing.T) { + tests := []struct { + name string + oldInitializers *metav1.Initializers + newInitializers *metav1.Initializers + verifyUpdatedObj func(runtime.Object) (pass bool, reason string) + err string + }{ + { + name: "updates on initialized resources are allowed", + oldInitializers: nil, + newInitializers: nil, + err: "", + }, + { + name: "updates on initialized resources are allowed", + oldInitializers: &metav1.Initializers{Pending: []metav1.Initializer{{Name: "init.k8s.io"}}}, + newInitializers: &metav1.Initializers{}, + verifyUpdatedObj: func(obj runtime.Object) (bool, string) { + accessor, err := meta.Accessor(obj) + if err != nil { + return false, "cannot get accessor" + } + if accessor.GetInitializers() != nil { + return false, "expect nil initializers" + } + return true, "" + }, + err: "", + }, + { + name: "initializers may not be set once initialized", + oldInitializers: nil, + newInitializers: &metav1.Initializers{Pending: []metav1.Initializer{{Name: "init.k8s.io"}}}, + err: "field is immutable once initialization has completed", + }, + } + + plugin := initializer{ + config: nil, + authorizer: &fakeAuthorizer{true}, + } + for _, tc := range tests { + oldObj := &v1.Pod{} + oldObj.Initializers = tc.oldInitializers + newObj := &v1.Pod{} + newObj.Initializers = tc.newInitializers + a := admission.NewAttributesRecord(newObj, oldObj, schema.GroupVersionKind{}, "", "foo", schema.GroupVersionResource{}, "", admission.Update, nil) + err := plugin.Admit(a) + switch { + case tc.err == "" && err != nil: + t.Errorf("%q: unexpected error: %v", tc.name, err) + case tc.err != "" && err == nil: + t.Errorf("%q: unexpected no error, expected %s", tc.name, tc.err) + case tc.err != "" && err != nil && !strings.Contains(err.Error(), tc.err): + t.Errorf("%q: expected %s, got %v", tc.name, tc.err, err) + } + } + +} diff --git a/plugin/pkg/admission/podnodeselector/BUILD b/plugin/pkg/admission/podnodeselector/BUILD index e89110ebb087a..3d589c9332d22 100644 --- a/plugin/pkg/admission/podnodeselector/BUILD +++ b/plugin/pkg/admission/podnodeselector/BUILD @@ -15,6 +15,7 @@ go_library( "//pkg/client/informers/informers_generated/internalversion:go_default_library", "//pkg/client/listers/core/internalversion:go_default_library", "//pkg/kubeapiserver/admission:go_default_library", + "//pkg/kubeapiserver/admission/util:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/plugin/pkg/admission/podnodeselector/admission.go b/plugin/pkg/admission/podnodeselector/admission.go index 8e00f0b825d9a..469286f1e9b6a 100644 --- a/plugin/pkg/admission/podnodeselector/admission.go +++ b/plugin/pkg/admission/podnodeselector/admission.go @@ -33,6 +33,7 @@ import ( informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" corelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" + "k8s.io/kubernetes/pkg/kubeapiserver/admission/util" ) // The annotation key scheduler.alpha.kubernetes.io/node-selector is for assigning @@ -112,11 +113,20 @@ func (p *podNodeSelector) Admit(a admission.Attributes) error { return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request")) } + updateInitialized, err := util.IsUpdatingInitializedObject(a) + if err != nil { + return err + } + if updateInitialized { + // node selector of an initialized pod is immutable + return nil + } + name := pod.Name nsName := a.GetNamespace() var namespace *api.Namespace - namespace, err := p.namespaceLister.Get(nsName) + namespace, err = p.namespaceLister.Get(nsName) if errors.IsNotFound(err) { namespace, err = p.defaultGetNamespace(nsName) if err != nil { @@ -158,7 +168,7 @@ func (p *podNodeSelector) Admit(a admission.Attributes) error { func NewPodNodeSelector(clusterNodeSelectors map[string]string) *podNodeSelector { return &podNodeSelector{ - Handler: admission.NewHandler(admission.Create), + Handler: admission.NewHandler(admission.Create, admission.Update), clusterNodeSelectors: clusterNodeSelectors, } } diff --git a/plugin/pkg/admission/podnodeselector/admission_test.go b/plugin/pkg/admission/podnodeselector/admission_test.go index f5bc4ac969e2d..0075468a89a28 100644 --- a/plugin/pkg/admission/podnodeselector/admission_test.go +++ b/plugin/pkg/admission/podnodeselector/admission_test.go @@ -52,6 +52,12 @@ func TestPodAdmission(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "testPod", Namespace: "testNamespace"}, } + oldPod := *pod + oldPod.Initializers = &metav1.Initializers{Pending: []metav1.Initializer{{Name: "init"}}} + oldPod.Spec.NodeSelector = map[string]string{ + "old": "true", + } + tests := []struct { defaultNodeSelector string namespaceNodeSelector string @@ -166,7 +172,17 @@ func TestPodAdmission(t *testing.T) { } else if !test.admit && err == nil { t.Errorf("Test: %s, expected an error", test.testName) } + if test.admit && !labels.Equals(test.mergedNodeSelector, labels.Set(pod.Spec.NodeSelector)) { + t.Errorf("Test: %s, expected: %s but got: %s", test.testName, test.mergedNodeSelector, pod.Spec.NodeSelector) + } + // handles update of uninitialized pod like it's newly created. + err = handler.Admit(admission.NewAttributesRecord(pod, &oldPod, api.Kind("Pod").WithVersion("version"), "testNamespace", namespace.ObjectMeta.Name, api.Resource("pods").WithVersion("version"), "", admission.Update, nil)) + if test.admit && err != nil { + t.Errorf("Test: %s, expected no error but got: %s", test.testName, err) + } else if !test.admit && err == nil { + t.Errorf("Test: %s, expected an error", test.testName) + } if test.admit && !labels.Equals(test.mergedNodeSelector, labels.Set(pod.Spec.NodeSelector)) { t.Errorf("Test: %s, expected: %s but got: %s", test.testName, test.mergedNodeSelector, pod.Spec.NodeSelector) } @@ -176,7 +192,7 @@ func TestPodAdmission(t *testing.T) { func TestHandles(t *testing.T) { for op, shouldHandle := range map[admission.Operation]bool{ admission.Create: true, - admission.Update: false, + admission.Update: true, admission.Connect: false, admission.Delete: false, } { @@ -187,6 +203,40 @@ func TestHandles(t *testing.T) { } } +func TestIgnoreUpdatingInitializedPod(t *testing.T) { + mockClient := &fake.Clientset{} + handler, informerFactory, err := newHandlerForTest(mockClient) + if err != nil { + t.Errorf("unexpected error initializing handler: %v", err) + } + handler.SetReadyFunc(func() bool { return true }) + + podNodeSelector := map[string]string{"infra": "false"} + pod := &api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "testPod", Namespace: "testNamespace"}, + Spec: api.PodSpec{NodeSelector: podNodeSelector}, + } + // this conflicts with podNodeSelector + namespaceNodeSelector := "infra=true" + namespace := &api.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testNamespace", + Namespace: "", + Annotations: map[string]string{"scheduler.alpha.kubernetes.io/node-selector": namespaceNodeSelector}, + }, + } + err = informerFactory.Core().InternalVersion().Namespaces().Informer().GetStore().Update(namespace) + if err != nil { + t.Fatal(err) + } + + // if the update of initialized pod is not ignored, an error will be returned because the pod's nodeSelector conflicts with namespace's nodeSelector. + err = handler.Admit(admission.NewAttributesRecord(pod, pod, api.Kind("Pod").WithVersion("version"), "testNamespace", namespace.ObjectMeta.Name, api.Resource("pods").WithVersion("version"), "", admission.Update, nil)) + if err != nil { + t.Errorf("expected no error, got: %v", err) + } +} + // newHandlerForTest returns the admission controller configured for testing. func newHandlerForTest(c clientset.Interface) (*podNodeSelector, informers.SharedInformerFactory, error) { f := informers.NewSharedInformerFactory(c, 5*time.Minute) diff --git a/plugin/pkg/admission/podtolerationrestriction/BUILD b/plugin/pkg/admission/podtolerationrestriction/BUILD index f8df1e2786b64..929165c92bf30 100644 --- a/plugin/pkg/admission/podtolerationrestriction/BUILD +++ b/plugin/pkg/admission/podtolerationrestriction/BUILD @@ -40,6 +40,7 @@ go_library( "//pkg/client/informers/informers_generated/internalversion:go_default_library", "//pkg/client/listers/core/internalversion:go_default_library", "//pkg/kubeapiserver/admission:go_default_library", + "//pkg/kubeapiserver/admission/util:go_default_library", "//pkg/util/tolerations:go_default_library", "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction:go_default_library", "//plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction/install:go_default_library", diff --git a/plugin/pkg/admission/podtolerationrestriction/admission.go b/plugin/pkg/admission/podtolerationrestriction/admission.go index a71f92ba4a121..655a2e365a021 100644 --- a/plugin/pkg/admission/podtolerationrestriction/admission.go +++ b/plugin/pkg/admission/podtolerationrestriction/admission.go @@ -34,6 +34,7 @@ import ( informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" corelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" + "k8s.io/kubernetes/pkg/kubeapiserver/admission/util" "k8s.io/kubernetes/pkg/util/tolerations" pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" @@ -112,7 +113,11 @@ func (p *podTolerationsPlugin) Admit(a admission.Attributes) error { } var finalTolerations []api.Toleration - if a.GetOperation() == admission.Create { + updateUninitialized, err := util.IsUpdatingUninitializedObject(a) + if err != nil { + return err + } + if a.GetOperation() == admission.Create || updateUninitialized { ts, err := p.getNamespaceDefaultTolerations(namespace) if err != nil { return err diff --git a/plugin/pkg/admission/podtolerationrestriction/admission_test.go b/plugin/pkg/admission/podtolerationrestriction/admission_test.go index 76faaa718949e..4673aa0692c8c 100644 --- a/plugin/pkg/admission/podtolerationrestriction/admission_test.go +++ b/plugin/pkg/admission/podtolerationrestriction/admission_test.go @@ -233,6 +233,11 @@ func TestPodAdmission(t *testing.T) { pod := test.pod pod.Spec.Tolerations = test.podTolerations + // copy the original pod for tests of uninitialized pod updates. + oldPod := *pod + oldPod.Initializers = &metav1.Initializers{Pending: []metav1.Initializer{{Name: "init"}}} + oldPod.Spec.Tolerations = []api.Toleration{{Key: "testKey", Operator: "Equal", Value: "testValue1", Effect: "NoSchedule", TolerationSeconds: nil}} + err := handler.Admit(admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), "testNamespace", namespace.ObjectMeta.Name, api.Resource("pods").WithVersion("version"), "", admission.Create, nil)) if test.admit && err != nil { t.Errorf("Test: %s, expected no error but got: %s", test.testName, err) @@ -244,6 +249,19 @@ func TestPodAdmission(t *testing.T) { if test.admit && !tolerations.EqualTolerations(updatedPodTolerations, test.mergedTolerations) { t.Errorf("Test: %s, expected: %#v but got: %#v", test.testName, test.mergedTolerations, updatedPodTolerations) } + + // handles update of uninitialized pod like it's newly created. + err = handler.Admit(admission.NewAttributesRecord(pod, &oldPod, api.Kind("Pod").WithVersion("version"), "testNamespace", namespace.ObjectMeta.Name, api.Resource("pods").WithVersion("version"), "", admission.Update, nil)) + if test.admit && err != nil { + t.Errorf("Test: %s, expected no error but got: %s", test.testName, err) + } else if !test.admit && err == nil { + t.Errorf("Test: %s, expected an error", test.testName) + } + + updatedPodTolerations = pod.Spec.Tolerations + if test.admit && !tolerations.EqualTolerations(updatedPodTolerations, test.mergedTolerations) { + t.Errorf("Test: %s, expected: %#v but got: %#v", test.testName, test.mergedTolerations, updatedPodTolerations) + } } } @@ -267,6 +285,54 @@ func TestHandles(t *testing.T) { } } +func TestIgnoreUpdatingInitializedPod(t *testing.T) { + mockClient := &fake.Clientset{} + handler, informerFactory, err := newHandlerForTest(mockClient) + if err != nil { + t.Errorf("unexpected error initializing handler: %v", err) + } + handler.SetReadyFunc(func() bool { return true }) + + pod := &api.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "testPod", Namespace: "testNamespace"}, + Spec: api.PodSpec{}, + } + podToleration := api.Toleration{ + Key: "testKey", + Operator: "Equal", + Value: "testValue1", + Effect: "NoSchedule", + TolerationSeconds: nil, + } + pod.Spec.Tolerations = []api.Toleration{podToleration} + + // this conflicts with pod's Tolerations + namespaceToleration := podToleration + namespaceToleration.Value = "testValue2" + namespaceTolerations := []api.Toleration{namespaceToleration} + tolerationsStr, err := json.Marshal(namespaceTolerations) + if err != nil { + t.Errorf("error in marshalling namespace tolerations %v", namespaceTolerations) + } + namespace := &api.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testNamespace", + Namespace: "", + }, + } + namespace.Annotations = map[string]string{NSDefaultTolerations: string(tolerationsStr)} + err = informerFactory.Core().InternalVersion().Namespaces().Informer().GetStore().Update(namespace) + if err != nil { + t.Fatal(err) + } + + // if the update of initialized pod is not ignored, an error will be returned because the pod's Tolerations conflicts with namespace's Tolerations. + err = handler.Admit(admission.NewAttributesRecord(pod, pod, api.Kind("Pod").WithVersion("version"), "testNamespace", pod.ObjectMeta.Name, api.Resource("pods").WithVersion("version"), "", admission.Update, nil)) + if err != nil { + t.Errorf("expected no error, got: %v", err) + } +} + // newHandlerForTest returns the admission controller configured for testing. func newHandlerForTest(c clientset.Interface) (*podTolerationsPlugin, informers.SharedInformerFactory, error) { f := informers.NewSharedInformerFactory(c, 5*time.Minute) diff --git a/plugin/pkg/admission/resourcequota/apis/resourcequota/v1alpha1/zz_generated.conversion.go b/plugin/pkg/admission/resourcequota/apis/resourcequota/v1alpha1/zz_generated.conversion.go index 786ae971711b3..43b75e6b832b8 100644 --- a/plugin/pkg/admission/resourcequota/apis/resourcequota/v1alpha1/zz_generated.conversion.go +++ b/plugin/pkg/admission/resourcequota/apis/resourcequota/v1alpha1/zz_generated.conversion.go @@ -53,11 +53,7 @@ func Convert_v1alpha1_Configuration_To_resourcequota_Configuration(in *Configura } func autoConvert_resourcequota_Configuration_To_v1alpha1_Configuration(in *resourcequota.Configuration, out *Configuration, s conversion.Scope) error { - if in.LimitedResources == nil { - out.LimitedResources = make([]LimitedResource, 0) - } else { - out.LimitedResources = *(*[]LimitedResource)(unsafe.Pointer(&in.LimitedResources)) - } + out.LimitedResources = *(*[]LimitedResource)(unsafe.Pointer(&in.LimitedResources)) return nil } diff --git a/plugin/pkg/admission/resourcequota/controller.go b/plugin/pkg/admission/resourcequota/controller.go index ab32bbf31bacb..84f196ba60da8 100644 --- a/plugin/pkg/admission/resourcequota/controller.go +++ b/plugin/pkg/admission/resourcequota/controller.go @@ -375,8 +375,7 @@ func (e *quotaEvaluator) checkRequest(quotas []api.ResourceQuota, a admission.At return quotas, nil } - op := a.GetOperation() - if !evaluator.Handles(op) { + if !evaluator.Handles(a) { return quotas, nil } @@ -463,7 +462,7 @@ func (e *quotaEvaluator) checkRequest(quotas []api.ResourceQuota, a admission.At return nil, admission.NewForbidden(a, fmt.Errorf("quota usage is negative for resource(s): %s", prettyPrintResourceNames(negativeUsage))) } - if admission.Update == op { + if admission.Update == a.GetOperation() { prevItem := a.GetOldObject() if prevItem == nil { return nil, admission.NewForbidden(a, fmt.Errorf("unable to get previous usage since prior version of object was not found")) @@ -529,8 +528,7 @@ func (e *quotaEvaluator) Evaluate(a admission.Attributes) error { } // for this kind, check if the operation could mutate any quota resources // if no resources tracked by quota are impacted, then just return - op := a.GetOperation() - if !evaluator.Handles(op) { + if !evaluator.Handles(a) { return nil } diff --git a/plugin/pkg/admission/serviceaccount/BUILD b/plugin/pkg/admission/serviceaccount/BUILD index dc51db0a95ac4..0d9bf3a764d8d 100644 --- a/plugin/pkg/admission/serviceaccount/BUILD +++ b/plugin/pkg/admission/serviceaccount/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/client/informers/informers_generated/internalversion:go_default_library", "//pkg/client/listers/core/internalversion:go_default_library", "//pkg/kubeapiserver/admission:go_default_library", + "//pkg/kubeapiserver/admission/util:go_default_library", "//pkg/serviceaccount:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/plugin/pkg/admission/serviceaccount/OWNERS b/plugin/pkg/admission/serviceaccount/OWNERS index 8e6172a01c674..a678215e98405 100644 --- a/plugin/pkg/admission/serviceaccount/OWNERS +++ b/plugin/pkg/admission/serviceaccount/OWNERS @@ -4,3 +4,4 @@ approvers: reviewers: - liggitt - deads2k +- enj diff --git a/plugin/pkg/admission/serviceaccount/admission.go b/plugin/pkg/admission/serviceaccount/admission.go index 2b7c4abfb8f10..9e319d26d6c82 100644 --- a/plugin/pkg/admission/serviceaccount/admission.go +++ b/plugin/pkg/admission/serviceaccount/admission.go @@ -36,6 +36,7 @@ import ( informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" corelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" + "k8s.io/kubernetes/pkg/kubeapiserver/admission/util" "k8s.io/kubernetes/pkg/serviceaccount" ) @@ -90,7 +91,7 @@ var _ = kubeapiserveradmission.WantsInternalKubeInformerFactory(&serviceAccount{ // 5. If MountServiceAccountToken is true, it adds a VolumeMount with the pod's ServiceAccount's api token secret to containers func NewServiceAccount() *serviceAccount { return &serviceAccount{ - Handler: admission.NewHandler(admission.Create), + Handler: admission.NewHandler(admission.Create, admission.Update), // TODO: enable this once we've swept secret usage to account for adding secret references to service accounts LimitSecretReferences: false, // Auto mount service account API token secrets @@ -143,6 +144,15 @@ func (s *serviceAccount) Admit(a admission.Attributes) (err error) { return nil } + updateInitialized, err := util.IsUpdatingInitializedObject(a) + if err != nil { + return err + } + if updateInitialized { + // related pod spec fields are immutable after the pod is initialized + return nil + } + // Don't modify the spec of mirror pods. // That makes the kubelet very angry and confused, and it immediately deletes the pod (because the spec doesn't match) // That said, don't allow mirror pods to reference ServiceAccounts or SecretVolumeSources either diff --git a/plugin/pkg/admission/serviceaccount/admission_test.go b/plugin/pkg/admission/serviceaccount/admission_test.go index f5b8631556603..80337481010fc 100644 --- a/plugin/pkg/admission/serviceaccount/admission_test.go +++ b/plugin/pkg/admission/serviceaccount/admission_test.go @@ -36,7 +36,7 @@ import ( func TestIgnoresNonCreate(t *testing.T) { pod := &api.Pod{} - for _, op := range []admission.Operation{admission.Update, admission.Delete, admission.Connect} { + for _, op := range []admission.Operation{admission.Delete, admission.Connect} { attrs := admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), "myns", "myname", api.Resource("pods").WithVersion("version"), "", op, nil) handler := admission.NewChainHandler(NewServiceAccount()) err := handler.Admit(attrs) @@ -46,6 +46,17 @@ func TestIgnoresNonCreate(t *testing.T) { } } +func TestIgnoresUpdateOfInitializedPod(t *testing.T) { + pod := &api.Pod{} + oldPod := &api.Pod{} + attrs := admission.NewAttributesRecord(pod, oldPod, api.Kind("Pod").WithVersion("version"), "myns", "myname", api.Resource("pods").WithVersion("version"), "", admission.Update, nil) + handler := admission.NewChainHandler(NewServiceAccount()) + err := handler.Admit(attrs) + if err != nil { + t.Errorf("Expected update of initialized pod allowed, got err: %v", err) + } +} + func TestIgnoresNonPodResource(t *testing.T) { pod := &api.Pod{} attrs := admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), "myns", "myname", api.Resource("CustomResource").WithVersion("version"), "", admission.Create, nil) @@ -309,6 +320,55 @@ func TestAutomountsAPIToken(t *testing.T) { t.Fatalf("Expected\n\t%#v\ngot\n\t%#v", expectedVolumeMount, pod.Spec.Containers[0].VolumeMounts[0]) } + // Test ServiceAccount admission plugin applies the same changes if the + // operation is an update to an uninitialized pod. + oldPod := &api.Pod{ + Spec: api.PodSpec{ + Containers: []api.Container{ + { + // the volumeMount in the oldPod shouldn't affect the result. + VolumeMounts: []api.VolumeMount{ + { + Name: "wrong-" + tokenName, + ReadOnly: true, + MountPath: DefaultAPITokenMountPath, + }, + }, + }, + }, + }, + } + // oldPod is not intialized. + oldPod.Initializers = &metav1.Initializers{Pending: []metav1.Initializer{{Name: "init"}}} + pod = &api.Pod{ + Spec: api.PodSpec{ + Containers: []api.Container{ + {}, + }, + }, + } + attrs = admission.NewAttributesRecord(pod, oldPod, api.Kind("Pod").WithVersion("version"), ns, "myname", api.Resource("pods").WithVersion("version"), "", admission.Update, nil) + err = admit.Admit(attrs) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if pod.Spec.ServiceAccountName != DefaultServiceAccountName { + t.Errorf("Expected service account %s assigned, got %s", DefaultServiceAccountName, pod.Spec.ServiceAccountName) + } + if len(pod.Spec.Volumes) != 1 { + t.Fatalf("Expected 1 volume, got %d", len(pod.Spec.Volumes)) + } + if !reflect.DeepEqual(expectedVolume, pod.Spec.Volumes[0]) { + t.Fatalf("Expected\n\t%#v\ngot\n\t%#v", expectedVolume, pod.Spec.Volumes[0]) + } + if len(pod.Spec.Containers[0].VolumeMounts) != 1 { + t.Fatalf("Expected 1 volume mount, got %d", len(pod.Spec.Containers[0].VolumeMounts)) + } + if !reflect.DeepEqual(expectedVolumeMount, pod.Spec.Containers[0].VolumeMounts[0]) { + t.Fatalf("Expected\n\t%#v\ngot\n\t%#v", expectedVolumeMount, pod.Spec.Containers[0].VolumeMounts[0]) + } + + // testing InitContainers pod = &api.Pod{ Spec: api.PodSpec{ InitContainers: []api.Container{ diff --git a/plugin/pkg/auth/OWNERS b/plugin/pkg/auth/OWNERS index ebac20047f920..3d732d11f2d96 100644 --- a/plugin/pkg/auth/OWNERS +++ b/plugin/pkg/auth/OWNERS @@ -7,3 +7,4 @@ reviewers: - liggitt - deads2k - ericchiang +- enj diff --git a/plugin/pkg/auth/authenticator/token/bootstrap/BUILD b/plugin/pkg/auth/authenticator/token/bootstrap/BUILD index abba54f6eee35..ff951252f0896 100644 --- a/plugin/pkg/auth/authenticator/token/bootstrap/BUILD +++ b/plugin/pkg/auth/authenticator/token/bootstrap/BUILD @@ -30,6 +30,7 @@ go_library( "//pkg/client/listers/core/internalversion:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library", ], ) diff --git a/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap.go b/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap.go index 62b3ad50d8cd9..fecea0f6855ff 100644 --- a/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap.go +++ b/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap.go @@ -23,11 +23,13 @@ import ( "crypto/subtle" "fmt" "regexp" + "strings" "time" "github.com/golang/glog" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/kubernetes/pkg/api" bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" @@ -79,6 +81,7 @@ func tokenErrorf(s *api.Secret, format string, i ...interface{}) { // token-id: ( token id ) // # Required key usage. // usage-bootstrap-authentication: true +// auth-extra-groups: "system:bootstrappers:custom-group1,system:bootstrappers:custom-group2" // # May also contain an expiry. // // Tokens are expected to be of the form: @@ -134,9 +137,15 @@ func (t *TokenAuthenticator) AuthenticateToken(token string) (user.Info, bool, e return nil, false, nil } + groups, err := getGroups(secret) + if err != nil { + tokenErrorf(secret, "has invalid value for key %s: %v.", bootstrapapi.BootstrapTokenExtraGroupsKey, err) + return nil, false, nil + } + return &user.DefaultInfo{ Name: bootstrapapi.BootstrapUserPrefix + string(id), - Groups: []string{bootstrapapi.BootstrapGroup}, + Groups: groups, }, true, nil } @@ -184,3 +193,28 @@ func parseToken(s string) (string, string, error) { } return split[1], split[2], nil } + +// getGroups loads and validates the bootstrapapi.BootstrapTokenExtraGroupsKey +// key from the bootstrap token secret, returning a list of group names or an +// error if any of the group names are invalid. +func getGroups(secret *api.Secret) ([]string, error) { + // always include the default group + groups := sets.NewString(bootstrapapi.BootstrapDefaultGroup) + + // grab any extra groups and if there are none, return just the default + extraGroupsString := getSecretString(secret, bootstrapapi.BootstrapTokenExtraGroupsKey) + if extraGroupsString == "" { + return groups.List(), nil + } + + // validate the names of the extra groups + for _, group := range strings.Split(extraGroupsString, ",") { + if err := bootstrapapi.ValidateBootstrapGroupName(group); err != nil { + return nil, err + } + groups.Insert(group) + } + + // return the result as a deduplicated, sorted list + return groups.List(), nil +} diff --git a/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap_test.go b/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap_test.go index e8586e9f1a4dc..7e76faaf63af8 100644 --- a/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap_test.go +++ b/plugin/pkg/auth/authenticator/token/bootstrap/bootstrap_test.go @@ -84,6 +84,47 @@ func TestTokenAuthenticator(t *testing.T) { Groups: []string{"system:bootstrappers"}, }, }, + { + name: "valid token with extra group", + secrets: []*api.Secret{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: bootstrapapi.BootstrapTokenSecretPrefix + tokenID, + }, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), + bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), + bootstrapapi.BootstrapTokenUsageAuthentication: []byte("true"), + bootstrapapi.BootstrapTokenExtraGroupsKey: []byte("system:bootstrappers:foo"), + }, + Type: "bootstrap.kubernetes.io/token", + }, + }, + token: tokenID + "." + tokenSecret, + wantUser: &user.DefaultInfo{ + Name: "system:bootstrap:" + tokenID, + Groups: []string{"system:bootstrappers", "system:bootstrappers:foo"}, + }, + }, + { + name: "invalid group", + secrets: []*api.Secret{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: bootstrapapi.BootstrapTokenSecretPrefix + tokenID, + }, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), + bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), + bootstrapapi.BootstrapTokenUsageAuthentication: []byte("true"), + bootstrapapi.BootstrapTokenExtraGroupsKey: []byte("foo"), + }, + Type: "bootstrap.kubernetes.io/token", + }, + }, + token: tokenID + "." + tokenSecret, + wantNotFound: true, + }, { name: "invalid secret name", secrets: []*api.Secret{ @@ -247,3 +288,72 @@ func TestTokenAuthenticator(t *testing.T) { }() } } + +func TestGetGroups(t *testing.T) { + tests := []struct { + name string + secret *api.Secret + expectResult []string + expectError bool + }{ + { + name: "not set", + secret: &api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{}, + }, + expectResult: []string{"system:bootstrappers"}, + }, + { + name: "set to empty value", + secret: &api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenExtraGroupsKey: []byte(""), + }, + }, + expectResult: []string{"system:bootstrappers"}, + }, + { + name: "invalid prefix", + secret: &api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenExtraGroupsKey: []byte("foo"), + }, + }, + expectError: true, + }, + { + name: "valid", + secret: &api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + bootstrapapi.BootstrapTokenExtraGroupsKey: []byte("system:bootstrappers:foo,system:bootstrappers:bar,system:bootstrappers:bar"), + }, + }, + // expect the results in deduplicated, sorted order + expectResult: []string{ + "system:bootstrappers", + "system:bootstrappers:bar", + "system:bootstrappers:foo", + }, + }, + } + for _, test := range tests { + result, err := getGroups(test.secret) + if test.expectError { + if err == nil { + t.Errorf("test %q expected an error, but didn't get one (result: %#v)", test.name, result) + } + continue + } + if err != nil { + t.Errorf("test %q return an unexpected error: %v", test.name, err) + continue + } + if !reflect.DeepEqual(result, test.expectResult) { + t.Errorf("test %q expected %#v, got %#v", test.name, test.expectResult, result) + } + } +} diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD index b3243e18ff505..501feb306622e 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/BUILD @@ -40,7 +40,7 @@ go_test( "//pkg/registry/rbac/validation:go_default_library", "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", - "//vendor/k8s.io/api/rbac/v1beta1:go_default_library", + "//vendor/k8s.io/api/rbac/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library", diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go index 955ed2e5562bd..c95800fb67aec 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go @@ -141,8 +141,6 @@ func init() { rbac.NewRule("get", "update").Groups(extensionsGroup).Resources("replicationcontrollers/scale").RuleOrDie(), rbac.NewRule("get", "update").Groups(extensionsGroup, appsGroup).Resources("deployments/scale", "replicasets/scale").RuleOrDie(), rbac.NewRule("list").Groups(legacyGroup).Resources("pods").RuleOrDie(), - // TODO: Remove the root /proxy permission in 1.7; MetricsClient no longer requires root proxy access as of 1.6 (fixed in https://github.com/kubernetes/kubernetes/pull/39636) - rbac.NewRule("proxy").Groups(legacyGroup).Resources("services").Names("https:heapster:", "http:heapster:").RuleOrDie(), // TODO: restrict this to the appropriate namespace rbac.NewRule("get").Groups(legacyGroup).Resources("services/proxy").Names("https:heapster:", "http:heapster:").RuleOrDie(), eventsRule(), diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy_test.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy_test.go index b9a6c2a0de2d6..0435a23abc927 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy_test.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy_test.go @@ -26,7 +26,7 @@ import ( "github.com/ghodss/yaml" "k8s.io/api/core/v1" - rbacv1beta1 "k8s.io/api/rbac/v1beta1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/diff" @@ -280,11 +280,11 @@ func testObjects(t *testing.T, list *api.List, fixtureFilename string) { t.Fatal(err) } - if err := runtime.EncodeList(api.Codecs.LegacyCodec(v1.SchemeGroupVersion, rbacv1beta1.SchemeGroupVersion), list.Items); err != nil { + if err := runtime.EncodeList(api.Codecs.LegacyCodec(v1.SchemeGroupVersion, rbacv1.SchemeGroupVersion), list.Items); err != nil { t.Fatal(err) } - jsonData, err := runtime.Encode(api.Codecs.LegacyCodec(v1.SchemeGroupVersion, rbacv1beta1.SchemeGroupVersion), list) + jsonData, err := runtime.Encode(api.Codecs.LegacyCodec(v1.SchemeGroupVersion, rbacv1.SchemeGroupVersion), list) if err != nil { t.Fatal(err) } diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-role-bindings.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-role-bindings.yaml index 87e8d85066c56..1dbc33dd4491d 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-role-bindings.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-role-bindings.yaml @@ -1,6 +1,6 @@ apiVersion: v1 items: -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -17,7 +17,7 @@ items: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:masters -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -37,7 +37,7 @@ items: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:unauthenticated -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -57,7 +57,7 @@ items: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:unauthenticated -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -74,7 +74,7 @@ items: - apiGroup: rbac.authorization.k8s.io kind: User name: system:kube-controller-manager -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -91,7 +91,7 @@ items: - kind: ServiceAccount name: kube-dns namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -108,7 +108,7 @@ items: - apiGroup: rbac.authorization.k8s.io kind: User name: system:kube-scheduler -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -121,8 +121,8 @@ items: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:node - subjects: [] -- apiVersion: rbac.authorization.k8s.io/v1beta1 + subjects: null +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml index 309fab58f27be..0f572371e2f5a 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml @@ -1,6 +1,6 @@ apiVersion: v1 items: -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -161,7 +161,7 @@ items: - patch - update - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -181,7 +181,7 @@ items: - '*' verbs: - '*' -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -322,7 +322,7 @@ items: - patch - update - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -344,7 +344,7 @@ items: - subjectaccessreviews verbs: - create -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -360,7 +360,7 @@ items: - selfsubjectaccessreviews verbs: - create -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -376,7 +376,7 @@ items: - certificatesigningrequests/nodeclient verbs: - create -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -392,7 +392,7 @@ items: - certificatesigningrequests/selfnodeclient verbs: - create -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -413,7 +413,7 @@ items: - /version verbs: - get -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -442,7 +442,7 @@ items: - get - list - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -461,7 +461,7 @@ items: - get - list - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -523,7 +523,7 @@ items: verbs: - list - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -541,7 +541,7 @@ items: verbs: - list - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -632,7 +632,7 @@ items: - get - list - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -751,7 +751,7 @@ items: - get - list - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -770,7 +770,7 @@ items: - get - list - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -800,7 +800,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -832,7 +832,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -883,7 +883,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml index eaa946c2cd2b1..ddb59dbcefdea 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml @@ -1,6 +1,6 @@ apiVersion: v1 items: -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -17,7 +17,7 @@ items: - kind: ServiceAccount name: attachdetach-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -34,7 +34,7 @@ items: - kind: ServiceAccount name: certificate-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -51,7 +51,7 @@ items: - kind: ServiceAccount name: cronjob-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -68,7 +68,7 @@ items: - kind: ServiceAccount name: daemon-set-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -85,7 +85,7 @@ items: - kind: ServiceAccount name: deployment-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -102,7 +102,7 @@ items: - kind: ServiceAccount name: disruption-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -119,7 +119,7 @@ items: - kind: ServiceAccount name: endpoint-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -136,7 +136,7 @@ items: - kind: ServiceAccount name: generic-garbage-collector namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -153,7 +153,7 @@ items: - kind: ServiceAccount name: horizontal-pod-autoscaler namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -170,7 +170,7 @@ items: - kind: ServiceAccount name: job-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -187,7 +187,7 @@ items: - kind: ServiceAccount name: namespace-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -204,7 +204,7 @@ items: - kind: ServiceAccount name: node-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -221,7 +221,7 @@ items: - kind: ServiceAccount name: persistent-volume-binder namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -238,7 +238,7 @@ items: - kind: ServiceAccount name: pod-garbage-collector namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -255,7 +255,7 @@ items: - kind: ServiceAccount name: replicaset-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -272,7 +272,7 @@ items: - kind: ServiceAccount name: replication-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -289,7 +289,7 @@ items: - kind: ServiceAccount name: resourcequota-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -306,7 +306,7 @@ items: - kind: ServiceAccount name: route-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -323,7 +323,7 @@ items: - kind: ServiceAccount name: service-account-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -340,7 +340,7 @@ items: - kind: ServiceAccount name: service-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: @@ -357,7 +357,7 @@ items: - kind: ServiceAccount name: statefulset-controller namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml index b85d951cd6af3..f7ca7ecd26992 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml @@ -1,6 +1,6 @@ apiVersion: v1 items: -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -48,7 +48,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -87,7 +87,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -139,7 +139,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -207,7 +207,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -263,7 +263,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -328,7 +328,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -371,7 +371,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -400,7 +400,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -453,15 +453,6 @@ items: - pods verbs: - list - - apiGroups: - - "" - resourceNames: - - 'http:heapster:' - - 'https:heapster:' - resources: - - services - verbs: - - proxy - apiGroups: - "" resourceNames: @@ -479,7 +470,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -522,7 +513,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -557,7 +548,7 @@ items: - deletecollection - get - list -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -605,7 +596,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -701,7 +692,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -725,7 +716,7 @@ items: - nodes verbs: - list -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -768,7 +759,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -811,7 +802,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -842,7 +833,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -873,7 +864,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -897,7 +888,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -936,7 +927,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: @@ -1004,7 +995,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/namespace-role-bindings.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/namespace-role-bindings.yaml index ca79803cd98c1..f0e126d1e8272 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/namespace-role-bindings.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/namespace-role-bindings.yaml @@ -1,6 +1,6 @@ apiVersion: v1 items: -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: annotations: @@ -18,7 +18,7 @@ items: - kind: ServiceAccount name: bootstrap-signer namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: annotations: @@ -36,7 +36,7 @@ items: - kind: ServiceAccount name: kube-controller-manager namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: annotations: @@ -54,7 +54,7 @@ items: - kind: ServiceAccount name: kube-scheduler namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: annotations: @@ -72,7 +72,7 @@ items: - kind: ServiceAccount name: bootstrap-signer namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: annotations: @@ -90,7 +90,7 @@ items: - kind: ServiceAccount name: cloud-provider namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: annotations: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/namespace-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/namespace-roles.yaml index e8fa714cb0ed2..efcc4bdbe192a 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/namespace-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/namespace-roles.yaml @@ -1,6 +1,6 @@ apiVersion: v1 items: -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: annotations: @@ -35,7 +35,7 @@ items: - create - patch - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: annotations: @@ -54,7 +54,7 @@ items: - configmaps verbs: - get -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: annotations: @@ -80,7 +80,7 @@ items: verbs: - get - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: annotations: @@ -106,7 +106,7 @@ items: verbs: - get - update -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: annotations: @@ -125,7 +125,7 @@ items: - get - list - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: annotations: @@ -145,7 +145,7 @@ items: - get - list - watch -- apiVersion: rbac.authorization.k8s.io/v1beta1 +- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: annotations: diff --git a/plugin/pkg/scheduler/algorithm/predicates/BUILD b/plugin/pkg/scheduler/algorithm/predicates/BUILD index fc15e457b3051..ecefec4112478 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/BUILD +++ b/plugin/pkg/scheduler/algorithm/predicates/BUILD @@ -40,6 +40,7 @@ go_library( go_test( name = "go_default_test", srcs = [ + "metadata_test.go", "predicates_test.go", "utils_test.go", ], diff --git a/plugin/pkg/scheduler/algorithm/predicates/metadata.go b/plugin/pkg/scheduler/algorithm/predicates/metadata.go index 269345a354c39..7289268f1cac1 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/metadata.go +++ b/plugin/pkg/scheduler/algorithm/predicates/metadata.go @@ -17,17 +17,53 @@ limitations under the License. package predicates import ( + "fmt" "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" schedutil "k8s.io/kubernetes/plugin/pkg/scheduler/util" + "sync" ) type PredicateMetadataFactory struct { podLister algorithm.PodLister } +// Note that predicateMetadata and matchingPodAntiAffinityTerm need to be declared in the same file +// due to the way declarations are processed in predicate declaration unit tests. +type matchingPodAntiAffinityTerm struct { + term *v1.PodAffinityTerm + node *v1.Node +} + +// NOTE: When new fields are added/removed or logic is changed, please make sure +// that RemovePod and AddPod functions are updated to work with the new changes. +type predicateMetadata struct { + pod *v1.Pod + podBestEffort bool + podRequest *schedulercache.Resource + podPorts map[int]bool + //key is a pod full name with the anti-affinity rules. + matchingAntiAffinityTerms map[string][]matchingPodAntiAffinityTerm + serviceAffinityInUse bool + serviceAffinityMatchingPodList []*v1.Pod + serviceAffinityMatchingPodServices []*v1.Service +} + +// PredicateMetadataProducer: Helper types/variables... +type PredicateMetadataProducer func(pm *predicateMetadata) + +var predicateMetaProducerRegisterLock sync.Mutex +var predicateMetadataProducers map[string]PredicateMetadataProducer = make(map[string]PredicateMetadataProducer) + +func RegisterPredicateMetadataProducer(predicateName string, precomp PredicateMetadataProducer) { + predicateMetaProducerRegisterLock.Lock() + defer predicateMetaProducerRegisterLock.Unlock() + predicateMetadataProducers[predicateName] = precomp +} + func NewPredicateMetadataFactory(podLister algorithm.PodLister) algorithm.MetadataProducer { factory := &PredicateMetadataFactory{ podLister, @@ -52,9 +88,72 @@ func (pfactory *PredicateMetadataFactory) GetMetadata(pod *v1.Pod, nodeNameToInf podPorts: schedutil.GetUsedPorts(pod), matchingAntiAffinityTerms: matchingTerms, } - for predicateName, precomputeFunc := range predicatePrecomputations { + for predicateName, precomputeFunc := range predicateMetadataProducers { glog.V(10).Infof("Precompute: %v", predicateName) precomputeFunc(predicateMetadata) } return predicateMetadata } + +// RemovePod changes predicateMetadata assuming that the given `deletedPod` is +// deleted from the system. +func (meta *predicateMetadata) RemovePod(deletedPod *v1.Pod) error { + deletedPodFullName := schedutil.GetPodFullName(deletedPod) + if deletedPodFullName == schedutil.GetPodFullName(meta.pod) { + return fmt.Errorf("deletedPod and meta.pod must not be the same.") + } + // Delete any anti-affinity rule from the deletedPod. + delete(meta.matchingAntiAffinityTerms, deletedPodFullName) + // All pods in the serviceAffinityMatchingPodList are in the same namespace. + // So, if the namespace of the first one is not the same as the namespace of the + // deletedPod, we don't need to check the list, as deletedPod isn't in the list. + if meta.serviceAffinityInUse && + len(meta.serviceAffinityMatchingPodList) > 0 && + deletedPod.Namespace == meta.serviceAffinityMatchingPodList[0].Namespace { + for i, pod := range meta.serviceAffinityMatchingPodList { + if schedutil.GetPodFullName(pod) == deletedPodFullName { + meta.serviceAffinityMatchingPodList = append( + meta.serviceAffinityMatchingPodList[:i], + meta.serviceAffinityMatchingPodList[i+1:]...) + break + } + } + } + return nil +} + +// AddPod changes predicateMetadata assuming that `newPod` is added to the +// system. +func (meta *predicateMetadata) AddPod(addedPod *v1.Pod, nodeInfo *schedulercache.NodeInfo) error { + addedPodFullName := schedutil.GetPodFullName(addedPod) + if addedPodFullName == schedutil.GetPodFullName(meta.pod) { + return fmt.Errorf("addedPod and meta.pod must not be the same.") + } + if nodeInfo.Node() == nil { + return fmt.Errorf("Invalid node in nodeInfo.") + } + // Add matching anti-affinity terms of the addedPod to the map. + podMatchingTerms, err := getMatchingAntiAffinityTermsOfExistingPod(meta.pod, addedPod, nodeInfo.Node()) + if err != nil { + return err + } + if len(podMatchingTerms) > 0 { + existingTerms, found := meta.matchingAntiAffinityTerms[addedPodFullName] + if found { + meta.matchingAntiAffinityTerms[addedPodFullName] = append(existingTerms, + podMatchingTerms...) + } else { + meta.matchingAntiAffinityTerms[addedPodFullName] = podMatchingTerms + } + } + // If addedPod is in the same namespace as the meta.pod, update the list + // of matching pods if applicable. + if meta.serviceAffinityInUse && addedPod.Namespace == meta.pod.Namespace { + selector := CreateSelectorFromLabels(meta.pod.Labels) + if selector.Matches(labels.Set(addedPod.Labels)) { + meta.serviceAffinityMatchingPodList = append(meta.serviceAffinityMatchingPodList, + addedPod) + } + } + return nil +} diff --git a/plugin/pkg/scheduler/algorithm/predicates/metadata_test.go b/plugin/pkg/scheduler/algorithm/predicates/metadata_test.go new file mode 100644 index 0000000000000..40476283e8c8d --- /dev/null +++ b/plugin/pkg/scheduler/algorithm/predicates/metadata_test.go @@ -0,0 +1,357 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package predicates + +import ( + "fmt" + "reflect" + "sort" + "testing" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" + schedulertesting "k8s.io/kubernetes/plugin/pkg/scheduler/testing" +) + +// sortableAntiAffinityTerms lets us to sort anti-affinity terms. +type sortableAntiAffinityTerms []matchingPodAntiAffinityTerm + +// Less establishes some ordering between two matchingPodAntiAffinityTerms for +// sorting. +func (s sortableAntiAffinityTerms) Less(i, j int) bool { + t1, t2 := s[i], s[j] + if t1.node.Name != t2.node.Name { + return t1.node.Name < t2.node.Name + } + if len(t1.term.Namespaces) != len(t2.term.Namespaces) { + return len(t1.term.Namespaces) < len(t2.term.Namespaces) + } + if t1.term.TopologyKey != t2.term.TopologyKey { + return t1.term.TopologyKey < t2.term.TopologyKey + } + if len(t1.term.LabelSelector.MatchLabels) != len(t2.term.LabelSelector.MatchLabels) { + return len(t1.term.LabelSelector.MatchLabels) < len(t2.term.LabelSelector.MatchLabels) + } + return false +} +func (s sortableAntiAffinityTerms) Len() int { return len(s) } +func (s sortableAntiAffinityTerms) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +var _ = sort.Interface(sortableAntiAffinityTerms{}) + +func sortAntiAffinityTerms(terms map[string][]matchingPodAntiAffinityTerm) { + for k, v := range terms { + sortableTerms := sortableAntiAffinityTerms(v) + sort.Sort(sortableTerms) + terms[k] = sortableTerms + } +} + +// sortablePods lets us to sort pods. +type sortablePods []*v1.Pod + +func (s sortablePods) Less(i, j int) bool { + return s[i].Namespace < s[j].Namespace || + (s[i].Namespace == s[j].Namespace && s[i].Name < s[j].Name) +} +func (s sortablePods) Len() int { return len(s) } +func (s sortablePods) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +var _ = sort.Interface(&sortablePods{}) + +// sortableServices allows us to sort services. +type sortableServices []*v1.Service + +func (s sortableServices) Less(i, j int) bool { + return s[i].Namespace < s[j].Namespace || + (s[i].Namespace == s[j].Namespace && s[i].Name < s[j].Name) +} +func (s sortableServices) Len() int { return len(s) } +func (s sortableServices) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +var _ = sort.Interface(&sortableServices{}) + +// predicateMetadataEquivalent returns true if the two metadata are equivalent. +// Note: this function does not compare podRequest. +func predicateMetadataEquivalent(meta1, meta2 *predicateMetadata) error { + if !reflect.DeepEqual(meta1.pod, meta2.pod) { + return fmt.Errorf("pods are not the same.") + } + if meta1.podBestEffort != meta2.podBestEffort { + return fmt.Errorf("podBestEfforts are not equal.") + } + if meta1.serviceAffinityInUse != meta1.serviceAffinityInUse { + return fmt.Errorf("serviceAffinityInUses are not equal.") + } + if len(meta1.podPorts) != len(meta2.podPorts) { + return fmt.Errorf("podPorts are not equal.") + } + for !reflect.DeepEqual(meta1.podPorts, meta2.podPorts) { + return fmt.Errorf("podPorts are not equal.") + } + sortAntiAffinityTerms(meta1.matchingAntiAffinityTerms) + sortAntiAffinityTerms(meta2.matchingAntiAffinityTerms) + if !reflect.DeepEqual(meta1.matchingAntiAffinityTerms, meta2.matchingAntiAffinityTerms) { + return fmt.Errorf("matchingAntiAffinityTerms are not euqal.") + } + if meta1.serviceAffinityInUse { + sortablePods1 := sortablePods(meta1.serviceAffinityMatchingPodList) + sort.Sort(sortablePods1) + sortablePods2 := sortablePods(meta2.serviceAffinityMatchingPodList) + sort.Sort(sortablePods2) + if !reflect.DeepEqual(sortablePods1, sortablePods2) { + return fmt.Errorf("serviceAffinityMatchingPodLists are not euqal.") + } + + sortableServices1 := sortableServices(meta1.serviceAffinityMatchingPodServices) + sort.Sort(sortableServices1) + sortableServices2 := sortableServices(meta2.serviceAffinityMatchingPodServices) + sort.Sort(sortableServices2) + if !reflect.DeepEqual(sortableServices1, sortableServices2) { + return fmt.Errorf("serviceAffinityMatchingPodServices are not euqal.") + } + } + return nil +} + +func TestPredicateMetadata_AddRemovePod(t *testing.T) { + var label1 = map[string]string{ + "region": "r1", + "zone": "z11", + } + var label2 = map[string]string{ + "region": "r1", + "zone": "z12", + } + var label3 = map[string]string{ + "region": "r2", + "zone": "z21", + } + selector1 := map[string]string{"foo": "bar"} + antiAffinityFooBar := &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "foo", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + } + antiAffinityComplex := &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "foo", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"bar", "buzz"}, + }, + }, + }, + TopologyKey: "region", + }, + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"bar", "security", "test"}, + }, + }, + }, + TopologyKey: "zone", + }, + }, + } + + tests := []struct { + description string + pendingPod *v1.Pod + addedPod *v1.Pod + existingPods []*v1.Pod + nodes []*v1.Node + services []*v1.Service + }{ + { + description: "no anti-affinity or service affinity exist", + pendingPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pending", Labels: selector1}, + }, + existingPods: []*v1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1}, + Spec: v1.PodSpec{NodeName: "nodeA"}, + }, + {ObjectMeta: metav1.ObjectMeta{Name: "p2"}, + Spec: v1.PodSpec{NodeName: "nodeC"}, + }, + }, + addedPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "addedPod", Labels: selector1}, + Spec: v1.PodSpec{NodeName: "nodeB"}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "nodeA", Labels: label1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "nodeB", Labels: label2}}, + {ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}}, + }, + }, + { + description: "metadata anti-affinity terms are updated correctly after adding and removing a pod", + pendingPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pending", Labels: selector1}, + }, + existingPods: []*v1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1}, + Spec: v1.PodSpec{NodeName: "nodeA"}, + }, + {ObjectMeta: metav1.ObjectMeta{Name: "p2"}, + Spec: v1.PodSpec{ + NodeName: "nodeC", + Affinity: &v1.Affinity{ + PodAntiAffinity: antiAffinityFooBar, + }, + }, + }, + }, + addedPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "addedPod", Labels: selector1}, + Spec: v1.PodSpec{ + NodeName: "nodeB", + Affinity: &v1.Affinity{ + PodAntiAffinity: antiAffinityFooBar, + }, + }, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "nodeA", Labels: label1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "nodeB", Labels: label2}}, + {ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}}, + }, + }, + { + description: "metadata service-affinity data are updated correctly after adding and removing a pod", + pendingPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pending", Labels: selector1}, + }, + existingPods: []*v1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1}, + Spec: v1.PodSpec{NodeName: "nodeA"}, + }, + {ObjectMeta: metav1.ObjectMeta{Name: "p2"}, + Spec: v1.PodSpec{NodeName: "nodeC"}, + }, + }, + addedPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "addedPod", Labels: selector1}, + Spec: v1.PodSpec{NodeName: "nodeB"}, + }, + services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: selector1}}}, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "nodeA", Labels: label1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "nodeB", Labels: label2}}, + {ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}}, + }, + }, + { + description: "metadata anti-affinity terms and service affinity data are updated correctly after adding and removing a pod", + pendingPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pending", Labels: selector1}, + }, + existingPods: []*v1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1}, + Spec: v1.PodSpec{NodeName: "nodeA"}, + }, + {ObjectMeta: metav1.ObjectMeta{Name: "p2"}, + Spec: v1.PodSpec{ + NodeName: "nodeC", + Affinity: &v1.Affinity{ + PodAntiAffinity: antiAffinityFooBar, + }, + }, + }, + }, + addedPod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "addedPod", Labels: selector1}, + Spec: v1.PodSpec{ + NodeName: "nodeA", + Affinity: &v1.Affinity{ + PodAntiAffinity: antiAffinityComplex, + }, + }, + }, + services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: selector1}}}, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "nodeA", Labels: label1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "nodeB", Labels: label2}}, + {ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}}, + }, + }, + } + + for _, test := range tests { + allPodLister := schedulertesting.FakePodLister(append(test.existingPods, test.addedPod)) + // getMeta creates predicate meta data given the list of pods. + getMeta := func(lister schedulertesting.FakePodLister) (*predicateMetadata, map[string]*schedulercache.NodeInfo) { + nodeInfoMap := schedulercache.CreateNodeNameToInfoMap(lister, test.nodes) + // nodeList is a list of non-pointer nodes to feed to FakeNodeListInfo. + nodeList := []v1.Node{} + for _, n := range test.nodes { + nodeList = append(nodeList, *n) + } + _, precompute := NewServiceAffinityPredicate(lister, schedulertesting.FakeServiceLister(test.services), FakeNodeListInfo(nodeList), nil) + RegisterPredicateMetadataProducer("ServiceAffinityMetaProducer", precompute) + pmf := PredicateMetadataFactory{lister} + meta := pmf.GetMetadata(test.pendingPod, nodeInfoMap) + return meta.(*predicateMetadata), nodeInfoMap + } + + // allPodsMeta is meta data produced when all pods, including test.addedPod + // are given to the metadata producer. + allPodsMeta, _ := getMeta(allPodLister) + // existingPodsMeta1 is meta data produced for test.existingPods (without test.addedPod). + existingPodsMeta1, nodeInfoMap := getMeta(schedulertesting.FakePodLister(test.existingPods)) + // Add test.addedPod to existingPodsMeta1 and make sure meta is equal to allPodsMeta + nodeInfo := nodeInfoMap[test.addedPod.Spec.NodeName] + if err := existingPodsMeta1.AddPod(test.addedPod, nodeInfo); err != nil { + t.Errorf("test [%v]: error adding pod to meta: %v", test.description, err) + } + if err := predicateMetadataEquivalent(allPodsMeta, existingPodsMeta1); err != nil { + t.Errorf("test [%v]: meta data are not equivalent: %v", test.description, err) + } + // Remove the added pod and from existingPodsMeta1 an make sure it is equal + // to meta generated for existing pods. + existingPodsMeta2, _ := getMeta(schedulertesting.FakePodLister(test.existingPods)) + if err := existingPodsMeta1.RemovePod(test.addedPod); err != nil { + t.Errorf("test [%v]: error removing pod from meta: %v", test.description, err) + } + if err := predicateMetadataEquivalent(existingPodsMeta1, existingPodsMeta2); err != nil { + t.Errorf("test [%v]: meta data are not equivalent: %v", test.description, err) + } + } +} diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go index 97b970c9e4d2b..bc1b05523d12b 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates.go @@ -44,18 +44,6 @@ import ( "k8s.io/metrics/pkg/client/clientset_generated/clientset" ) -// PredicateMetadataModifier: Helper types/variables... -type PredicateMetadataModifier func(pm *predicateMetadata) - -var predicatePrecomputeRegisterLock sync.Mutex -var predicatePrecomputations map[string]PredicateMetadataModifier = make(map[string]PredicateMetadataModifier) - -func RegisterPredicatePrecomputation(predicateName string, precomp PredicateMetadataModifier) { - predicatePrecomputeRegisterLock.Lock() - defer predicatePrecomputeRegisterLock.Unlock() - predicatePrecomputations[predicateName] = precomp -} - // NodeInfo: Other types for predicate functions... type NodeInfo interface { GetNodeInfo(nodeID string) (*v1.Node, error) @@ -107,23 +95,6 @@ func (c *CachedNodeInfo) GetNodeInfo(id string) (*v1.Node, error) { return node, nil } -// Note that predicateMetadata and matchingPodAntiAffinityTerm need to be declared in the same file -// due to the way declarations are processed in predicate declaration unit tests. -type matchingPodAntiAffinityTerm struct { - term *v1.PodAffinityTerm - node *v1.Node -} - -type predicateMetadata struct { - pod *v1.Pod - podBestEffort bool - podRequest *schedulercache.Resource - podPorts map[int]bool - matchingAntiAffinityTerms []matchingPodAntiAffinityTerm - serviceAffinityMatchingPodList []*v1.Pod - serviceAffinityMatchingPodServices []*v1.Service -} - func isVolumeConflict(volume v1.Volume, pod *v1.Pod) bool { // fast path if there is no conflict checking targets. if volume.GCEPersistentDisk == nil && volume.AWSElasticBlockStore == nil && volume.RBD == nil && volume.ISCSI == nil { @@ -504,14 +475,6 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource { result.Add(container.Resources.Requests) } - // Account for storage requested by emptydir volumes - // If the storage medium is memory, should exclude the size - for _, vol := range pod.Spec.Volumes { - if vol.EmptyDir != nil && vol.EmptyDir.Medium != v1.StorageMediumMemory { - result.StorageScratch += vol.EmptyDir.SizeLimit.Value() - } - } - // take max_resource(sum_pod, any_init_container) for _, container := range pod.Spec.InitContainers { for rName, rQuantity := range container.Resources.Requests { @@ -520,6 +483,10 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource { if mem := rQuantity.Value(); mem > result.Memory { result.Memory = mem } + case v1.ResourceEphemeralStorage: + if ephemeralStorage := rQuantity.Value(); ephemeralStorage > result.EphemeralStorage { + result.EphemeralStorage = ephemeralStorage + } case v1.ResourceCPU: if cpu := rQuantity.MilliValue(); cpu > result.MilliCPU { result.MilliCPU = cpu @@ -528,10 +495,6 @@ func GetResourceRequest(pod *v1.Pod) *schedulercache.Resource { if gpu := rQuantity.Value(); gpu > result.NvidiaGPU { result.NvidiaGPU = gpu } - case v1.ResourceStorageOverlay: - if overlay := rQuantity.Value(); overlay > result.StorageOverlay { - result.StorageOverlay = overlay - } default: if v1helper.IsExtendedResourceName(rName) { value := rQuantity.Value() @@ -572,7 +535,7 @@ func PodFitsResources(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.No // We couldn't parse metadata - fallback to computing it. podRequest = GetResourceRequest(pod) } - if podRequest.MilliCPU == 0 && podRequest.Memory == 0 && podRequest.NvidiaGPU == 0 && podRequest.StorageOverlay == 0 && podRequest.StorageScratch == 0 && len(podRequest.ExtendedResources) == 0 { + if podRequest.MilliCPU == 0 && podRequest.Memory == 0 && podRequest.NvidiaGPU == 0 && podRequest.EphemeralStorage == 0 && len(podRequest.ExtendedResources) == 0 { return len(predicateFails) == 0, predicateFails, nil } @@ -587,20 +550,8 @@ func PodFitsResources(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.No predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceNvidiaGPU, podRequest.NvidiaGPU, nodeInfo.RequestedResource().NvidiaGPU, allocatable.NvidiaGPU)) } - scratchSpaceRequest := podRequest.StorageScratch - if allocatable.StorageOverlay == 0 { - scratchSpaceRequest += podRequest.StorageOverlay - //scratchSpaceRequest += nodeInfo.RequestedResource().StorageOverlay - nodeScratchRequest := nodeInfo.RequestedResource().StorageOverlay + nodeInfo.RequestedResource().StorageScratch - if allocatable.StorageScratch < scratchSpaceRequest+nodeScratchRequest { - predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceStorageScratch, scratchSpaceRequest, nodeScratchRequest, allocatable.StorageScratch)) - } - - } else if allocatable.StorageScratch < scratchSpaceRequest+nodeInfo.RequestedResource().StorageScratch { - predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceStorageScratch, scratchSpaceRequest, nodeInfo.RequestedResource().StorageScratch, allocatable.StorageScratch)) - } - if allocatable.StorageOverlay > 0 && allocatable.StorageOverlay < podRequest.StorageOverlay+nodeInfo.RequestedResource().StorageOverlay { - predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceStorageOverlay, podRequest.StorageOverlay, nodeInfo.RequestedResource().StorageOverlay, allocatable.StorageOverlay)) + if allocatable.EphemeralStorage < podRequest.EphemeralStorage+nodeInfo.RequestedResource().EphemeralStorage { + predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourceEphemeralStorage, podRequest.EphemeralStorage, nodeInfo.RequestedResource().EphemeralStorage, allocatable.EphemeralStorage)) } for rName, rQuant := range podRequest.ExtendedResources { @@ -758,43 +709,42 @@ type ServiceAffinity struct { labels []string } -// serviceAffinityPrecomputation should be run once by the scheduler before looping through the Predicate. It is a helper function that +// serviceAffinityMetadataProducer should be run once by the scheduler before looping through the Predicate. It is a helper function that // only should be referenced by NewServiceAffinityPredicate. -func (s *ServiceAffinity) serviceAffinityPrecomputation(pm *predicateMetadata) { +func (s *ServiceAffinity) serviceAffinityMetadataProducer(pm *predicateMetadata) { if pm.pod == nil { glog.Errorf("Cannot precompute service affinity, a pod is required to calculate service affinity.") return } - + pm.serviceAffinityInUse = true var errSvc, errList error // Store services which match the pod. pm.serviceAffinityMatchingPodServices, errSvc = s.serviceLister.GetPodServices(pm.pod) selector := CreateSelectorFromLabels(pm.pod.Labels) - // consider only the pods that belong to the same namespace allMatches, errList := s.podLister.List(selector) // In the future maybe we will return them as part of the function. if errSvc != nil || errList != nil { glog.Errorf("Some Error were found while precomputing svc affinity: \nservices:%v , \npods:%v", errSvc, errList) } + // consider only the pods that belong to the same namespace pm.serviceAffinityMatchingPodList = FilterPodsByNamespace(allMatches, pm.pod.Namespace) } -func NewServiceAffinityPredicate(podLister algorithm.PodLister, serviceLister algorithm.ServiceLister, nodeInfo NodeInfo, labels []string) (algorithm.FitPredicate, PredicateMetadataModifier) { +func NewServiceAffinityPredicate(podLister algorithm.PodLister, serviceLister algorithm.ServiceLister, nodeInfo NodeInfo, labels []string) (algorithm.FitPredicate, PredicateMetadataProducer) { affinity := &ServiceAffinity{ podLister: podLister, serviceLister: serviceLister, nodeInfo: nodeInfo, labels: labels, } - return affinity.checkServiceAffinity, affinity.serviceAffinityPrecomputation + return affinity.checkServiceAffinity, affinity.serviceAffinityMetadataProducer } // checkServiceAffinity is a predicate which matches nodes in such a way to force that // ServiceAffinity.labels are homogenous for pods that are scheduled to a node. // (i.e. it returns true IFF this pod can be added to this node such that all other pods in -// the same service are running on nodes with -// the exact same ServiceAffinity.label values). +// the same service are running on nodes with the exact same ServiceAffinity.label values). // // For example: // If the first pod of a service was scheduled to a node with label "region=foo", @@ -827,7 +777,7 @@ func (s *ServiceAffinity) checkServiceAffinity(pod *v1.Pod, meta interface{}, no } else { // Make the predicate resilient in case metadata is missing. pm = &predicateMetadata{pod: pod} - s.serviceAffinityPrecomputation(pm) + s.serviceAffinityMetadataProducer(pm) pods, services = pm.serviceAffinityMatchingPodList, pm.serviceAffinityMatchingPodServices } node := nodeInfo.Node() @@ -984,7 +934,7 @@ func (c *PodAffinityChecker) InterPodAffinityMatches(pod *v1.Pod, meta interface if node == nil { return false, nil, fmt.Errorf("node not found") } - if !c.satisfiesExistingPodsAntiAffinity(pod, meta, node) { + if !c.satisfiesExistingPodsAntiAffinity(pod, meta, nodeInfo) { return false, []algorithm.PredicateFailureReason{ErrPodAffinityNotMatch}, nil } @@ -993,7 +943,7 @@ func (c *PodAffinityChecker) InterPodAffinityMatches(pod *v1.Pod, meta interface if affinity == nil || (affinity.PodAffinity == nil && affinity.PodAntiAffinity == nil) { return true, nil, nil } - if !c.satisfiesPodsAffinityAntiAffinity(pod, node, affinity) { + if !c.satisfiesPodsAffinityAntiAffinity(pod, nodeInfo, affinity) { return false, []algorithm.PredicateFailureReason{ErrPodAffinityNotMatch}, nil } @@ -1062,19 +1012,21 @@ func getPodAntiAffinityTerms(podAntiAffinity *v1.PodAntiAffinity) (terms []v1.Po return terms } -func getMatchingAntiAffinityTerms(pod *v1.Pod, nodeInfoMap map[string]*schedulercache.NodeInfo) ([]matchingPodAntiAffinityTerm, error) { +func getMatchingAntiAffinityTerms(pod *v1.Pod, nodeInfoMap map[string]*schedulercache.NodeInfo) (map[string][]matchingPodAntiAffinityTerm, error) { allNodeNames := make([]string, 0, len(nodeInfoMap)) for name := range nodeInfoMap { allNodeNames = append(allNodeNames, name) } var lock sync.Mutex - var result []matchingPodAntiAffinityTerm var firstError error - appendResult := func(toAppend []matchingPodAntiAffinityTerm) { + result := make(map[string][]matchingPodAntiAffinityTerm) + appendResult := func(toAppend map[string][]matchingPodAntiAffinityTerm) { lock.Lock() defer lock.Unlock() - result = append(result, toAppend...) + for uid, terms := range toAppend { + result[uid] = append(result[uid], terms...) + } } catchError := func(err error) { lock.Lock() @@ -1091,7 +1043,7 @@ func getMatchingAntiAffinityTerms(pod *v1.Pod, nodeInfoMap map[string]*scheduler catchError(fmt.Errorf("node not found")) return } - var nodeResult []matchingPodAntiAffinityTerm + nodeResult := make(map[string][]matchingPodAntiAffinityTerm) for _, existingPod := range nodeInfo.PodsWithAffinity() { affinity := existingPod.Spec.Affinity if affinity == nil { @@ -1105,7 +1057,10 @@ func getMatchingAntiAffinityTerms(pod *v1.Pod, nodeInfoMap map[string]*scheduler return } if priorityutil.PodMatchesTermsNamespaceAndSelector(pod, namespaces, selector) { - nodeResult = append(nodeResult, matchingPodAntiAffinityTerm{term: &term, node: node}) + existingPodFullName := schedutil.GetPodFullName(existingPod) + nodeResult[existingPodFullName] = append( + nodeResult[existingPodFullName], + matchingPodAntiAffinityTerm{term: &term, node: node}) } } } @@ -1117,8 +1072,26 @@ func getMatchingAntiAffinityTerms(pod *v1.Pod, nodeInfoMap map[string]*scheduler return result, firstError } -func (c *PodAffinityChecker) getMatchingAntiAffinityTerms(pod *v1.Pod, allPods []*v1.Pod) ([]matchingPodAntiAffinityTerm, error) { +func getMatchingAntiAffinityTermsOfExistingPod(newPod *v1.Pod, existingPod *v1.Pod, node *v1.Node) ([]matchingPodAntiAffinityTerm, error) { var result []matchingPodAntiAffinityTerm + affinity := existingPod.Spec.Affinity + if affinity != nil && affinity.PodAntiAffinity != nil { + for _, term := range getPodAntiAffinityTerms(affinity.PodAntiAffinity) { + namespaces := priorityutil.GetNamespacesFromPodAffinityTerm(existingPod, &term) + selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) + if err != nil { + return nil, err + } + if priorityutil.PodMatchesTermsNamespaceAndSelector(newPod, namespaces, selector) { + result = append(result, matchingPodAntiAffinityTerm{term: &term, node: node}) + } + } + } + return result, nil +} + +func (c *PodAffinityChecker) getMatchingAntiAffinityTerms(pod *v1.Pod, allPods []*v1.Pod) (map[string][]matchingPodAntiAffinityTerm, error) { + result := make(map[string][]matchingPodAntiAffinityTerm) for _, existingPod := range allPods { affinity := existingPod.Spec.Affinity if affinity != nil && affinity.PodAntiAffinity != nil { @@ -1126,15 +1099,13 @@ func (c *PodAffinityChecker) getMatchingAntiAffinityTerms(pod *v1.Pod, allPods [ if err != nil { return nil, err } - for _, term := range getPodAntiAffinityTerms(affinity.PodAntiAffinity) { - namespaces := priorityutil.GetNamespacesFromPodAffinityTerm(existingPod, &term) - selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) - if err != nil { - return nil, err - } - if priorityutil.PodMatchesTermsNamespaceAndSelector(pod, namespaces, selector) { - result = append(result, matchingPodAntiAffinityTerm{term: &term, node: existingPodNode}) - } + existingPodMatchingTerms, err := getMatchingAntiAffinityTermsOfExistingPod(pod, existingPod, existingPodNode) + if err != nil { + return nil, err + } + if len(existingPodMatchingTerms) > 0 { + existingPodFullName := schedutil.GetPodFullName(existingPod) + result[existingPodFullName] = existingPodMatchingTerms } } } @@ -1143,30 +1114,39 @@ func (c *PodAffinityChecker) getMatchingAntiAffinityTerms(pod *v1.Pod, allPods [ // Checks if scheduling the pod onto this node would break any anti-affinity // rules indicated by the existing pods. -func (c *PodAffinityChecker) satisfiesExistingPodsAntiAffinity(pod *v1.Pod, meta interface{}, node *v1.Node) bool { - var matchingTerms []matchingPodAntiAffinityTerm +func (c *PodAffinityChecker) satisfiesExistingPodsAntiAffinity(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.NodeInfo) bool { + node := nodeInfo.Node() + if node == nil { + return false + } + var matchingTerms map[string][]matchingPodAntiAffinityTerm if predicateMeta, ok := meta.(*predicateMetadata); ok { matchingTerms = predicateMeta.matchingAntiAffinityTerms } else { - allPods, err := c.podLister.List(labels.Everything()) + // Filter out pods whose nodeName is equal to nodeInfo.node.Name, but are not + // present in nodeInfo. Pods on other nodes pass the filter. + filteredPods, err := c.podLister.FilteredList(nodeInfo.Filter, labels.Everything()) if err != nil { glog.Errorf("Failed to get all pods, %+v", err) return false } - if matchingTerms, err = c.getMatchingAntiAffinityTerms(pod, allPods); err != nil { + if matchingTerms, err = c.getMatchingAntiAffinityTerms(pod, filteredPods); err != nil { glog.Errorf("Failed to get all terms that pod %+v matches, err: %+v", podName(pod), err) return false } } - for _, term := range matchingTerms { - if len(term.term.TopologyKey) == 0 { - glog.Error("Empty topologyKey is not allowed except for PreferredDuringScheduling pod anti-affinity") - return false - } - if priorityutil.NodesHaveSameTopologyKey(node, term.node, term.term.TopologyKey) { - glog.V(10).Infof("Cannot schedule pod %+v onto node %v,because of PodAntiAffinityTerm %v", - podName(pod), node.Name, term.term) - return false + for _, terms := range matchingTerms { + for i := range terms { + term := &terms[i] + if len(term.term.TopologyKey) == 0 { + glog.Error("Empty topologyKey is not allowed except for PreferredDuringScheduling pod anti-affinity") + return false + } + if priorityutil.NodesHaveSameTopologyKey(node, term.node, term.term.TopologyKey) { + glog.V(10).Infof("Cannot schedule pod %+v onto node %v,because of PodAntiAffinityTerm %v", + podName(pod), node.Name, term.term) + return false + } } } if glog.V(10) { @@ -1179,15 +1159,19 @@ func (c *PodAffinityChecker) satisfiesExistingPodsAntiAffinity(pod *v1.Pod, meta } // Checks if scheduling the pod onto this node would break any rules of this pod. -func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, node *v1.Node, affinity *v1.Affinity) bool { - allPods, err := c.podLister.List(labels.Everything()) +func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, nodeInfo *schedulercache.NodeInfo, affinity *v1.Affinity) bool { + node := nodeInfo.Node() + if node == nil { + return false + } + filteredPods, err := c.podLister.FilteredList(nodeInfo.Filter, labels.Everything()) if err != nil { return false } // Check all affinity terms. for _, term := range getPodAffinityTerms(affinity.PodAffinity) { - termMatches, matchingPodExists, err := c.anyPodMatchesPodAffinityTerm(pod, allPods, node, &term) + termMatches, matchingPodExists, err := c.anyPodMatchesPodAffinityTerm(pod, filteredPods, node, &term) if err != nil { glog.Errorf("Cannot schedule pod %+v onto node %v, because of PodAffinityTerm %v, err: %v", podName(pod), node.Name, term, err) @@ -1220,7 +1204,7 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, node // Check all anti-affinity terms. for _, term := range getPodAntiAffinityTerms(affinity.PodAntiAffinity) { - termMatches, _, err := c.anyPodMatchesPodAffinityTerm(pod, allPods, node, &term) + termMatches, _, err := c.anyPodMatchesPodAffinityTerm(pod, filteredPods, node, &term) if err != nil || termMatches { glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAntiAffinityTerm %v, err: %v", podName(pod), node.Name, term, err) @@ -1237,7 +1221,7 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, node return true } -// PodToleratesNodeTaints checks if a pod tolertaions can tolerate the node taints +// PodToleratesNodeTaints checks if a pod tolerations can tolerate the node taints func PodToleratesNodeTaints(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.NodeInfo) (bool, []algorithm.PredicateFailureReason, error) { return podToleratesNodeTaints(pod, nodeInfo, func(t *v1.Taint) bool { // PodToleratesNodeTaints is only interested in NoSchedule and NoExecute taints. @@ -1245,7 +1229,7 @@ func PodToleratesNodeTaints(pod *v1.Pod, meta interface{}, nodeInfo *schedulerca }) } -// PodToleratesNodeNoExecuteTaints checks if a pod tolertaions can tolerate the node's NoExecute taints +// PodToleratesNodeNoExecuteTaints checks if a pod tolerations can tolerate the node's NoExecute taints func PodToleratesNodeNoExecuteTaints(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.NodeInfo) (bool, []algorithm.PredicateFailureReason, error) { return podToleratesNodeTaints(pod, nodeInfo, func(t *v1.Taint) bool { return t.Effect == v1.TaintEffectNoExecute @@ -1284,7 +1268,7 @@ func CheckNodeMemoryPressurePredicate(pod *v1.Pod, meta interface{}, nodeInfo *s return true, nil, nil } - // check if node is under memory preasure + // check if node is under memory pressure if nodeInfo.MemoryPressureCondition() == v1.ConditionTrue { return false, []algorithm.PredicateFailureReason{ErrNodeUnderMemoryPressure}, nil } @@ -1294,7 +1278,7 @@ func CheckNodeMemoryPressurePredicate(pod *v1.Pod, meta interface{}, nodeInfo *s // CheckNodeDiskPressurePredicate checks if a pod can be scheduled on a node // reporting disk pressure condition. func CheckNodeDiskPressurePredicate(pod *v1.Pod, meta interface{}, nodeInfo *schedulercache.NodeInfo) (bool, []algorithm.PredicateFailureReason, error) { - // check if node is under disk preasure + // check if node is under disk pressure if nodeInfo.DiskPressureCondition() == v1.ConditionTrue { return false, []algorithm.PredicateFailureReason{ErrNodeUnderDiskPressure}, nil } diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go index 0dba44d4ac474..7c878c5ac3c80 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -80,24 +80,24 @@ var ( func makeResources(milliCPU, memory, nvidiaGPUs, pods, opaqueA, storage int64) v1.NodeResources { return v1.NodeResources{ Capacity: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), - v1.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), - v1.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), - opaqueResourceA: *resource.NewQuantity(opaqueA, resource.DecimalSI), - v1.ResourceStorageScratch: *resource.NewQuantity(storage, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), + v1.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), + v1.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), + opaqueResourceA: *resource.NewQuantity(opaqueA, resource.DecimalSI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(storage, resource.BinarySI), }, } } func makeAllocatableResources(milliCPU, memory, nvidiaGPUs, pods, opaqueA, storage int64) v1.ResourceList { return v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), - v1.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), - v1.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), - opaqueResourceA: *resource.NewQuantity(opaqueA, resource.DecimalSI), - v1.ResourceStorageScratch: *resource.NewQuantity(storage, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), + v1.ResourcePods: *resource.NewQuantity(pods, resource.DecimalSI), + v1.ResourceNvidiaGPU: *resource.NewQuantity(nvidiaGPUs, resource.DecimalSI), + opaqueResourceA: *resource.NewQuantity(opaqueA, resource.DecimalSI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(storage, resource.BinarySI), } } @@ -421,92 +421,52 @@ func TestPodFitsResources(t *testing.T) { } storagePodsTests := []struct { - pod *v1.Pod - emptyDirLimit int64 - storageMedium v1.StorageMedium - nodeInfo *schedulercache.NodeInfo - fits bool - test string - reasons []algorithm.PredicateFailureReason + pod *v1.Pod + nodeInfo *schedulercache.NodeInfo + fits bool + test string + reasons []algorithm.PredicateFailureReason }{ { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 1}), + pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1}), nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 10, Memory: 10, StorageOverlay: 20})), + newResourcePod(schedulercache.Resource{MilliCPU: 10, Memory: 10})), fits: false, test: "due to container scratch disk", reasons: []algorithm.PredicateFailureReason{ NewInsufficientResourceError(v1.ResourceCPU, 1, 10, 10), - NewInsufficientResourceError(v1.ResourceStorageScratch, 1, 20, 20), }, }, { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 10}), + pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1}), nodeInfo: schedulercache.NewNodeInfo( newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 10})), fits: true, test: "pod fit", }, { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 18}), - nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), - fits: false, - test: "request exceeds allocatable overlay storage resource", - reasons: []algorithm.PredicateFailureReason{ - NewInsufficientResourceError(v1.ResourceStorageScratch, 18, 5, 20), - }, - }, - { - pod: newResourcePod(schedulercache.Resource{StorageOverlay: 18}), + pod: newResourcePod(schedulercache.Resource{EphemeralStorage: 25}), nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), + newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2})), fits: false, - test: "request exceeds allocatable overlay storage resource", + test: "storage ephemeral local storage request exceeds allocatable", reasons: []algorithm.PredicateFailureReason{ - NewInsufficientResourceError(v1.ResourceStorageScratch, 18, 5, 20), + NewInsufficientResourceError(v1.ResourceEphemeralStorage, 25, 0, 20), }, }, { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 10}), - emptyDirLimit: 15, - storageMedium: v1.StorageMediumDefault, + pod: newResourcePod(schedulercache.Resource{EphemeralStorage: 10}), nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), - fits: false, - test: "storage scratchrequest exceeds allocatable", - reasons: []algorithm.PredicateFailureReason{ - NewInsufficientResourceError(v1.ResourceStorageScratch, 25, 5, 20), - }, - }, - { - pod: newResourcePod(schedulercache.Resource{}), - emptyDirLimit: 25, - storageMedium: v1.StorageMediumDefault, - nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), - fits: false, - test: "storage scratchrequest exceeds allocatable", - reasons: []algorithm.PredicateFailureReason{ - NewInsufficientResourceError(v1.ResourceStorageScratch, 25, 5, 20), - }, - }, - { - pod: newResourcePod(schedulercache.Resource{MilliCPU: 1, Memory: 1, StorageOverlay: 10}), - emptyDirLimit: 15, - storageMedium: v1.StorageMediumMemory, - nodeInfo: schedulercache.NewNodeInfo( - newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2, StorageOverlay: 5})), + newResourcePod(schedulercache.Resource{MilliCPU: 2, Memory: 2})), fits: true, - test: "pod fit with memory medium", + test: "pod fits", }, } for _, test := range storagePodsTests { node := v1.Node{Status: v1.NodeStatus{Capacity: makeResources(10, 20, 0, 32, 5, 20).Capacity, Allocatable: makeAllocatableResources(10, 20, 0, 32, 5, 20)}} test.nodeInfo.SetNode(&node) - pod := addStorageLimit(test.pod, test.emptyDirLimit, test.storageMedium) - fits, reasons, err := PodFitsResources(pod, PredicateMetadata(pod, nil), test.nodeInfo) + fits, reasons, err := PodFitsResources(test.pod, PredicateMetadata(test.pod, nil), test.nodeInfo) if err != nil { t.Errorf("%s: unexpected error: %v", test.test, err) } @@ -1590,7 +1550,7 @@ func TestServiceAffinity(t *testing.T) { // Reimplementing the logic that the scheduler implements: Any time it makes a predicate, it registers any precomputations. predicate, precompute := NewServiceAffinityPredicate(schedulertesting.FakePodLister(test.pods), schedulertesting.FakeServiceLister(test.services), FakeNodeListInfo(nodes), test.labels) // Register a precomputation or Rewrite the precomputation to a no-op, depending on the state we want to test. - RegisterPredicatePrecomputation("checkServiceAffinity-unitTestPredicate", func(pm *predicateMetadata) { + RegisterPredicateMetadataProducer("ServiceAffinityMetaProducer", func(pm *predicateMetadata) { if !skipPrecompute { precompute(pm) } diff --git a/plugin/pkg/scheduler/algorithm/priorities/balanced_resource_allocation_test.go b/plugin/pkg/scheduler/algorithm/priorities/balanced_resource_allocation_test.go index 09a4afd9bf5a7..777be1b14991f 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/balanced_resource_allocation_test.go +++ b/plugin/pkg/scheduler/algorithm/priorities/balanced_resource_allocation_test.go @@ -51,16 +51,16 @@ func TestBalancedResourceAllocation(t *testing.T) { { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("1000m"), - "memory": resource.MustParse("0"), + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("0"), }, }, }, { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("2000m"), - "memory": resource.MustParse("0"), + v1.ResourceCPU: resource.MustParse("2000m"), + v1.ResourceMemory: resource.MustParse("0"), }, }, }, @@ -74,16 +74,16 @@ func TestBalancedResourceAllocation(t *testing.T) { { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("1000m"), - "memory": resource.MustParse("2000"), + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("2000"), }, }, }, { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("2000m"), - "memory": resource.MustParse("3000"), + v1.ResourceCPU: resource.MustParse("2000m"), + v1.ResourceMemory: resource.MustParse("3000"), }, }, }, diff --git a/plugin/pkg/scheduler/algorithm/priorities/least_requested_test.go b/plugin/pkg/scheduler/algorithm/priorities/least_requested_test.go index 05afb586a1ebf..08e083361b43f 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/least_requested_test.go +++ b/plugin/pkg/scheduler/algorithm/priorities/least_requested_test.go @@ -51,16 +51,16 @@ func TestLeastRequested(t *testing.T) { { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("1000m"), - "memory": resource.MustParse("0"), + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("0"), }, }, }, { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("2000m"), - "memory": resource.MustParse("0"), + v1.ResourceCPU: resource.MustParse("2000m"), + v1.ResourceMemory: resource.MustParse("0"), }, }, }, @@ -74,16 +74,16 @@ func TestLeastRequested(t *testing.T) { { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("1000m"), - "memory": resource.MustParse("2000"), + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("2000"), }, }, }, { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("2000m"), - "memory": resource.MustParse("3000"), + v1.ResourceCPU: resource.MustParse("2000m"), + v1.ResourceMemory: resource.MustParse("3000"), }, }, }, diff --git a/plugin/pkg/scheduler/algorithm/priorities/metadata_test.go b/plugin/pkg/scheduler/algorithm/priorities/metadata_test.go index c8bc1e1cdbb5a..b8fd653ba245d 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/metadata_test.go +++ b/plugin/pkg/scheduler/algorithm/priorities/metadata_test.go @@ -85,8 +85,8 @@ func TestPriorityMetadata(t *testing.T) { ImagePullPolicy: "Always", Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("200m"), - "memory": resource.MustParse("2000"), + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("2000"), }, }, }, diff --git a/plugin/pkg/scheduler/algorithm/priorities/most_requested_test.go b/plugin/pkg/scheduler/algorithm/priorities/most_requested_test.go index 5497ae2dd34e3..a77692b4af9ef 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/most_requested_test.go +++ b/plugin/pkg/scheduler/algorithm/priorities/most_requested_test.go @@ -45,16 +45,16 @@ func TestMostRequested(t *testing.T) { { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("1000m"), - "memory": resource.MustParse("0"), + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("0"), }, }, }, { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("2000m"), - "memory": resource.MustParse("0"), + v1.ResourceCPU: resource.MustParse("2000m"), + v1.ResourceMemory: resource.MustParse("0"), }, }, }, @@ -68,16 +68,16 @@ func TestMostRequested(t *testing.T) { { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("1000m"), - "memory": resource.MustParse("2000"), + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("2000"), }, }, }, { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("2000m"), - "memory": resource.MustParse("3000"), + v1.ResourceCPU: resource.MustParse("2000m"), + v1.ResourceMemory: resource.MustParse("3000"), }, }, }, @@ -89,16 +89,16 @@ func TestMostRequested(t *testing.T) { { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("2000m"), - "memory": resource.MustParse("4000"), + v1.ResourceCPU: resource.MustParse("2000m"), + v1.ResourceMemory: resource.MustParse("4000"), }, }, }, { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse("3000m"), - "memory": resource.MustParse("5000"), + v1.ResourceCPU: resource.MustParse("3000m"), + v1.ResourceMemory: resource.MustParse("5000"), }, }, }, diff --git a/plugin/pkg/scheduler/algorithm/priorities/test_util.go b/plugin/pkg/scheduler/algorithm/priorities/test_util.go index fd21ea8ac2493..9eb26f2d93c99 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/test_util.go +++ b/plugin/pkg/scheduler/algorithm/priorities/test_util.go @@ -30,12 +30,12 @@ func makeNode(node string, milliCPU, memory int64) *v1.Node { ObjectMeta: metav1.ObjectMeta{Name: node}, Status: v1.NodeStatus{ Capacity: v1.ResourceList{ - "cpu": *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - "memory": *resource.NewQuantity(memory, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), }, Allocatable: v1.ResourceList{ - "cpu": *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - "memory": *resource.NewQuantity(memory, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), }, }, } diff --git a/plugin/pkg/scheduler/algorithm/priorities/util/topologies.go b/plugin/pkg/scheduler/algorithm/priorities/util/topologies.go index 2764d9a0acba2..511ffc137061f 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/util/topologies.go +++ b/plugin/pkg/scheduler/algorithm/priorities/util/topologies.go @@ -75,7 +75,6 @@ type Topologies struct { } // NodesHaveSameTopologyKey checks if nodeA and nodeB have same label value with given topologyKey as label key. -// If the topologyKey is empty, check if the two nodes have any of the default topologyKeys, and have same corresponding label value. func (tps *Topologies) NodesHaveSameTopologyKey(nodeA, nodeB *v1.Node, topologyKey string) bool { return NodesHaveSameTopologyKey(nodeA, nodeB, topologyKey) } diff --git a/plugin/pkg/scheduler/algorithm/types.go b/plugin/pkg/scheduler/algorithm/types.go index 905d3045524da..4223f69a8a2a6 100644 --- a/plugin/pkg/scheduler/algorithm/types.go +++ b/plugin/pkg/scheduler/algorithm/types.go @@ -80,6 +80,9 @@ type PodLister interface { // We explicitly return []*v1.Pod, instead of v1.PodList, to avoid // performing expensive copies that are unneeded. List(labels.Selector) ([]*v1.Pod, error) + // This is similar to "List()", but the returned slice does not + // contain pods that don't pass `podFilter`. + FilteredList(podFilter schedulercache.PodFilter, selector labels.Selector) ([]*v1.Pod, error) } // ServiceLister interface represents anything that can produce a list of services; the list is consumed by a scheduler. diff --git a/plugin/pkg/scheduler/core/generic_scheduler.go b/plugin/pkg/scheduler/core/generic_scheduler.go index 9e882ef9ec5cb..984c783b90987 100644 --- a/plugin/pkg/scheduler/core/generic_scheduler.go +++ b/plugin/pkg/scheduler/core/generic_scheduler.go @@ -315,10 +315,8 @@ func PrioritizeNodes( errs = append(errs, err) } - results := make([]schedulerapi.HostPriorityList, 0, len(priorityConfigs)) - for range priorityConfigs { - results = append(results, nil) - } + results := make([]schedulerapi.HostPriorityList, len(priorityConfigs), len(priorityConfigs)) + for i, priorityConfig := range priorityConfigs { if priorityConfig.Function != nil { // DEPRECATED diff --git a/plugin/pkg/scheduler/core/generic_scheduler_test.go b/plugin/pkg/scheduler/core/generic_scheduler_test.go index 48828b3bb98e8..f4abcaa505732 100644 --- a/plugin/pkg/scheduler/core/generic_scheduler_test.go +++ b/plugin/pkg/scheduler/core/generic_scheduler_test.go @@ -390,12 +390,12 @@ func makeNode(node string, milliCPU, memory int64) *v1.Node { ObjectMeta: metav1.ObjectMeta{Name: node}, Status: v1.NodeStatus{ Capacity: v1.ResourceList{ - "cpu": *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - "memory": *resource.NewQuantity(memory, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), }, Allocatable: v1.ResourceList{ - "cpu": *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), - "memory": *resource.NewQuantity(memory, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), }, }, } @@ -438,9 +438,9 @@ func TestZeroRequest(t *testing.T) { { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse( + v1.ResourceCPU: resource.MustParse( strconv.FormatInt(priorityutil.DefaultMilliCpuRequest, 10) + "m"), - "memory": resource.MustParse( + v1.ResourceMemory: resource.MustParse( strconv.FormatInt(priorityutil.DefaultMemoryRequest, 10)), }, }, @@ -455,9 +455,9 @@ func TestZeroRequest(t *testing.T) { { Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - "cpu": resource.MustParse( + v1.ResourceCPU: resource.MustParse( strconv.FormatInt(priorityutil.DefaultMilliCpuRequest*3, 10) + "m"), - "memory": resource.MustParse( + v1.ResourceMemory: resource.MustParse( strconv.FormatInt(priorityutil.DefaultMemoryRequest*3, 10)), }, }, diff --git a/plugin/pkg/scheduler/factory/factory.go b/plugin/pkg/scheduler/factory/factory.go index cd58377635a57..ae9cd535bd36c 100644 --- a/plugin/pkg/scheduler/factory/factory.go +++ b/plugin/pkg/scheduler/factory/factory.go @@ -570,6 +570,12 @@ func (c *ConfigFactory) invalidateCachedPredicatesOnNodeUpdate(newNode *v1.Node, if oldConditions[v1.NodeDiskPressure] != newConditions[v1.NodeDiskPressure] { invalidPredicates.Insert("CheckNodeDiskPressure") } + if oldConditions[v1.NodeReady] != newConditions[v1.NodeReady] || + oldConditions[v1.NodeOutOfDisk] != newConditions[v1.NodeOutOfDisk] || + oldConditions[v1.NodeNetworkUnavailable] != newConditions[v1.NodeNetworkUnavailable] || + newNode.Spec.Unschedulable != oldNode.Spec.Unschedulable { + invalidPredicates.Insert("CheckNodeCondition") + } } c.equivalencePodCache.InvalidateCachedPredicateItem(newNode.GetName(), invalidPredicates) } diff --git a/plugin/pkg/scheduler/factory/plugins.go b/plugin/pkg/scheduler/factory/plugins.go index 48cdca881a456..bb4db34610643 100644 --- a/plugin/pkg/scheduler/factory/plugins.go +++ b/plugin/pkg/scheduler/factory/plugins.go @@ -142,7 +142,7 @@ func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy) string { ) // Once we generate the predicate we should also Register the Precomputation - predicates.RegisterPredicatePrecomputation(policy.Name, precomputationFunction) + predicates.RegisterPredicateMetadataProducer(policy.Name, precomputationFunction) return predicate } } else if policy.Argument.LabelsPresence != nil { diff --git a/plugin/pkg/scheduler/schedulercache/BUILD b/plugin/pkg/scheduler/schedulercache/BUILD index 596bf7858a7d6..d1f213cd7d842 100644 --- a/plugin/pkg/scheduler/schedulercache/BUILD +++ b/plugin/pkg/scheduler/schedulercache/BUILD @@ -17,6 +17,7 @@ go_library( deps = [ "//pkg/api/v1/helper:go_default_library", "//plugin/pkg/scheduler/algorithm/priorities/util:go_default_library", + "//plugin/pkg/scheduler/util:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", diff --git a/plugin/pkg/scheduler/schedulercache/cache.go b/plugin/pkg/scheduler/schedulercache/cache.go index 61720ee8041bb..40c85eb99ab84 100644 --- a/plugin/pkg/scheduler/schedulercache/cache.go +++ b/plugin/pkg/scheduler/schedulercache/cache.go @@ -93,12 +93,17 @@ func (cache *schedulerCache) UpdateNodeNameToInfoMap(nodeNameToInfo map[string]* } func (cache *schedulerCache) List(selector labels.Selector) ([]*v1.Pod, error) { + alwaysTrue := func(p *v1.Pod) bool { return true } + return cache.FilteredList(alwaysTrue, selector) +} + +func (cache *schedulerCache) FilteredList(podFilter PodFilter, selector labels.Selector) ([]*v1.Pod, error) { cache.mu.Lock() defer cache.mu.Unlock() var pods []*v1.Pod for _, info := range cache.nodes { for _, pod := range info.pods { - if selector.Matches(labels.Set(pod.Labels)) { + if podFilter(pod) && selector.Matches(labels.Set(pod.Labels)) { pods = append(pods, pod) } } diff --git a/plugin/pkg/scheduler/schedulercache/interface.go b/plugin/pkg/scheduler/schedulercache/interface.go index 8a26218b279fb..6acb92cd53dd4 100644 --- a/plugin/pkg/scheduler/schedulercache/interface.go +++ b/plugin/pkg/scheduler/schedulercache/interface.go @@ -21,6 +21,8 @@ import ( "k8s.io/apimachinery/pkg/labels" ) +type PodFilter func(*v1.Pod) bool + // Cache collects pods' information and provides node-level aggregated information. // It's intended for generic scheduler to do efficient lookup. // Cache's operations are pod centric. It does incremental updates based on pod events. @@ -93,4 +95,7 @@ type Cache interface { // List lists all cached pods (including assumed ones). List(labels.Selector) ([]*v1.Pod, error) + + // FilteredList returns all cached pods that pass the filter. + FilteredList(filter PodFilter, selector labels.Selector) ([]*v1.Pod, error) } diff --git a/plugin/pkg/scheduler/schedulercache/node_info.go b/plugin/pkg/scheduler/schedulercache/node_info.go index 6aa97213e0181..dd3f8206b0908 100644 --- a/plugin/pkg/scheduler/schedulercache/node_info.go +++ b/plugin/pkg/scheduler/schedulercache/node_info.go @@ -26,6 +26,7 @@ import ( clientcache "k8s.io/client-go/tools/cache" v1helper "k8s.io/kubernetes/pkg/api/v1/helper" priorityutil "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities/util" + "k8s.io/kubernetes/plugin/pkg/scheduler/util" ) var emptyResource = Resource{} @@ -63,11 +64,10 @@ type NodeInfo struct { // Resource is a collection of compute resource. type Resource struct { - MilliCPU int64 - Memory int64 - NvidiaGPU int64 - StorageScratch int64 - StorageOverlay int64 + MilliCPU int64 + Memory int64 + NvidiaGPU int64 + EphemeralStorage int64 // We store allowedPodNumber (which is Node.Status.Allocatable.Pods().Value()) // explicitly as int, to avoid conversions and improve performance. AllowedPodNumber int @@ -97,10 +97,8 @@ func (r *Resource) Add(rl v1.ResourceList) { r.NvidiaGPU += rQuant.Value() case v1.ResourcePods: r.AllowedPodNumber += int(rQuant.Value()) - case v1.ResourceStorageScratch: - r.StorageScratch += rQuant.Value() - case v1.ResourceStorageOverlay: - r.StorageOverlay += rQuant.Value() + case v1.ResourceEphemeralStorage: + r.EphemeralStorage += rQuant.Value() default: if v1helper.IsExtendedResourceName(rName) { r.AddExtended(rName, rQuant.Value()) @@ -111,12 +109,11 @@ func (r *Resource) Add(rl v1.ResourceList) { func (r *Resource) ResourceList() v1.ResourceList { result := v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(r.MilliCPU, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(r.Memory, resource.BinarySI), - v1.ResourceNvidiaGPU: *resource.NewQuantity(r.NvidiaGPU, resource.DecimalSI), - v1.ResourcePods: *resource.NewQuantity(int64(r.AllowedPodNumber), resource.BinarySI), - v1.ResourceStorageOverlay: *resource.NewQuantity(r.StorageOverlay, resource.BinarySI), - v1.ResourceStorageScratch: *resource.NewQuantity(r.StorageScratch, resource.BinarySI), + v1.ResourceCPU: *resource.NewMilliQuantity(r.MilliCPU, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(r.Memory, resource.BinarySI), + v1.ResourceNvidiaGPU: *resource.NewQuantity(r.NvidiaGPU, resource.DecimalSI), + v1.ResourcePods: *resource.NewQuantity(int64(r.AllowedPodNumber), resource.BinarySI), + v1.ResourceEphemeralStorage: *resource.NewQuantity(r.EphemeralStorage, resource.BinarySI), } for rName, rQuant := range r.ExtendedResources { result[rName] = *resource.NewQuantity(rQuant, resource.DecimalSI) @@ -130,8 +127,7 @@ func (r *Resource) Clone() *Resource { Memory: r.Memory, NvidiaGPU: r.NvidiaGPU, AllowedPodNumber: r.AllowedPodNumber, - StorageOverlay: r.StorageOverlay, - StorageScratch: r.StorageScratch, + EphemeralStorage: r.EphemeralStorage, } if r.ExtendedResources != nil { res.ExtendedResources = make(map[v1.ResourceName]int64) @@ -304,8 +300,7 @@ func (n *NodeInfo) addPod(pod *v1.Pod) { n.requestedResource.MilliCPU += res.MilliCPU n.requestedResource.Memory += res.Memory n.requestedResource.NvidiaGPU += res.NvidiaGPU - n.requestedResource.StorageOverlay += res.StorageOverlay - n.requestedResource.StorageScratch += res.StorageScratch + n.requestedResource.EphemeralStorage += res.EphemeralStorage if n.requestedResource.ExtendedResources == nil && len(res.ExtendedResources) > 0 { n.requestedResource.ExtendedResources = map[v1.ResourceName]int64{} } @@ -392,14 +387,6 @@ func calculateResource(pod *v1.Pod) (res Resource, non0_cpu int64, non0_mem int6 // No non-zero resources for GPUs or opaque resources. } - // Account for storage requested by emptydir volumes - // If the storage medium is memory, should exclude the size - for _, vol := range pod.Spec.Volumes { - if vol.EmptyDir != nil && vol.EmptyDir.Medium != v1.StorageMediumMemory { - res.StorageScratch += vol.EmptyDir.SizeLimit.Value() - } - } - return } @@ -458,3 +445,19 @@ func (n *NodeInfo) RemoveNode(node *v1.Node) error { func getPodKey(pod *v1.Pod) (string, error) { return clientcache.MetaNamespaceKeyFunc(pod) } + +// Filter implements PodFilter interface. It returns false only if the pod node name +// matches NodeInfo.node and the pod is not found in the pods list. Otherwise, +// returns true. +func (n *NodeInfo) Filter(pod *v1.Pod) bool { + pFullName := util.GetPodFullName(pod) + if pod.Spec.NodeName != n.node.Name { + return true + } + for _, p := range n.pods { + if util.GetPodFullName(p) == pFullName { + return true + } + } + return false +} diff --git a/plugin/pkg/scheduler/testing/fake_cache.go b/plugin/pkg/scheduler/testing/fake_cache.go index 5b03e30241b9f..b85b165aa3d92 100644 --- a/plugin/pkg/scheduler/testing/fake_cache.go +++ b/plugin/pkg/scheduler/testing/fake_cache.go @@ -57,3 +57,7 @@ func (f *FakeCache) UpdateNodeNameToInfoMap(infoMap map[string]*schedulercache.N } func (f *FakeCache) List(s labels.Selector) ([]*v1.Pod, error) { return nil, nil } + +func (f *FakeCache) FilteredList(filter schedulercache.PodFilter, selector labels.Selector) ([]*v1.Pod, error) { + return nil, nil +} diff --git a/plugin/pkg/scheduler/testing/fake_lister.go b/plugin/pkg/scheduler/testing/fake_lister.go index 8f6aa8fd9bced..35f763087fe13 100644 --- a/plugin/pkg/scheduler/testing/fake_lister.go +++ b/plugin/pkg/scheduler/testing/fake_lister.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" . "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" + "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" ) var _ NodeLister = &FakeNodeLister{} @@ -52,6 +53,15 @@ func (f FakePodLister) List(s labels.Selector) (selected []*v1.Pod, err error) { return selected, nil } +func (f FakePodLister) FilteredList(podFilter schedulercache.PodFilter, s labels.Selector) (selected []*v1.Pod, err error) { + for _, pod := range f { + if podFilter(pod) && s.Matches(labels.Set(pod.Labels)) { + selected = append(selected, pod) + } + } + return selected, nil +} + var _ ServiceLister = &FakeServiceLister{} // FakeServiceLister implements ServiceLister on []v1.Service for test purposes. diff --git a/plugin/pkg/scheduler/util/utils.go b/plugin/pkg/scheduler/util/utils.go index 81d7d7c1d5eac..a230cf55fa637 100644 --- a/plugin/pkg/scheduler/util/utils.go +++ b/plugin/pkg/scheduler/util/utils.go @@ -39,3 +39,10 @@ func GetUsedPorts(pods ...*v1.Pod) map[int]bool { } return ports } + +// GetPodFullName returns a name that uniquely identifies a pod. +func GetPodFullName(pod *v1.Pod) string { + // Use underscore as the delimiter because it is not allowed in pod name + // (DNS subdomain format). + return pod.Name + "_" + pod.Namespace +} diff --git a/staging/BUILD b/staging/BUILD index 36ba8ac835f37..442a01c5ba089 100644 --- a/staging/BUILD +++ b/staging/BUILD @@ -196,25 +196,25 @@ filegroup( "//staging/src/k8s.io/client-go/util/retry:all-srcs", "//staging/src/k8s.io/client-go/util/testing:all-srcs", "//staging/src/k8s.io/client-go/util/workqueue:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/client-gen:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/conversion-gen:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/deepcopy-gen:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/defaulter-gen:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/go-to-protobuf:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/import-boss:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/informer-gen:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/lister-gen:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/openapi-gen:all-srcs", + "//staging/src/k8s.io/code-generator/cmd/set-gen:all-srcs", + "//staging/src/k8s.io/code-generator/test/apis/testgroup:all-srcs", + "//staging/src/k8s.io/code-generator/test/clientset/internal:all-srcs", + "//staging/src/k8s.io/code-generator/test/clientset/versioned:all-srcs", + "//staging/src/k8s.io/code-generator/test/informers/externalversions:all-srcs", + "//staging/src/k8s.io/code-generator/test/informers/internalversion:all-srcs", + "//staging/src/k8s.io/code-generator/test/listers/testgroup/internalversion:all-srcs", + "//staging/src/k8s.io/code-generator/test/listers/testgroup/v1:all-srcs", + "//staging/src/k8s.io/code-generator/third_party/forked/golang/reflect:all-srcs", "//staging/src/k8s.io/kube-aggregator:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/client-gen:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/conversion-gen:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/deepcopy-gen:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/defaulter-gen:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/go-to-protobuf:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/import-boss:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/informer-gen:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/lister-gen:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/openapi-gen:all-srcs", - "//staging/src/k8s.io/kube-gen/cmd/set-gen:all-srcs", - "//staging/src/k8s.io/kube-gen/test/apis/testgroup:all-srcs", - "//staging/src/k8s.io/kube-gen/test/clientset/internal:all-srcs", - "//staging/src/k8s.io/kube-gen/test/clientset/versioned:all-srcs", - "//staging/src/k8s.io/kube-gen/test/informers/externalversions:all-srcs", - "//staging/src/k8s.io/kube-gen/test/informers/internalversion:all-srcs", - "//staging/src/k8s.io/kube-gen/test/listers/testgroup/internalversion:all-srcs", - "//staging/src/k8s.io/kube-gen/test/listers/testgroup/v1:all-srcs", - "//staging/src/k8s.io/kube-gen/third_party/forked/golang/reflect:all-srcs", "//staging/src/k8s.io/metrics/pkg/apis/custom_metrics:all-srcs", "//staging/src/k8s.io/metrics/pkg/apis/metrics:all-srcs", "//staging/src/k8s.io/metrics/pkg/client/clientset_generated/clientset:all-srcs", diff --git a/staging/README.md b/staging/README.md index ca9b88ed3a4dc..f750d3147c441 100644 --- a/staging/README.md +++ b/staging/README.md @@ -12,7 +12,7 @@ Repositories currently staged here: - [`k8s.io/apiserver`](https://github.com/kubernetes/apiserver) - [`k8s.io/client-go`](https://github.com/kubernetes/client-go) - [`k8s.io/kube-aggregator`](https://github.com/kubernetes/kube-aggregator) -- [`k8s.io/kube-gen`](https://github.com/kubernetes/kube-gen) (about to be published) +- [`k8s.io/code-generator`](https://github.com/kubernetes/code-generator) (about to be published) - [`k8s.io/metrics`](https://github.com/kubernetes/metrics) - [`k8s.io/sample-apiserver`](https://github.com/kubernetes/sample-apiserver) diff --git a/staging/prime-apimachinery.sh b/staging/prime-apimachinery.sh index 0a05d7c8f24ba..c9d4e96fb11ef 100755 --- a/staging/prime-apimachinery.sh +++ b/staging/prime-apimachinery.sh @@ -72,7 +72,7 @@ while read package; do done <${dir}/packages.txt # this file generates something or other, but we don't want to accidentally have it generate into an apimachinery package -git checkout vendor/k8s.io/kube-gen/cmd/set-gen/main.go +git checkout vendor/k8s.io/code-generator/cmd/set-gen/main.go # now run gofmt to get the sorting right diff --git a/staging/src/k8s.io/api/Godeps/Godeps.json b/staging/src/k8s.io/api/Godeps/Godeps.json index 0da1215b6b451..b4ec17ea4fb74 100644 --- a/staging/src/k8s.io/api/Godeps/Godeps.json +++ b/staging/src/k8s.io/api/Godeps/Godeps.json @@ -14,10 +14,6 @@ "ImportPath": "github.com/PuerkitoBio/urlesc", "Rev": "5bd2802263f21d8788851d5305584c82a5c75d7e" }, - { - "ImportPath": "github.com/davecgh/go-spew/spew", - "Rev": "782f4967f2dc4564575ca782fe2d04090b5faca8" - }, { "ImportPath": "github.com/emicklei/go-restful", "Rev": "ff4f55a206334ef123e4f79bbf348980da81ca46" @@ -26,10 +22,6 @@ "ImportPath": "github.com/emicklei/go-restful/log", "Rev": "ff4f55a206334ef123e4f79bbf348980da81ca46" }, - { - "ImportPath": "github.com/ghodss/yaml", - "Rev": "73d445a93680fa1a78ae23a5839bad48f32ba1ee" - }, { "ImportPath": "github.com/go-openapi/jsonpointer", "Rev": "46af16f9f7b149af66e5d1bd010e3574dc06de98" @@ -84,67 +76,67 @@ }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/text/cases", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" + }, + { + "ImportPath": "golang.org/x/text/internal", + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/internal/tag", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/language", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/runes", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/bidirule", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/precis", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/transform", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/bidi", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/norm", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/width", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "gopkg.in/inf.v0", "Rev": "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4" - }, - { - "ImportPath": "gopkg.in/yaml.v2", - "Rev": "53feefa2559fb8dfa8d81baad31be332c97d6c77" } ] } diff --git a/staging/src/k8s.io/api/OWNERS b/staging/src/k8s.io/api/OWNERS index b397c6953ebe3..cfc4c438e818a 100644 --- a/staging/src/k8s.io/api/OWNERS +++ b/staging/src/k8s.io/api/OWNERS @@ -3,6 +3,8 @@ approvers: - lavalamp - smarterclayton - thockin +- liggitt +# - bgrant0607 # manual escalations only reviewers: - brendandburns - caesarxuchao diff --git a/staging/src/k8s.io/api/admission/v1alpha1/BUILD b/staging/src/k8s.io/api/admission/v1alpha1/BUILD index 389b56c3a9dbb..714b425a4a1e3 100644 --- a/staging/src/k8s.io/api/admission/v1alpha1/BUILD +++ b/staging/src/k8s.io/api/admission/v1alpha1/BUILD @@ -25,7 +25,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", ], ) diff --git a/staging/src/k8s.io/api/admission/v1alpha1/generated.pb.go b/staging/src/k8s.io/api/admission/v1alpha1/generated.pb.go index 9d3ea2eb58313..03fa1f6be8d1a 100644 --- a/staging/src/k8s.io/api/admission/v1alpha1/generated.pb.go +++ b/staging/src/k8s.io/api/admission/v1alpha1/generated.pb.go @@ -37,8 +37,6 @@ import math "math" import k8s_io_apimachinery_pkg_apis_meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -import k8s_io_apiserver_pkg_admission "k8s.io/apiserver/pkg/admission" - import strings "strings" import reflect "reflect" @@ -605,7 +603,7 @@ func (m *AdmissionReviewSpec) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Operation = k8s_io_apiserver_pkg_admission.Operation(dAtA[iNdEx:postIndex]) + m.Operation = Operation(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 5: if wireType != 2 { @@ -988,47 +986,46 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 663 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x94, 0xcf, 0x4b, 0x1c, 0x49, - 0x14, 0xc7, 0xa7, 0x77, 0xc7, 0x71, 0xa6, 0x5c, 0xd6, 0xdd, 0x92, 0x85, 0x46, 0xd8, 0x56, 0x3c, - 0x2c, 0x2e, 0x68, 0x35, 0x1a, 0x23, 0x21, 0x90, 0x83, 0x03, 0x49, 0x08, 0x01, 0x0d, 0xa5, 0x86, - 0x90, 0x84, 0x40, 0x4d, 0xcf, 0x73, 0xa6, 0x32, 0xd3, 0x55, 0x4d, 0x55, 0xf5, 0x98, 0xdc, 0xf2, - 0x27, 0xe4, 0x90, 0x7f, 0x29, 0xe0, 0x25, 0xe0, 0xd1, 0x93, 0xc4, 0xc9, 0x7f, 0x91, 0x53, 0xe8, - 0xea, 0xea, 0x6e, 0x1d, 0x35, 0x89, 0x39, 0xcd, 0xbc, 0x1f, 0xdf, 0x4f, 0xbd, 0x7a, 0xf5, 0x5e, - 0xa3, 0x07, 0x83, 0x3b, 0x9a, 0x70, 0x19, 0x0e, 0xd2, 0x0e, 0x28, 0x01, 0x06, 0x74, 0x38, 0x02, - 0xd1, 0x95, 0x2a, 0x74, 0x01, 0x96, 0xf0, 0x90, 0x75, 0x63, 0xae, 0x35, 0x97, 0x22, 0x1c, 0xad, - 0xb1, 0x61, 0xd2, 0x67, 0x6b, 0x61, 0x0f, 0x04, 0x28, 0x66, 0xa0, 0x4b, 0x12, 0x25, 0x8d, 0xc4, - 0xff, 0xe6, 0xe9, 0x84, 0x25, 0x9c, 0x94, 0xe9, 0xa4, 0x48, 0x9f, 0x5f, 0xed, 0x71, 0xd3, 0x4f, - 0x3b, 0x24, 0x92, 0x71, 0xd8, 0x93, 0x3d, 0x19, 0x5a, 0x55, 0x27, 0x3d, 0xb0, 0x96, 0x35, 0xec, - 0xbf, 0x9c, 0x36, 0xbf, 0x72, 0xfe, 0xf0, 0xd4, 0xf4, 0x41, 0x18, 0x1e, 0x31, 0x93, 0x57, 0x30, - 0x79, 0xf6, 0xfc, 0x46, 0x95, 0x1d, 0xb3, 0xa8, 0xcf, 0x05, 0xa8, 0xb7, 0x61, 0x32, 0xe8, 0x65, - 0x0e, 0x1d, 0xc6, 0x60, 0xd8, 0x55, 0xaa, 0xf0, 0x3a, 0x95, 0x4a, 0x85, 0xe1, 0x31, 0x5c, 0x12, - 0x6c, 0xfe, 0x48, 0xa0, 0xa3, 0x3e, 0xc4, 0xec, 0x92, 0xee, 0xd6, 0x75, 0xba, 0xd4, 0xf0, 0x61, - 0xc8, 0x85, 0xd1, 0x46, 0x4d, 0x8a, 0x96, 0x3e, 0x7a, 0x68, 0x76, 0xab, 0xe8, 0x23, 0x85, 0x11, - 0x87, 0x43, 0xbc, 0x87, 0xea, 0x3a, 0x81, 0xc8, 0xf7, 0x16, 0xbd, 0xe5, 0x99, 0xf5, 0x75, 0xf2, - 0xdd, 0x96, 0x93, 0x09, 0xf5, 0x6e, 0x02, 0x51, 0xfb, 0x8f, 0xa3, 0xd3, 0x85, 0xda, 0xf8, 0x74, - 0xa1, 0x9e, 0x59, 0xd4, 0xd2, 0xf0, 0x4b, 0xd4, 0xd0, 0x86, 0x99, 0x54, 0xfb, 0xbf, 0x59, 0xee, - 0xc6, 0x0d, 0xb9, 0x56, 0xdb, 0xfe, 0xd3, 0x91, 0x1b, 0xb9, 0x4d, 0x1d, 0x73, 0xe9, 0xd3, 0x14, - 0x9a, 0xbb, 0xa2, 0x12, 0xfc, 0x0c, 0xd5, 0x07, 0x5c, 0x74, 0xdd, 0x5d, 0x36, 0xcf, 0x9d, 0x59, - 0xf6, 0x88, 0x24, 0x83, 0x5e, 0xe6, 0xd0, 0x24, 0x7b, 0x42, 0x32, 0x5a, 0x23, 0x0f, 0x95, 0x4c, - 0x93, 0xa7, 0xa0, 0x32, 0xd6, 0x63, 0x2e, 0xba, 0xd5, 0x7d, 0x32, 0x8b, 0x5a, 0x22, 0xde, 0x47, - 0x0d, 0xd9, 0x79, 0x0d, 0x91, 0x71, 0xf7, 0x59, 0xbd, 0x96, 0xed, 0xde, 0x8d, 0x50, 0x76, 0x78, - 0xff, 0x8d, 0x01, 0x91, 0x61, 0xab, 0x8b, 0xec, 0x58, 0x08, 0x75, 0x30, 0xfc, 0x0a, 0xb5, 0xe4, - 0xb0, 0x9b, 0x3b, 0xfd, 0xdf, 0x7f, 0x85, 0xfc, 0xb7, 0x23, 0xb7, 0x76, 0x0a, 0x0e, 0xad, 0x90, - 0xf8, 0x05, 0x6a, 0xc9, 0x24, 0x1b, 0x01, 0x2e, 0x85, 0x5f, 0x5f, 0xf4, 0x96, 0x5b, 0xed, 0x7b, - 0xa5, 0xa0, 0x08, 0x7c, 0x3d, 0x5d, 0x58, 0xae, 0xa6, 0x49, 0x83, 0x1a, 0x81, 0xca, 0x27, 0xbd, - 0x7c, 0xa7, 0x32, 0x97, 0x56, 0x3c, 0xbc, 0x88, 0xea, 0x82, 0xc5, 0xe0, 0x4f, 0x59, 0x6e, 0xd9, - 0xb5, 0x6d, 0x16, 0x03, 0xb5, 0x11, 0x1c, 0xa2, 0x56, 0xf6, 0xab, 0x13, 0x16, 0x81, 0xdf, 0xb0, - 0x69, 0x65, 0xbd, 0xdb, 0x45, 0x80, 0x56, 0x39, 0xb8, 0x8f, 0x9a, 0x0a, 0xb4, 0x4c, 0x55, 0x04, - 0xfe, 0xb4, 0x6d, 0xc7, 0xdd, 0x9b, 0x3f, 0x22, 0x75, 0x84, 0xf6, 0x5f, 0xee, 0xac, 0x66, 0xe1, - 0xa1, 0x25, 0x1d, 0xdf, 0x46, 0x33, 0x3a, 0xed, 0x14, 0x01, 0xbf, 0x69, 0x8b, 0x9b, 0x73, 0x82, - 0x99, 0xdd, 0x2a, 0x44, 0xcf, 0xe7, 0xe1, 0x3d, 0xd4, 0x4c, 0x35, 0xa8, 0x47, 0xe2, 0x40, 0xfa, - 0x2d, 0x5b, 0xe0, 0x7f, 0x17, 0x26, 0xfb, 0xc2, 0x67, 0x25, 0x2b, 0x6c, 0xdf, 0x65, 0x57, 0xc5, - 0x14, 0x1e, 0x5a, 0x92, 0x96, 0x3e, 0x78, 0xe8, 0x9f, 0x2b, 0x37, 0x00, 0xff, 0x8f, 0xa6, 0xd9, - 0x70, 0x28, 0x0f, 0x21, 0x1f, 0xea, 0x66, 0x7b, 0xd6, 0x61, 0xa6, 0xb7, 0x72, 0x37, 0x2d, 0xe2, - 0xf8, 0xc9, 0xc4, 0xca, 0xad, 0xfc, 0x5c, 0xe7, 0xdc, 0xaa, 0xa1, 0x6c, 0x3a, 0x29, 0xe8, 0x74, - 0x68, 0x8a, 0x35, 0x6b, 0x93, 0xa3, 0xb3, 0xa0, 0x76, 0x7c, 0x16, 0xd4, 0x4e, 0xce, 0x82, 0xda, - 0xbb, 0x71, 0xe0, 0x1d, 0x8d, 0x03, 0xef, 0x78, 0x1c, 0x78, 0x27, 0xe3, 0xc0, 0xfb, 0x3c, 0x0e, - 0xbc, 0xf7, 0x5f, 0x82, 0xda, 0xf3, 0x66, 0xb1, 0xc4, 0xdf, 0x02, 0x00, 0x00, 0xff, 0xff, 0x42, - 0x53, 0xac, 0x65, 0xf7, 0x05, 0x00, 0x00, + // 645 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x94, 0xcf, 0x4f, 0x13, 0x41, + 0x14, 0xc7, 0xbb, 0x5a, 0x4a, 0x3b, 0x18, 0xd1, 0x21, 0x26, 0x1b, 0x12, 0x17, 0xc2, 0xc1, 0x60, + 0x02, 0xb3, 0x01, 0x91, 0x18, 0xe3, 0x85, 0x26, 0x6a, 0x8c, 0x09, 0x98, 0x01, 0x8c, 0x31, 0xc6, + 0x64, 0xba, 0x7d, 0xb4, 0x63, 0xbb, 0x33, 0x9b, 0x9d, 0xd9, 0xa2, 0x37, 0xff, 0x04, 0x0f, 0xfe, + 0x1d, 0xfe, 0x17, 0x26, 0x1c, 0x39, 0x72, 0x22, 0x52, 0xff, 0x0b, 0x4f, 0x66, 0x67, 0x67, 0x77, + 0x4b, 0xa1, 0x2a, 0x9e, 0xda, 0xf7, 0xe3, 0xfb, 0x99, 0xf7, 0xde, 0xbc, 0x59, 0xf4, 0xac, 0xf7, + 0x48, 0x11, 0x2e, 0xfd, 0x5e, 0xd2, 0x82, 0x58, 0x80, 0x06, 0xe5, 0x0f, 0x40, 0xb4, 0x65, 0xec, + 0xdb, 0x00, 0x8b, 0xb8, 0xcf, 0xda, 0x21, 0x57, 0x8a, 0x4b, 0xe1, 0x0f, 0xd6, 0x58, 0x3f, 0xea, + 0xb2, 0x35, 0xbf, 0x03, 0x02, 0x62, 0xa6, 0xa1, 0x4d, 0xa2, 0x58, 0x6a, 0x89, 0xef, 0x66, 0xe9, + 0x84, 0x45, 0x9c, 0x14, 0xe9, 0x24, 0x4f, 0x9f, 0x5f, 0xed, 0x70, 0xdd, 0x4d, 0x5a, 0x24, 0x90, + 0xa1, 0xdf, 0x91, 0x1d, 0xe9, 0x1b, 0x55, 0x2b, 0x39, 0x30, 0x96, 0x31, 0xcc, 0xbf, 0x8c, 0x36, + 0xbf, 0x32, 0x7a, 0x78, 0xa2, 0xbb, 0x20, 0x34, 0x0f, 0x98, 0xce, 0x2a, 0x18, 0x3f, 0x7b, 0x7e, + 0xa3, 0xcc, 0x0e, 0x59, 0xd0, 0xe5, 0x02, 0xe2, 0x4f, 0x7e, 0xd4, 0xeb, 0xa4, 0x0e, 0xe5, 0x87, + 0xa0, 0xd9, 0x65, 0x2a, 0x7f, 0x92, 0x2a, 0x4e, 0x84, 0xe6, 0x21, 0x5c, 0x10, 0x6c, 0xfe, 0x4d, + 0xa0, 0x82, 0x2e, 0x84, 0xec, 0x82, 0xee, 0xc1, 0x24, 0x5d, 0xa2, 0x79, 0xdf, 0xe7, 0x42, 0x2b, + 0x1d, 0x8f, 0x8b, 0x96, 0xbe, 0x3b, 0x68, 0x76, 0x2b, 0x9f, 0x23, 0x85, 0x01, 0x87, 0x43, 0xbc, + 0x87, 0xaa, 0x2a, 0x82, 0xc0, 0x75, 0x16, 0x9d, 0xe5, 0x99, 0xf5, 0x75, 0xf2, 0xc7, 0x91, 0x93, + 0x31, 0xf5, 0x6e, 0x04, 0x41, 0xf3, 0xc6, 0xd1, 0xe9, 0x42, 0x65, 0x78, 0xba, 0x50, 0x4d, 0x2d, + 0x6a, 0x68, 0xf8, 0x1d, 0xaa, 0x29, 0xcd, 0x74, 0xa2, 0xdc, 0x6b, 0x86, 0xbb, 0x71, 0x45, 0xae, + 0xd1, 0x36, 0x6f, 0x5a, 0x72, 0x2d, 0xb3, 0xa9, 0x65, 0x2e, 0x7d, 0x9b, 0x42, 0x73, 0x97, 0x54, + 0x82, 0xdf, 0xa0, 0x6a, 0x8f, 0x8b, 0xb6, 0xed, 0x65, 0x73, 0xe4, 0xcc, 0x62, 0x46, 0x24, 0xea, + 0x75, 0x52, 0x87, 0x22, 0xe9, 0x15, 0x92, 0xc1, 0x1a, 0x79, 0x1e, 0xcb, 0x24, 0x7a, 0x0d, 0x71, + 0xca, 0x7a, 0xc9, 0x45, 0xbb, 0xec, 0x27, 0xb5, 0xa8, 0x21, 0xe2, 0x7d, 0x54, 0x93, 0xad, 0x0f, + 0x10, 0x68, 0xdb, 0xcf, 0xea, 0x44, 0xb6, 0xbd, 0x37, 0x42, 0xd9, 0xe1, 0xd3, 0x8f, 0x1a, 0x44, + 0x8a, 0x2d, 0x1b, 0xd9, 0x31, 0x10, 0x6a, 0x61, 0xf8, 0x3d, 0x6a, 0xc8, 0x7e, 0x3b, 0x73, 0xba, + 0xd7, 0xff, 0x87, 0x7c, 0xdb, 0x92, 0x1b, 0x3b, 0x39, 0x87, 0x96, 0x48, 0xfc, 0x04, 0x35, 0x64, + 0x94, 0xae, 0x00, 0x97, 0xc2, 0xad, 0x2e, 0x3a, 0xcb, 0x8d, 0xa6, 0x57, 0x08, 0xf2, 0xc0, 0xaf, + 0x51, 0x83, 0x96, 0x02, 0xbc, 0x88, 0xaa, 0x82, 0x85, 0xe0, 0x4e, 0x19, 0x61, 0x31, 0x96, 0x6d, + 0x16, 0x02, 0x35, 0x11, 0xec, 0xa3, 0x46, 0xfa, 0xab, 0x22, 0x16, 0x80, 0x5b, 0x33, 0x69, 0x45, + 0x41, 0xdb, 0x79, 0x80, 0x96, 0x39, 0xb8, 0x8b, 0xea, 0x31, 0x28, 0x99, 0xc4, 0x01, 0xb8, 0xd3, + 0xa6, 0xdf, 0xc7, 0x57, 0xbf, 0x25, 0x6a, 0x09, 0xcd, 0x5b, 0xf6, 0xac, 0x7a, 0xee, 0xa1, 0x05, + 0x1d, 0x3f, 0x44, 0x33, 0x2a, 0x69, 0xe5, 0x01, 0xb7, 0x6e, 0x8a, 0x9b, 0xb3, 0x82, 0x99, 0xdd, + 0x32, 0x44, 0x47, 0xf3, 0xf0, 0x1e, 0xaa, 0x27, 0x0a, 0xe2, 0x17, 0xe2, 0x40, 0xba, 0x0d, 0x53, + 0xe0, 0xbd, 0x73, 0xab, 0x7b, 0xee, 0xbb, 0x91, 0x16, 0xb6, 0x6f, 0xb3, 0xcb, 0x62, 0x72, 0x0f, + 0x2d, 0x48, 0x4b, 0x5f, 0x1d, 0x74, 0xe7, 0xd2, 0x15, 0xc7, 0xf7, 0xd1, 0x34, 0xeb, 0xf7, 0xe5, + 0x21, 0x64, 0x5b, 0x5b, 0x6f, 0xce, 0x5a, 0xcc, 0xf4, 0x56, 0xe6, 0xa6, 0x79, 0x1c, 0xbf, 0x1a, + 0x7b, 0x53, 0x2b, 0xff, 0x36, 0x39, 0xfb, 0x96, 0x50, 0xba, 0x7e, 0x14, 0x54, 0xd2, 0xd7, 0xf9, + 0x3b, 0x6a, 0x92, 0xa3, 0x33, 0xaf, 0x72, 0x7c, 0xe6, 0x55, 0x4e, 0xce, 0xbc, 0xca, 0xe7, 0xa1, + 0xe7, 0x1c, 0x0d, 0x3d, 0xe7, 0x78, 0xe8, 0x39, 0x27, 0x43, 0xcf, 0xf9, 0x31, 0xf4, 0x9c, 0x2f, + 0x3f, 0xbd, 0xca, 0xdb, 0x7a, 0xfe, 0x4a, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa0, 0x57, 0xa7, + 0x50, 0xd8, 0x05, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/admission/v1alpha1/types.generated.go b/staging/src/k8s.io/api/admission/v1alpha1/types.generated.go index 6ca9a63bc4249..ee49a96c39f54 100644 --- a/staging/src/k8s.io/api/admission/v1alpha1/types.generated.go +++ b/staging/src/k8s.io/api/admission/v1alpha1/types.generated.go @@ -25,11 +25,10 @@ import ( "errors" "fmt" codec1978 "github.com/ugorji/go/codec" - pkg4_v1 "k8s.io/api/authentication/v1" + pkg3_v1 "k8s.io/api/authentication/v1" pkg1_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" pkg2_runtime "k8s.io/apimachinery/pkg/runtime" - pkg5_types "k8s.io/apimachinery/pkg/types" - pkg3_admission "k8s.io/apiserver/pkg/admission" + pkg4_types "k8s.io/apimachinery/pkg/types" "reflect" "runtime" ) @@ -64,12 +63,11 @@ func init() { panic(err) } if false { // reference the types, but skip this branch at build/run time - var v0 pkg4_v1.UserInfo + var v0 pkg3_v1.UserInfo var v1 pkg1_v1.TypeMeta var v2 pkg2_runtime.RawExtension - var v3 pkg5_types.UID - var v4 pkg3_admission.Operation - _, _, _, _, _ = v0, v1, v2, v3, v4 + var v3 pkg4_types.UID + _, _, _, _ = v0, v1, v2, v3 } } @@ -535,13 +533,7 @@ func (x *AdmissionReviewSpec) CodecEncodeSelf(e *codec1978.Encoder) { if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayElem1234) if yyq2[3] { - yym19 := z.EncBinary() - _ = yym19 - if false { - } else if z.HasExtensions() && z.EncExt(x.Operation) { - } else { - r.EncodeString(codecSelferC_UTF81234, string(x.Operation)) - } + x.Operation.CodecEncodeSelf(e) } else { r.EncodeString(codecSelferC_UTF81234, "") } @@ -550,13 +542,7 @@ func (x *AdmissionReviewSpec) CodecEncodeSelf(e *codec1978.Encoder) { z.EncSendContainerState(codecSelfer_containerMapKey1234) r.EncodeString(codecSelferC_UTF81234, string("operation")) z.EncSendContainerState(codecSelfer_containerMapValue1234) - yym20 := z.EncBinary() - _ = yym20 - if false { - } else if z.HasExtensions() && z.EncExt(x.Operation) { - } else { - r.EncodeString(codecSelferC_UTF81234, string(x.Operation)) - } + x.Operation.CodecEncodeSelf(e) } } if yyr2 || yy2arr2 { @@ -789,69 +775,63 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromMap(l int, d *codec1978.Decoder x.Operation = "" } else { yyv10 := &x.Operation - yym11 := z.DecBinary() - _ = yym11 - if false { - } else if z.HasExtensions() && z.DecExt(yyv10) { - } else { - *((*string)(yyv10)) = r.DecodeString() - } + yyv10.CodecDecodeSelf(d) } case "name": if r.TryDecodeAsNil() { x.Name = "" } else { - yyv12 := &x.Name - yym13 := z.DecBinary() - _ = yym13 + yyv11 := &x.Name + yym12 := z.DecBinary() + _ = yym12 if false { } else { - *((*string)(yyv12)) = r.DecodeString() + *((*string)(yyv11)) = r.DecodeString() } } case "namespace": if r.TryDecodeAsNil() { x.Namespace = "" } else { - yyv14 := &x.Namespace - yym15 := z.DecBinary() - _ = yym15 + yyv13 := &x.Namespace + yym14 := z.DecBinary() + _ = yym14 if false { } else { - *((*string)(yyv14)) = r.DecodeString() + *((*string)(yyv13)) = r.DecodeString() } } case "resource": if r.TryDecodeAsNil() { x.Resource = pkg1_v1.GroupVersionResource{} } else { - yyv16 := &x.Resource - yym17 := z.DecBinary() - _ = yym17 + yyv15 := &x.Resource + yym16 := z.DecBinary() + _ = yym16 if false { - } else if z.HasExtensions() && z.DecExt(yyv16) { + } else if z.HasExtensions() && z.DecExt(yyv15) { } else { - z.DecFallback(yyv16, false) + z.DecFallback(yyv15, false) } } case "subResource": if r.TryDecodeAsNil() { x.SubResource = "" } else { - yyv18 := &x.SubResource - yym19 := z.DecBinary() - _ = yym19 + yyv17 := &x.SubResource + yym18 := z.DecBinary() + _ = yym18 if false { } else { - *((*string)(yyv18)) = r.DecodeString() + *((*string)(yyv17)) = r.DecodeString() } } case "userInfo": if r.TryDecodeAsNil() { - x.UserInfo = pkg4_v1.UserInfo{} + x.UserInfo = pkg3_v1.UserInfo{} } else { - yyv20 := &x.UserInfo - yyv20.CodecDecodeSelf(d) + yyv19 := &x.UserInfo + yyv19.CodecDecodeSelf(d) } default: z.DecStructFieldNotFound(-1, yys3) @@ -864,16 +844,16 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decod var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj21 int - var yyb21 bool - var yyhl21 bool = l >= 0 - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + var yyj20 int + var yyb20 bool + var yyhl20 bool = l >= 0 + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -881,22 +861,22 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decod if r.TryDecodeAsNil() { x.Kind = pkg1_v1.GroupVersionKind{} } else { - yyv22 := &x.Kind - yym23 := z.DecBinary() - _ = yym23 + yyv21 := &x.Kind + yym22 := z.DecBinary() + _ = yym22 if false { - } else if z.HasExtensions() && z.DecExt(yyv22) { + } else if z.HasExtensions() && z.DecExt(yyv21) { } else { - z.DecFallback(yyv22, false) + z.DecFallback(yyv21, false) } } - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -904,24 +884,24 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decod if r.TryDecodeAsNil() { x.Object = pkg2_runtime.RawExtension{} } else { - yyv24 := &x.Object - yym25 := z.DecBinary() - _ = yym25 + yyv23 := &x.Object + yym24 := z.DecBinary() + _ = yym24 if false { - } else if z.HasExtensions() && z.DecExt(yyv24) { - } else if !yym25 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv24) + } else if z.HasExtensions() && z.DecExt(yyv23) { + } else if !yym24 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv23) } else { - z.DecFallback(yyv24, false) + z.DecFallback(yyv23, false) } } - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -929,24 +909,24 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decod if r.TryDecodeAsNil() { x.OldObject = pkg2_runtime.RawExtension{} } else { - yyv26 := &x.OldObject - yym27 := z.DecBinary() - _ = yym27 + yyv25 := &x.OldObject + yym26 := z.DecBinary() + _ = yym26 if false { - } else if z.HasExtensions() && z.DecExt(yyv26) { - } else if !yym27 && z.IsJSONHandle() { - z.DecJSONUnmarshal(yyv26) + } else if z.HasExtensions() && z.DecExt(yyv25) { + } else if !yym26 && z.IsJSONHandle() { + z.DecJSONUnmarshal(yyv25) } else { - z.DecFallback(yyv26, false) + z.DecFallback(yyv25, false) } } - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -954,22 +934,16 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decod if r.TryDecodeAsNil() { x.Operation = "" } else { - yyv28 := &x.Operation - yym29 := z.DecBinary() - _ = yym29 - if false { - } else if z.HasExtensions() && z.DecExt(yyv28) { - } else { - *((*string)(yyv28)) = r.DecodeString() - } + yyv27 := &x.Operation + yyv27.CodecDecodeSelf(d) } - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -977,21 +951,21 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decod if r.TryDecodeAsNil() { x.Name = "" } else { - yyv30 := &x.Name - yym31 := z.DecBinary() - _ = yym31 + yyv28 := &x.Name + yym29 := z.DecBinary() + _ = yym29 if false { } else { - *((*string)(yyv30)) = r.DecodeString() + *((*string)(yyv28)) = r.DecodeString() } } - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -999,21 +973,21 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decod if r.TryDecodeAsNil() { x.Namespace = "" } else { - yyv32 := &x.Namespace - yym33 := z.DecBinary() - _ = yym33 + yyv30 := &x.Namespace + yym31 := z.DecBinary() + _ = yym31 if false { } else { - *((*string)(yyv32)) = r.DecodeString() + *((*string)(yyv30)) = r.DecodeString() } } - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1021,22 +995,22 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decod if r.TryDecodeAsNil() { x.Resource = pkg1_v1.GroupVersionResource{} } else { - yyv34 := &x.Resource - yym35 := z.DecBinary() - _ = yym35 + yyv32 := &x.Resource + yym33 := z.DecBinary() + _ = yym33 if false { - } else if z.HasExtensions() && z.DecExt(yyv34) { + } else if z.HasExtensions() && z.DecExt(yyv32) { } else { - z.DecFallback(yyv34, false) + z.DecFallback(yyv32, false) } } - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1044,43 +1018,43 @@ func (x *AdmissionReviewSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decod if r.TryDecodeAsNil() { x.SubResource = "" } else { - yyv36 := &x.SubResource - yym37 := z.DecBinary() - _ = yym37 + yyv34 := &x.SubResource + yym35 := z.DecBinary() + _ = yym35 if false { } else { - *((*string)(yyv36)) = r.DecodeString() + *((*string)(yyv34)) = r.DecodeString() } } - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } z.DecSendContainerState(codecSelfer_containerArrayElem1234) if r.TryDecodeAsNil() { - x.UserInfo = pkg4_v1.UserInfo{} + x.UserInfo = pkg3_v1.UserInfo{} } else { - yyv38 := &x.UserInfo - yyv38.CodecDecodeSelf(d) + yyv36 := &x.UserInfo + yyv36.CodecDecodeSelf(d) } for { - yyj21++ - if yyhl21 { - yyb21 = yyj21 > l + yyj20++ + if yyhl20 { + yyb20 = yyj20 > l } else { - yyb21 = r.CheckBreak() + yyb20 = r.CheckBreak() } - if yyb21 { + if yyb20 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj21-1, "") + z.DecStructFieldNotFound(yyj20-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } @@ -1338,3 +1312,29 @@ func (x *AdmissionReviewStatus) codecDecodeSelfFromArray(l int, d *codec1978.Dec } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } + +func (x Operation) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x)) + } +} + +func (x *Operation) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + *((*string)(x)) = r.DecodeString() + } +} diff --git a/staging/src/k8s.io/api/admission/v1alpha1/types.go b/staging/src/k8s.io/api/admission/v1alpha1/types.go index c727f05292b79..5defadca6bdb6 100644 --- a/staging/src/k8s.io/api/admission/v1alpha1/types.go +++ b/staging/src/k8s.io/api/admission/v1alpha1/types.go @@ -20,7 +20,6 @@ import ( authenticationv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apiserver/pkg/admission" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -48,7 +47,7 @@ type AdmissionReviewSpec struct { // +optional OldObject runtime.RawExtension `json:"oldObject,omitempty" protobuf:"bytes,3,opt,name=oldObject"` // Operation is the operation being performed - Operation admission.Operation `json:"operation,omitempty" protobuf:"bytes,4,opt,name=operation"` + Operation Operation `json:"operation,omitempty" protobuf:"bytes,4,opt,name=operation"` // Name is the name of the object as presented in the request. On a CREATE operation, the client may omit name and // rely on the server to generate the name. If that is the case, this method will return the empty string. // +optional @@ -78,3 +77,14 @@ type AdmissionReviewStatus struct { // +optional Result *metav1.Status `json:"status,omitempty" protobuf:"bytes,2,opt,name=status"` } + +// Operation is the type of resource operation being checked for admission control +type Operation string + +// Operation constants +const ( + Create Operation = "CREATE" + Update Operation = "UPDATE" + Delete Operation = "DELETE" + Connect Operation = "CONNECT" +) diff --git a/staging/src/k8s.io/api/admissionregistration/v1alpha1/generated.pb.go b/staging/src/k8s.io/api/admissionregistration/v1alpha1/generated.pb.go index d062a980dbd8b..6a1977f8331e7 100644 --- a/staging/src/k8s.io/api/admissionregistration/v1alpha1/generated.pb.go +++ b/staging/src/k8s.io/api/admissionregistration/v1alpha1/generated.pb.go @@ -309,12 +309,6 @@ func (m *Initializer) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.FailurePolicy != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintGenerated(dAtA, i, uint64(len(*m.FailurePolicy))) - i += copy(dAtA[i:], *m.FailurePolicy) - } return i, nil } @@ -622,10 +616,6 @@ func (m *Initializer) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) } } - if m.FailurePolicy != nil { - l = len(*m.FailurePolicy) - n += 1 + l + sovGenerated(uint64(l)) - } return n } @@ -771,7 +761,6 @@ func (this *Initializer) String() string { s := strings.Join([]string{`&Initializer{`, `Name:` + fmt.Sprintf("%v", this.Name) + `,`, `Rules:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.Rules), "Rule", "Rule", 1), `&`, ``, 1) + `,`, - `FailurePolicy:` + valueToStringGenerated(this.FailurePolicy) + `,`, `}`, }, "") return s @@ -1432,36 +1421,6 @@ func (m *Initializer) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field FailurePolicy", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - s := FailurePolicyType(dAtA[iNdEx:postIndex]) - m.FailurePolicy = &s - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -2169,60 +2128,60 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 875 bytes of a gzipped FileDescriptorProto + // 871 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0xcd, 0x8b, 0x23, 0x45, 0x14, 0x4f, 0x65, 0x32, 0x6c, 0x52, 0x49, 0xd8, 0xdd, 0x42, 0x97, 0x38, 0x48, 0x77, 0xe8, 0xc3, 0x12, 0x11, 0xbb, 0x9d, 0x51, 0x16, 0x41, 0x44, 0xa7, 0xc7, 0xaf, 0x81, 0xfd, 0x18, 0xcb, 0x45, - 0x41, 0x3c, 0x58, 0xe9, 0xbc, 0x24, 0x65, 0xfa, 0x8b, 0xaa, 0xea, 0xe0, 0x78, 0x10, 0xff, 0x04, - 0xc1, 0x8b, 0x57, 0x6f, 0x5e, 0xfc, 0x3f, 0xe6, 0xb8, 0xe0, 0x65, 0x4f, 0xc1, 0x69, 0xc1, 0x8b, - 0xe0, 0x1f, 0x30, 0x27, 0xe9, 0xaf, 0xa4, 0xb3, 0x49, 0xd8, 0xc4, 0x85, 0xb9, 0x75, 0xbd, 0x57, - 0xbf, 0xf7, 0x7e, 0xef, 0xc7, 0xef, 0x55, 0x63, 0x3a, 0x79, 0x47, 0x9a, 0x3c, 0xb0, 0x26, 0x51, - 0x1f, 0x84, 0x0f, 0x0a, 0xa4, 0x35, 0x05, 0x7f, 0x10, 0x08, 0x2b, 0x4f, 0xb0, 0x90, 0x5b, 0x6c, - 0xe0, 0x71, 0x29, 0x79, 0xe0, 0x0b, 0x18, 0x71, 0xa9, 0x04, 0x53, 0x3c, 0xf0, 0xad, 0xe9, 0x21, - 0x73, 0xc3, 0x31, 0x3b, 0xb4, 0x46, 0xe0, 0x83, 0x60, 0x0a, 0x06, 0x66, 0x28, 0x02, 0x15, 0x90, - 0xd7, 0x32, 0xa8, 0xc9, 0x42, 0x6e, 0xae, 0x85, 0x9a, 0x05, 0xf4, 0xe0, 0x8d, 0x11, 0x57, 0xe3, - 0xa8, 0x6f, 0x3a, 0x81, 0x67, 0x8d, 0x82, 0x51, 0x60, 0xa5, 0x15, 0xfa, 0xd1, 0x30, 0x3d, 0xa5, - 0x87, 0xf4, 0x2b, 0xab, 0x7c, 0xf0, 0xf6, 0x82, 0x94, 0xc7, 0x9c, 0x31, 0xf7, 0x41, 0x9c, 0x5b, - 0xe1, 0x64, 0x94, 0x04, 0xa4, 0xe5, 0x81, 0x62, 0xd6, 0x74, 0x85, 0xcf, 0x81, 0xb5, 0x09, 0x25, - 0x22, 0x5f, 0x71, 0x0f, 0x56, 0x00, 0xf7, 0x9e, 0x07, 0x90, 0xce, 0x18, 0x3c, 0xb6, 0x82, 0x7b, - 0x6b, 0x13, 0x2e, 0x52, 0xdc, 0xb5, 0xb8, 0xaf, 0xa4, 0x12, 0xcf, 0x82, 0x8c, 0xdf, 0x11, 0x7e, - 0xe5, 0xb8, 0x50, 0xe9, 0xd3, 0x20, 0x98, 0x9c, 0xb8, 0x1c, 0x7c, 0x75, 0x12, 0xf8, 0x43, 0x3e, - 0x22, 0x43, 0x7c, 0x43, 0x82, 0x98, 0x72, 0x07, 0x3a, 0xa8, 0x8b, 0x7a, 0xcd, 0xa3, 0x77, 0xcd, - 0xad, 0xd5, 0x35, 0x3f, 0xcf, 0x90, 0x14, 0x86, 0x20, 0xc0, 0x77, 0xc0, 0xbe, 0x79, 0x31, 0xd3, - 0x2b, 0xf1, 0x4c, 0xbf, 0x51, 0x64, 0x8a, 0xe2, 0xa4, 0x87, 0xeb, 0x0e, 0xb3, 0x23, 0x7f, 0xe0, - 0x42, 0xa7, 0xda, 0x45, 0xbd, 0x96, 0xdd, 0x8a, 0x67, 0x7a, 0xfd, 0xe4, 0x38, 0x8b, 0xd1, 0x79, - 0xd6, 0xf8, 0xa7, 0x8a, 0x5f, 0xfe, 0xe8, 0x3b, 0x05, 0xc2, 0x67, 0xee, 0x12, 0x6f, 0xd2, 0xc5, - 0x35, 0x9f, 0x79, 0x19, 0xd1, 0x86, 0xdd, 0xca, 0x7b, 0xd5, 0x1e, 0x32, 0x0f, 0x68, 0x9a, 0x21, - 0x3f, 0xe0, 0x96, 0x53, 0x9a, 0x2e, 0xed, 0xd4, 0x3c, 0xfa, 0x70, 0x87, 0x91, 0x36, 0x2a, 0x65, - 0xbf, 0x94, 0xf7, 0x6b, 0x95, 0xa3, 0x74, 0xa9, 0x1f, 0xe9, 0xe3, 0x7d, 0x11, 0xb9, 0x20, 0x3b, - 0x7b, 0xdd, 0xbd, 0x5e, 0xf3, 0xe8, 0xbd, 0x1d, 0x1a, 0xd3, 0xc8, 0x85, 0x2f, 0xb9, 0x1a, 0x3f, - 0x0a, 0x21, 0x4b, 0x49, 0xbb, 0x9d, 0x77, 0xdc, 0x4f, 0x72, 0x92, 0x66, 0xa5, 0xc9, 0x7d, 0xdc, - 0x1e, 0x32, 0xee, 0x46, 0x02, 0xce, 0x02, 0x97, 0x3b, 0xe7, 0x9d, 0x5a, 0x2a, 0xc7, 0xdd, 0x78, - 0xa6, 0xb7, 0x3f, 0x2e, 0x27, 0xae, 0x66, 0xfa, 0xed, 0xa5, 0xc0, 0xe3, 0xf3, 0x10, 0xe8, 0x32, - 0xd8, 0xf8, 0xb5, 0x8a, 0x8d, 0xb5, 0x6a, 0x67, 0x13, 0x45, 0x19, 0x17, 0xf2, 0x0d, 0xae, 0x27, - 0xee, 0x1f, 0x30, 0xc5, 0x72, 0x9f, 0xbc, 0x59, 0x9a, 0x6d, 0x6e, 0x46, 0x33, 0x9c, 0x8c, 0x92, - 0x80, 0x34, 0x93, 0xdb, 0xe6, 0xf4, 0xd0, 0x7c, 0xd4, 0xff, 0x16, 0x1c, 0xf5, 0x00, 0x14, 0xb3, - 0x49, 0x3e, 0x0e, 0x5e, 0xc4, 0xe8, 0xbc, 0x2a, 0xf9, 0x05, 0xe1, 0x3b, 0xb0, 0x8e, 0x88, 0xec, - 0x54, 0x53, 0x31, 0x3f, 0xd8, 0x41, 0xcc, 0xb5, 0x13, 0xd9, 0x5a, 0x4e, 0xe0, 0xce, 0xda, 0xb4, - 0xa4, 0x1b, 0xfa, 0x1b, 0x57, 0x08, 0xdf, 0x7d, 0xbe, 0x46, 0xf7, 0xb9, 0x54, 0xe4, 0xeb, 0x15, - 0x9d, 0xcc, 0xed, 0x74, 0x4a, 0xd0, 0xa9, 0x4a, 0xb7, 0x72, 0x92, 0xf5, 0x22, 0x52, 0xd2, 0x48, - 0xe0, 0x7d, 0xae, 0xc0, 0x2b, 0x14, 0x79, 0xf0, 0xa2, 0x8a, 0x2c, 0xf1, 0x5f, 0xd8, 0xed, 0x34, - 0xe9, 0x41, 0xb3, 0x56, 0xc6, 0x1f, 0x08, 0x37, 0x4f, 0x7d, 0xae, 0x38, 0x73, 0xf9, 0xf7, 0x20, - 0xb6, 0x58, 0xc2, 0xc7, 0xc5, 0x12, 0x64, 0x2c, 0xad, 0x1d, 0x97, 0x60, 0x5b, 0xdb, 0xef, 0xbd, - 0x88, 0xed, 0xff, 0x45, 0xb8, 0x53, 0x9a, 0xea, 0xba, 0xcd, 0x1e, 0xe2, 0x16, 0x5f, 0x74, 0x2f, - 0x94, 0xba, 0xb7, 0x83, 0x52, 0x25, 0xf2, 0x8b, 0x97, 0xa9, 0x14, 0x94, 0x74, 0xa9, 0x83, 0xf1, - 0x37, 0xc2, 0xaf, 0x6e, 0x1a, 0xf8, 0x1a, 0x9c, 0x3b, 0x5e, 0x76, 0xee, 0xc9, 0xff, 0x9b, 0x74, - 0x1b, 0xbf, 0xfe, 0x8c, 0x70, 0x2d, 0x31, 0x0e, 0x79, 0x1d, 0x37, 0x58, 0xc8, 0x3f, 0x11, 0x41, - 0x14, 0xca, 0x0e, 0xea, 0xee, 0xf5, 0x1a, 0x76, 0x3b, 0x9e, 0xe9, 0x8d, 0xe3, 0xb3, 0xd3, 0x2c, - 0x48, 0x17, 0x79, 0x72, 0x88, 0x9b, 0x2c, 0xe4, 0x5f, 0x80, 0x48, 0x78, 0x64, 0x2c, 0x1b, 0xf6, - 0xcd, 0x78, 0xa6, 0x37, 0x8f, 0xcf, 0x4e, 0x8b, 0x30, 0x2d, 0xdf, 0x49, 0xea, 0x0b, 0x90, 0x41, - 0x24, 0x9c, 0xfc, 0xbd, 0xcf, 0xeb, 0xd3, 0x22, 0x48, 0x17, 0x79, 0xe3, 0x37, 0x84, 0xc9, 0xea, - 0x0b, 0x4f, 0xde, 0xc7, 0x38, 0x98, 0x9f, 0x72, 0x92, 0x7a, 0xea, 0x9a, 0x79, 0xf4, 0x6a, 0xa6, - 0xb7, 0xe7, 0xa7, 0xd4, 0xca, 0x25, 0x08, 0xf9, 0x0c, 0xd7, 0x92, 0xf5, 0xc8, 0x7f, 0x74, 0x3b, - 0xaf, 0xda, 0x7c, 0x7d, 0x93, 0x13, 0x4d, 0x4b, 0x19, 0x80, 0x6f, 0x3d, 0xfb, 0x5f, 0x27, 0x16, - 0x6e, 0x24, 0xab, 0x2d, 0x43, 0xe6, 0x14, 0x9b, 0x7f, 0x3b, 0x87, 0x36, 0x1e, 0x16, 0x09, 0xba, - 0xb8, 0x33, 0x7f, 0x25, 0xaa, 0x9b, 0x5e, 0x09, 0xdb, 0xbc, 0xb8, 0xd4, 0x2a, 0x4f, 0x2e, 0xb5, - 0xca, 0xd3, 0x4b, 0xad, 0xf2, 0x63, 0xac, 0xa1, 0x8b, 0x58, 0x43, 0x4f, 0x62, 0x0d, 0x3d, 0x8d, - 0x35, 0xf4, 0x67, 0xac, 0xa1, 0x9f, 0xfe, 0xd2, 0x2a, 0x5f, 0xd5, 0x0b, 0xbe, 0xff, 0x05, 0x00, - 0x00, 0xff, 0xff, 0xd2, 0x64, 0x2c, 0xcb, 0x49, 0x0a, 0x00, 0x00, + 0x41, 0x3c, 0x58, 0xe9, 0xbc, 0x24, 0x65, 0xfa, 0x8b, 0xaa, 0xea, 0xe0, 0x78, 0x10, 0x2f, 0xde, + 0x05, 0x2f, 0x5e, 0xbd, 0x79, 0xf1, 0xff, 0x98, 0xe3, 0x1e, 0xf7, 0x14, 0x9c, 0x16, 0xbc, 0x08, + 0xfe, 0x01, 0x73, 0x92, 0xfe, 0x4a, 0x3a, 0x9b, 0x84, 0x4d, 0x5c, 0x98, 0x5b, 0xea, 0xf7, 0xea, + 0xf7, 0xde, 0xef, 0xfd, 0xf2, 0x5e, 0x35, 0xa6, 0x93, 0x77, 0xa4, 0xc9, 0x03, 0x6b, 0x12, 0xf5, + 0x41, 0xf8, 0xa0, 0x40, 0x5a, 0x53, 0xf0, 0x07, 0x81, 0xb0, 0xf2, 0x00, 0x0b, 0xb9, 0xc5, 0x06, + 0x1e, 0x97, 0x92, 0x07, 0xbe, 0x80, 0x11, 0x97, 0x4a, 0x30, 0xc5, 0x03, 0xdf, 0x9a, 0x1e, 0x32, + 0x37, 0x1c, 0xb3, 0x43, 0x6b, 0x04, 0x3e, 0x08, 0xa6, 0x60, 0x60, 0x86, 0x22, 0x50, 0x01, 0x79, + 0x2d, 0xa3, 0x9a, 0x2c, 0xe4, 0xe6, 0x5a, 0xaa, 0x59, 0x50, 0x0f, 0xde, 0x18, 0x71, 0x35, 0x8e, + 0xfa, 0xa6, 0x13, 0x78, 0xd6, 0x28, 0x18, 0x05, 0x56, 0x9a, 0xa1, 0x1f, 0x0d, 0xd3, 0x53, 0x7a, + 0x48, 0x7f, 0x65, 0x99, 0x0f, 0xde, 0x5e, 0x88, 0xf2, 0x98, 0x33, 0xe6, 0x3e, 0x88, 0x73, 0x2b, + 0x9c, 0x8c, 0x12, 0x40, 0x5a, 0x1e, 0x28, 0x66, 0x4d, 0x57, 0xf4, 0x1c, 0x58, 0x9b, 0x58, 0x22, + 0xf2, 0x15, 0xf7, 0x60, 0x85, 0x70, 0xef, 0x79, 0x04, 0xe9, 0x8c, 0xc1, 0x63, 0x2b, 0xbc, 0xb7, + 0x36, 0xf1, 0x22, 0xc5, 0x5d, 0x8b, 0xfb, 0x4a, 0x2a, 0xf1, 0x2c, 0xc9, 0xf8, 0x03, 0xe1, 0x57, + 0x8e, 0x0b, 0x97, 0x3e, 0x0d, 0x82, 0xc9, 0x89, 0xcb, 0xc1, 0x57, 0x27, 0x81, 0x3f, 0xe4, 0x23, + 0x32, 0xc4, 0x37, 0x24, 0x88, 0x29, 0x77, 0xa0, 0x83, 0xba, 0xa8, 0xd7, 0x3c, 0x7a, 0xd7, 0xdc, + 0xda, 0x5d, 0xf3, 0xf3, 0x8c, 0x49, 0x61, 0x08, 0x02, 0x7c, 0x07, 0xec, 0x9b, 0x17, 0x33, 0xbd, + 0x12, 0xcf, 0xf4, 0x1b, 0x45, 0xa4, 0x48, 0x4e, 0x7a, 0xb8, 0xee, 0x30, 0x3b, 0xf2, 0x07, 0x2e, + 0x74, 0xaa, 0x5d, 0xd4, 0x6b, 0xd9, 0xad, 0x78, 0xa6, 0xd7, 0x4f, 0x8e, 0x33, 0x8c, 0xce, 0xa3, + 0xc6, 0x3f, 0x55, 0xfc, 0xf2, 0x47, 0xdf, 0x29, 0x10, 0x3e, 0x73, 0x97, 0x74, 0x93, 0x2e, 0xae, + 0xf9, 0xcc, 0xcb, 0x84, 0x36, 0xec, 0x56, 0x5e, 0xab, 0xf6, 0x90, 0x79, 0x40, 0xd3, 0x08, 0xf9, + 0x01, 0xb7, 0x9c, 0x52, 0x77, 0x69, 0xa5, 0xe6, 0xd1, 0x87, 0x3b, 0xb4, 0xb4, 0xd1, 0x29, 0xfb, + 0xa5, 0xbc, 0x5e, 0xab, 0x8c, 0xd2, 0xa5, 0x7a, 0xa4, 0x8f, 0xf7, 0x45, 0xe4, 0x82, 0xec, 0xec, + 0x75, 0xf7, 0x7a, 0xcd, 0xa3, 0xf7, 0x76, 0x28, 0x4c, 0x23, 0x17, 0xbe, 0xe4, 0x6a, 0xfc, 0x28, + 0x84, 0x2c, 0x24, 0xed, 0x76, 0x5e, 0x71, 0x3f, 0x89, 0x49, 0x9a, 0xa5, 0x26, 0xf7, 0x71, 0x7b, + 0xc8, 0xb8, 0x1b, 0x09, 0x38, 0x0b, 0x5c, 0xee, 0x9c, 0x77, 0x6a, 0xa9, 0x1d, 0x77, 0xe3, 0x99, + 0xde, 0xfe, 0xb8, 0x1c, 0xb8, 0x9a, 0xe9, 0xb7, 0x97, 0x80, 0xc7, 0xe7, 0x21, 0xd0, 0x65, 0xb2, + 0xf1, 0x5b, 0x15, 0x1b, 0x6b, 0xdd, 0xce, 0x3a, 0x8a, 0x32, 0x2d, 0xe4, 0x1b, 0x5c, 0x4f, 0xa6, + 0x7f, 0xc0, 0x14, 0xcb, 0xe7, 0xe4, 0xcd, 0x52, 0x6f, 0xf3, 0x61, 0x34, 0xc3, 0xc9, 0x28, 0x01, + 0xa4, 0x99, 0xdc, 0x36, 0xa7, 0x87, 0xe6, 0xa3, 0xfe, 0xb7, 0xe0, 0xa8, 0x07, 0xa0, 0x98, 0x4d, + 0xf2, 0x76, 0xf0, 0x02, 0xa3, 0xf3, 0xac, 0xe4, 0x57, 0x84, 0xef, 0xc0, 0x3a, 0x21, 0xb2, 0x53, + 0x4d, 0xcd, 0xfc, 0x60, 0x07, 0x33, 0xd7, 0x76, 0x64, 0x6b, 0xb9, 0x80, 0x3b, 0x6b, 0xc3, 0x92, + 0x6e, 0xa8, 0x6f, 0x5c, 0x21, 0x7c, 0xf7, 0xf9, 0x1e, 0xdd, 0xe7, 0x52, 0x91, 0xaf, 0x57, 0x7c, + 0x32, 0xb7, 0xf3, 0x29, 0x61, 0xa7, 0x2e, 0xdd, 0xca, 0x45, 0xd6, 0x0b, 0xa4, 0xe4, 0x91, 0xc0, + 0xfb, 0x5c, 0x81, 0x57, 0x38, 0xf2, 0xe0, 0x45, 0x1d, 0x59, 0xd2, 0xbf, 0x18, 0xb7, 0xd3, 0xa4, + 0x06, 0xcd, 0x4a, 0x19, 0x3f, 0x21, 0xdc, 0x3c, 0xf5, 0xb9, 0xe2, 0xcc, 0xe5, 0xdf, 0x83, 0xd8, + 0x62, 0x09, 0x1f, 0x17, 0x4b, 0x90, 0xa9, 0xb4, 0x76, 0x5c, 0x82, 0xf5, 0x63, 0x6f, 0xfc, 0x8b, + 0x70, 0xa7, 0xa4, 0xe3, 0xba, 0xc7, 0x33, 0xc4, 0x2d, 0xbe, 0xa8, 0x5e, 0xf4, 0x76, 0x6f, 0x87, + 0xde, 0x4a, 0xe2, 0x17, 0x6f, 0x49, 0x09, 0x94, 0x74, 0xa9, 0x82, 0xf1, 0x37, 0xc2, 0xaf, 0x6e, + 0x6a, 0xf8, 0x1a, 0x66, 0x6d, 0xbc, 0x3c, 0x6b, 0x27, 0xff, 0xaf, 0xd3, 0x6d, 0x26, 0xec, 0x17, + 0x84, 0x6b, 0xc9, 0x5f, 0x4d, 0x5e, 0xc7, 0x0d, 0x16, 0xf2, 0x4f, 0x44, 0x10, 0x85, 0xb2, 0x83, + 0xba, 0x7b, 0xbd, 0x86, 0xdd, 0x8e, 0x67, 0x7a, 0xe3, 0xf8, 0xec, 0x34, 0x03, 0xe9, 0x22, 0x4e, + 0x0e, 0x71, 0x93, 0x85, 0xfc, 0x0b, 0x10, 0x89, 0x8e, 0x4c, 0x65, 0xc3, 0xbe, 0x19, 0xcf, 0xf4, + 0xe6, 0xf1, 0xd9, 0x69, 0x01, 0xd3, 0xf2, 0x9d, 0x24, 0xbf, 0x00, 0x19, 0x44, 0xc2, 0xc9, 0x5f, + 0xe8, 0x3c, 0x3f, 0x2d, 0x40, 0xba, 0x88, 0x1b, 0xbf, 0x23, 0x4c, 0x56, 0xdf, 0x64, 0xf2, 0x3e, + 0xc6, 0xc1, 0xfc, 0x94, 0x8b, 0xd4, 0xd3, 0xa9, 0x99, 0xa3, 0x57, 0x33, 0xbd, 0x3d, 0x3f, 0xa5, + 0x6f, 0x6e, 0x89, 0x42, 0x3e, 0xc3, 0xb5, 0x64, 0xa0, 0xf3, 0x4f, 0xd3, 0xce, 0xcb, 0x31, 0x5f, + 0xb8, 0xe4, 0x44, 0xd3, 0x54, 0x06, 0xe0, 0x5b, 0xcf, 0x7e, 0x89, 0x89, 0x85, 0x1b, 0xc9, 0x32, + 0xca, 0x90, 0x39, 0xc5, 0xae, 0xde, 0xce, 0xa9, 0x8d, 0x87, 0x45, 0x80, 0x2e, 0xee, 0xcc, 0xf7, + 0xba, 0xba, 0x69, 0xaf, 0x6d, 0xf3, 0xe2, 0x52, 0xab, 0x3c, 0xb9, 0xd4, 0x2a, 0x4f, 0x2f, 0xb5, + 0xca, 0x8f, 0xb1, 0x86, 0x2e, 0x62, 0x0d, 0x3d, 0x89, 0x35, 0xf4, 0x34, 0xd6, 0xd0, 0x9f, 0xb1, + 0x86, 0x7e, 0xfe, 0x4b, 0xab, 0x7c, 0x55, 0x2f, 0xf4, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0xe7, + 0xb5, 0x5f, 0xd5, 0xfb, 0x09, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/admissionregistration/v1alpha1/generated.proto b/staging/src/k8s.io/api/admissionregistration/v1alpha1/generated.proto index 68e162d7f61ea..5b0d4f0065b8b 100644 --- a/staging/src/k8s.io/api/admissionregistration/v1alpha1/generated.proto +++ b/staging/src/k8s.io/api/admissionregistration/v1alpha1/generated.proto @@ -107,13 +107,6 @@ message Initializer { // The initializer cares about an operation if it matches _any_ Rule. // Rule.Resources must not include subresources. repeated Rule rules = 2; - - // FailurePolicy defines what happens if the responsible initializer controller - // fails to takes action. Allowed values are Ignore, or Fail. If "Ignore" is - // set, initializer is removed from the initializers list of an object if - // the timeout is reached; If "Fail" is set, admissionregistration returns timeout error - // if the timeout is reached. - optional string failurePolicy = 3; } // InitializerConfiguration describes the configuration of initializers. diff --git a/staging/src/k8s.io/api/admissionregistration/v1alpha1/types.generated.go b/staging/src/k8s.io/api/admissionregistration/v1alpha1/types.generated.go index 8a6563730bc40..466b6a9f9de19 100644 --- a/staging/src/k8s.io/api/admissionregistration/v1alpha1/types.generated.go +++ b/staging/src/k8s.io/api/admissionregistration/v1alpha1/types.generated.go @@ -826,14 +826,13 @@ func (x *Initializer) CodecEncodeSelf(e *codec1978.Encoder) { } else { yysep2 := !z.EncBinary() yy2arr2 := z.EncBasicHandle().StructToArray - var yyq2 [3]bool + var yyq2 [2]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false yyq2[1] = len(x.Rules) != 0 - yyq2[2] = x.FailurePolicy != nil var yynn2 int if yyr2 || yy2arr2 { - r.EncodeArrayStart(3) + r.EncodeArrayStart(2) } else { yynn2 = 1 for _, b := range yyq2 { @@ -896,31 +895,6 @@ func (x *Initializer) CodecEncodeSelf(e *codec1978.Encoder) { } } } - if yyr2 || yy2arr2 { - z.EncSendContainerState(codecSelfer_containerArrayElem1234) - if yyq2[2] { - if x.FailurePolicy == nil { - r.EncodeNil() - } else { - yy10 := *x.FailurePolicy - yy10.CodecEncodeSelf(e) - } - } else { - r.EncodeNil() - } - } else { - if yyq2[2] { - z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("failurePolicy")) - z.EncSendContainerState(codecSelfer_containerMapValue1234) - if x.FailurePolicy == nil { - r.EncodeNil() - } else { - yy12 := *x.FailurePolicy - yy12.CodecEncodeSelf(e) - } - } - } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayEnd1234) } else { @@ -1006,17 +980,6 @@ func (x *Initializer) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { h.decSliceRule((*[]Rule)(yyv6), d) } } - case "failurePolicy": - if r.TryDecodeAsNil() { - if x.FailurePolicy != nil { - x.FailurePolicy = nil - } - } else { - if x.FailurePolicy == nil { - x.FailurePolicy = new(FailurePolicyType) - } - x.FailurePolicy.CodecDecodeSelf(d) - } default: z.DecStructFieldNotFound(-1, yys3) } // end switch yys3 @@ -1028,16 +991,16 @@ func (x *Initializer) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj9 int - var yyb9 bool - var yyhl9 bool = l >= 0 - yyj9++ - if yyhl9 { - yyb9 = yyj9 > l + var yyj8 int + var yyb8 bool + var yyhl8 bool = l >= 0 + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l } else { - yyb9 = r.CheckBreak() + yyb8 = r.CheckBreak() } - if yyb9 { + if yyb8 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1045,21 +1008,21 @@ func (x *Initializer) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.Name = "" } else { - yyv10 := &x.Name - yym11 := z.DecBinary() - _ = yym11 + yyv9 := &x.Name + yym10 := z.DecBinary() + _ = yym10 if false { } else { - *((*string)(yyv10)) = r.DecodeString() + *((*string)(yyv9)) = r.DecodeString() } } - yyj9++ - if yyhl9 { - yyb9 = yyj9 > l + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l } else { - yyb9 = r.CheckBreak() + yyb8 = r.CheckBreak() } - if yyb9 { + if yyb8 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1067,47 +1030,26 @@ func (x *Initializer) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.Rules = nil } else { - yyv12 := &x.Rules - yym13 := z.DecBinary() - _ = yym13 + yyv11 := &x.Rules + yym12 := z.DecBinary() + _ = yym12 if false { } else { - h.decSliceRule((*[]Rule)(yyv12), d) - } - } - yyj9++ - if yyhl9 { - yyb9 = yyj9 > l - } else { - yyb9 = r.CheckBreak() - } - if yyb9 { - z.DecSendContainerState(codecSelfer_containerArrayEnd1234) - return - } - z.DecSendContainerState(codecSelfer_containerArrayElem1234) - if r.TryDecodeAsNil() { - if x.FailurePolicy != nil { - x.FailurePolicy = nil + h.decSliceRule((*[]Rule)(yyv11), d) } - } else { - if x.FailurePolicy == nil { - x.FailurePolicy = new(FailurePolicyType) - } - x.FailurePolicy.CodecDecodeSelf(d) } for { - yyj9++ - if yyhl9 { - yyb9 = yyj9 > l + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l } else { - yyb9 = r.CheckBreak() + yyb8 = r.CheckBreak() } - if yyb9 { + if yyb8 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj9-1, "") + z.DecStructFieldNotFound(yyj8-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } @@ -3443,7 +3385,7 @@ func (x codecSelfer1234) decSliceInitializer(v *[]Initializer, d *codec1978.Deco yyrg1 := len(yyv1) > 0 yyv21 := yyv1 - yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 48) + yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 40) if yyrt1 { if yyrl1 <= cap(yyv1) { yyv1 = yyv1[:yyrl1] diff --git a/staging/src/k8s.io/api/admissionregistration/v1alpha1/types.go b/staging/src/k8s.io/api/admissionregistration/v1alpha1/types.go index 1b5f5be80b3fc..d4827e59d33f9 100644 --- a/staging/src/k8s.io/api/admissionregistration/v1alpha1/types.go +++ b/staging/src/k8s.io/api/admissionregistration/v1alpha1/types.go @@ -72,13 +72,6 @@ type Initializer struct { // The initializer cares about an operation if it matches _any_ Rule. // Rule.Resources must not include subresources. Rules []Rule `json:"rules,omitempty" protobuf:"bytes,2,rep,name=rules"` - - // FailurePolicy defines what happens if the responsible initializer controller - // fails to takes action. Allowed values are Ignore, or Fail. If "Ignore" is - // set, initializer is removed from the initializers list of an object if - // the timeout is reached; If "Fail" is set, admissionregistration returns timeout error - // if the timeout is reached. - FailurePolicy *FailurePolicyType `json:"failurePolicy,omitempty" protobuf:"bytes,3,opt,name=failurePolicy,casttype=FailurePolicyType"` } // Rule is a tuple of APIGroups, APIVersion, and Resources.It is recommended diff --git a/staging/src/k8s.io/api/admissionregistration/v1alpha1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/admissionregistration/v1alpha1/types_swagger_doc_generated.go index 9597deb59df5a..0b30ecc802ed1 100644 --- a/staging/src/k8s.io/api/admissionregistration/v1alpha1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/admissionregistration/v1alpha1/types_swagger_doc_generated.go @@ -70,10 +70,9 @@ func (ExternalAdmissionHookConfigurationList) SwaggerDoc() map[string]string { } var map_Initializer = map[string]string{ - "": "Initializer describes the name and the failure policy of an initializer, and what resources it applies to.", - "name": "Name is the identifier of the initializer. It will be added to the object that needs to be initialized. Name should be fully qualified, e.g., alwayspullimages.kubernetes.io, where \"alwayspullimages\" is the name of the webhook, and kubernetes.io is the name of the organization. Required", - "rules": "Rules describes what resources/subresources the initializer cares about. The initializer cares about an operation if it matches _any_ Rule. Rule.Resources must not include subresources.", - "failurePolicy": "FailurePolicy defines what happens if the responsible initializer controller fails to takes action. Allowed values are Ignore, or Fail. If \"Ignore\" is set, initializer is removed from the initializers list of an object if the timeout is reached; If \"Fail\" is set, admissionregistration returns timeout error if the timeout is reached.", + "": "Initializer describes the name and the failure policy of an initializer, and what resources it applies to.", + "name": "Name is the identifier of the initializer. It will be added to the object that needs to be initialized. Name should be fully qualified, e.g., alwayspullimages.kubernetes.io, where \"alwayspullimages\" is the name of the webhook, and kubernetes.io is the name of the organization. Required", + "rules": "Rules describes what resources/subresources the initializer cares about. The initializer cares about an operation if it matches _any_ Rule. Rule.Resources must not include subresources.", } func (Initializer) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/admissionregistration/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/admissionregistration/v1alpha1/zz_generated.deepcopy.go index 05a3a797f97b8..118fed750312c 100644 --- a/staging/src/k8s.io/api/admissionregistration/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/admissionregistration/v1alpha1/zz_generated.deepcopy.go @@ -212,15 +212,6 @@ func (in *Initializer) DeepCopyInto(out *Initializer) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.FailurePolicy != nil { - in, out := &in.FailurePolicy, &out.FailurePolicy - if *in == nil { - *out = nil - } else { - *out = new(FailurePolicyType) - **out = **in - } - } return } diff --git a/staging/src/k8s.io/api/apps/v1beta1/generated.pb.go b/staging/src/k8s.io/api/apps/v1beta1/generated.pb.go index bc0a74ab76a6f..6d5e1f7a1ebeb 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/generated.pb.go +++ b/staging/src/k8s.io/api/apps/v1beta1/generated.pb.go @@ -3050,7 +3050,7 @@ func (m *DeploymentStatus) Unmarshal(dAtA []byte) error { if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CollisionCount", wireType) } - var v int64 + var v int32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -3060,7 +3060,7 @@ func (m *DeploymentStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= (int32(b) & 0x7F) << shift if b < 0x80 { break } @@ -4587,7 +4587,7 @@ func (m *StatefulSetStatus) Unmarshal(dAtA []byte) error { if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CollisionCount", wireType) } - var v int64 + var v int32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -4597,7 +4597,7 @@ func (m *StatefulSetStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= (int32(b) & 0x7F) << shift if b < 0x80 { break } @@ -4847,118 +4847,118 @@ func init() { var fileDescriptorGenerated = []byte{ // 1820 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcd, 0x6f, 0x1c, 0x49, - 0x15, 0x77, 0xcf, 0x87, 0x3d, 0x53, 0x5e, 0x8f, 0xe3, 0xb2, 0xb1, 0x67, 0xbd, 0x30, 0xb6, 0x86, - 0xd5, 0xae, 0xb3, 0xbb, 0xee, 0xd9, 0x78, 0x97, 0xd5, 0x26, 0x91, 0x22, 0x3c, 0xe3, 0x90, 0x38, - 0xb2, 0xb1, 0x53, 0x63, 0x07, 0x11, 0x40, 0x4a, 0x4d, 0x4f, 0x65, 0xdc, 0x71, 0x7f, 0xa9, 0xbb, - 0x7a, 0xc8, 0x88, 0x0b, 0x77, 0x90, 0xc2, 0x99, 0xbf, 0x82, 0x23, 0x82, 0x1b, 0x27, 0x5f, 0x90, - 0x22, 0x2e, 0xe4, 0x64, 0x91, 0xc9, 0x95, 0x2b, 0x97, 0x48, 0x48, 0xa8, 0xaa, 0xab, 0xbf, 0xbb, - 0xed, 0x31, 0x12, 0x3e, 0x70, 0x9b, 0xae, 0xf7, 0xde, 0xef, 0xbd, 0xaa, 0x7a, 0xf5, 0xde, 0xef, - 0x0d, 0xf8, 0xe1, 0xe9, 0xb7, 0x8e, 0xac, 0x9a, 0xad, 0x53, 0xb7, 0x47, 0x6c, 0x83, 0x50, 0xe2, - 0xb4, 0x86, 0xc4, 0xe8, 0x9b, 0x76, 0x4b, 0x08, 0xb0, 0xa5, 0xb6, 0xb0, 0x65, 0x39, 0xad, 0xe1, - 0xad, 0x1e, 0xa1, 0xf8, 0x56, 0x6b, 0x40, 0x0c, 0x62, 0x63, 0x4a, 0xfa, 0xb2, 0x65, 0x9b, 0xd4, - 0x84, 0x2b, 0x9e, 0xa2, 0x8c, 0x2d, 0x55, 0x66, 0x8a, 0xb2, 0x50, 0x5c, 0xdd, 0x1c, 0xa8, 0xf4, - 0xc4, 0xed, 0xc9, 0x8a, 0xa9, 0xb7, 0x06, 0xe6, 0xc0, 0x6c, 0x71, 0xfd, 0x9e, 0xfb, 0x9c, 0x7f, - 0xf1, 0x0f, 0xfe, 0xcb, 0xc3, 0x59, 0x6d, 0x46, 0x1c, 0x2a, 0xa6, 0x4d, 0x5a, 0xc3, 0x94, 0xaf, - 0xd5, 0x9b, 0x11, 0x1d, 0xcb, 0xd4, 0x54, 0x65, 0x94, 0x17, 0xd6, 0xea, 0xd7, 0xa1, 0xaa, 0x8e, - 0x95, 0x13, 0xd5, 0x20, 0xf6, 0xa8, 0x65, 0x9d, 0x0e, 0xd8, 0x82, 0xd3, 0xd2, 0x09, 0xc5, 0x59, - 0x0e, 0x5a, 0x79, 0x56, 0xb6, 0x6b, 0x50, 0x55, 0x27, 0x29, 0x83, 0x6f, 0x2e, 0x33, 0x70, 0x94, - 0x13, 0xa2, 0xe3, 0x94, 0xdd, 0x57, 0x79, 0x76, 0x2e, 0x55, 0xb5, 0x96, 0x6a, 0x50, 0x87, 0xda, - 0x49, 0xa3, 0xe6, 0xbf, 0x24, 0x00, 0x3b, 0xa6, 0x41, 0x6d, 0x53, 0xd3, 0x88, 0x8d, 0xc8, 0x50, - 0x75, 0x54, 0xd3, 0x80, 0xcf, 0x40, 0x85, 0xed, 0xa7, 0x8f, 0x29, 0xae, 0x4b, 0xeb, 0xd2, 0xc6, - 0xec, 0xd6, 0x97, 0x72, 0x78, 0x29, 0x01, 0xbc, 0x6c, 0x9d, 0x0e, 0xd8, 0x82, 0x23, 0x33, 0x6d, - 0x79, 0x78, 0x4b, 0x3e, 0xe8, 0xbd, 0x20, 0x0a, 0xdd, 0x27, 0x14, 0xb7, 0xe1, 0xd9, 0xf9, 0xda, - 0xd4, 0xf8, 0x7c, 0x0d, 0x84, 0x6b, 0x28, 0x40, 0x85, 0x07, 0xa0, 0xc4, 0xd1, 0x0b, 0x1c, 0x7d, - 0x33, 0x17, 0x5d, 0x6c, 0x5a, 0x46, 0xf8, 0x97, 0xf7, 0x5f, 0x52, 0x62, 0xb0, 0xf0, 0xda, 0x1f, - 0x08, 0xe8, 0xd2, 0x0e, 0xa6, 0x18, 0x71, 0x20, 0xf8, 0x05, 0xa8, 0xd8, 0x22, 0xfc, 0x7a, 0x71, - 0x5d, 0xda, 0x28, 0xb6, 0x6f, 0x08, 0xad, 0x8a, 0xbf, 0x2d, 0x14, 0x68, 0x34, 0xcf, 0x24, 0xb0, - 0x9c, 0xde, 0xf7, 0x9e, 0xea, 0x50, 0xf8, 0xf3, 0xd4, 0xde, 0xe5, 0xc9, 0xf6, 0xce, 0xac, 0xf9, - 0xce, 0x03, 0xc7, 0xfe, 0x4a, 0x64, 0xdf, 0x87, 0xa0, 0xac, 0x52, 0xa2, 0x3b, 0xf5, 0xc2, 0x7a, - 0x71, 0x63, 0x76, 0xeb, 0x73, 0x39, 0x27, 0xd7, 0xe5, 0x74, 0x74, 0xed, 0x39, 0x81, 0x5b, 0xde, - 0x65, 0x08, 0xc8, 0x03, 0x6a, 0xfe, 0xb6, 0x00, 0xc0, 0x0e, 0xb1, 0x34, 0x73, 0xa4, 0x13, 0x83, - 0x5e, 0xc3, 0xd5, 0xed, 0x82, 0x92, 0x63, 0x11, 0x45, 0x5c, 0xdd, 0xa7, 0xb9, 0x3b, 0x08, 0x83, - 0xea, 0x5a, 0x44, 0x09, 0x2f, 0x8d, 0x7d, 0x21, 0x0e, 0x01, 0x1f, 0x83, 0x69, 0x87, 0x62, 0xea, - 0x3a, 0xfc, 0xca, 0x66, 0xb7, 0x6e, 0x4e, 0x02, 0xc6, 0x0d, 0xda, 0x35, 0x01, 0x37, 0xed, 0x7d, - 0x23, 0x01, 0xd4, 0xfc, 0x7b, 0x11, 0x2c, 0x86, 0xca, 0x1d, 0xd3, 0xe8, 0xab, 0x94, 0xa5, 0xf4, - 0x5d, 0x50, 0xa2, 0x23, 0x8b, 0xf0, 0x33, 0xa9, 0xb6, 0x3f, 0xf5, 0x83, 0x39, 0x1a, 0x59, 0xe4, - 0xfd, 0xf9, 0xda, 0x4a, 0x86, 0x09, 0x13, 0x21, 0x6e, 0x04, 0xf7, 0x82, 0x38, 0x0b, 0xdc, 0xfc, - 0xeb, 0xb8, 0xf3, 0xf7, 0xe7, 0x6b, 0x19, 0xb5, 0x46, 0x0e, 0x90, 0xe2, 0x21, 0xc2, 0x4f, 0xc0, - 0xb4, 0x4d, 0xb0, 0x63, 0x1a, 0xf5, 0x12, 0x47, 0x0b, 0xb6, 0x82, 0xf8, 0x2a, 0x12, 0x52, 0x78, - 0x13, 0xcc, 0xe8, 0xc4, 0x71, 0xf0, 0x80, 0xd4, 0xcb, 0x5c, 0x71, 0x5e, 0x28, 0xce, 0xec, 0x7b, - 0xcb, 0xc8, 0x97, 0xc3, 0x17, 0xa0, 0xa6, 0x61, 0x87, 0x1e, 0x5b, 0x7d, 0x4c, 0xc9, 0x91, 0xaa, - 0x93, 0xfa, 0x34, 0x3f, 0xd0, 0xcf, 0x26, 0xbb, 0x7b, 0x66, 0xd1, 0x5e, 0x16, 0xe8, 0xb5, 0xbd, - 0x18, 0x12, 0x4a, 0x20, 0xc3, 0x21, 0x80, 0x6c, 0xe5, 0xc8, 0xc6, 0x86, 0xe3, 0x1d, 0x14, 0xf3, - 0x37, 0x73, 0x65, 0x7f, 0xab, 0xc2, 0x1f, 0xdc, 0x4b, 0xa1, 0xa1, 0x0c, 0x0f, 0xcd, 0x3f, 0x4a, - 0xa0, 0x16, 0x5e, 0xd3, 0x35, 0xbc, 0xd5, 0x87, 0xf1, 0xb7, 0xfa, 0xfd, 0x09, 0x92, 0x33, 0xe7, - 0x8d, 0xfe, 0xb3, 0x00, 0x60, 0xa8, 0x84, 0x4c, 0x4d, 0xeb, 0x61, 0xe5, 0x14, 0xae, 0x83, 0x92, - 0x81, 0x75, 0x3f, 0x27, 0x83, 0x07, 0xf2, 0x63, 0xac, 0x13, 0xc4, 0x25, 0xf0, 0x95, 0x04, 0xa0, - 0xcb, 0x8f, 0xbe, 0xbf, 0x6d, 0x18, 0x26, 0xc5, 0xec, 0x34, 0xfc, 0x80, 0x3a, 0x13, 0x04, 0xe4, - 0xfb, 0x92, 0x8f, 0x53, 0x28, 0xf7, 0x0d, 0x6a, 0x8f, 0xc2, 0x5b, 0x48, 0x2b, 0xa0, 0x0c, 0xd7, - 0xf0, 0x67, 0x00, 0xd8, 0x02, 0xf3, 0xc8, 0x14, 0xcf, 0x36, 0xbf, 0x06, 0xf8, 0xee, 0x3b, 0xa6, - 0xf1, 0x5c, 0x1d, 0x84, 0x85, 0x05, 0x05, 0x10, 0x28, 0x02, 0xb7, 0x7a, 0x1f, 0xac, 0xe4, 0xc4, - 0x09, 0x6f, 0x80, 0xe2, 0x29, 0x19, 0x79, 0x47, 0x85, 0xd8, 0x4f, 0xb8, 0x04, 0xca, 0x43, 0xac, - 0xb9, 0xc4, 0x7b, 0x93, 0xc8, 0xfb, 0xb8, 0x53, 0xf8, 0x56, 0x6a, 0xfe, 0xa1, 0x1c, 0xcd, 0x14, - 0x56, 0x6f, 0xe0, 0x06, 0x6b, 0x0f, 0x96, 0xa6, 0x2a, 0xd8, 0xe1, 0x18, 0xe5, 0xf6, 0x07, 0x5e, - 0x6b, 0xf0, 0xd6, 0x50, 0x20, 0x85, 0xbf, 0x00, 0x15, 0x87, 0x68, 0x44, 0xa1, 0xa6, 0x2d, 0x4a, - 0xdc, 0x57, 0x13, 0xe6, 0x14, 0xee, 0x11, 0xad, 0x2b, 0x4c, 0x3d, 0x78, 0xff, 0x0b, 0x05, 0x90, - 0xf0, 0x31, 0xa8, 0x50, 0xa2, 0x5b, 0x1a, 0xa6, 0x44, 0x9c, 0x5e, 0x2c, 0xaf, 0x58, 0xed, 0x60, - 0x60, 0x87, 0x66, 0xff, 0x48, 0xa8, 0xf1, 0xea, 0x19, 0xe4, 0xa9, 0xbf, 0x8a, 0x02, 0x18, 0xf8, - 0x53, 0x50, 0x71, 0x28, 0xeb, 0xea, 0x83, 0x11, 0xaf, 0x28, 0x17, 0xb5, 0x95, 0x68, 0x1d, 0xf5, - 0x4c, 0x42, 0x68, 0x7f, 0x05, 0x05, 0x70, 0x70, 0x1b, 0xcc, 0xeb, 0xaa, 0x81, 0x08, 0xee, 0x8f, - 0xba, 0x44, 0x31, 0x8d, 0xbe, 0xc3, 0x4b, 0x51, 0xb9, 0xbd, 0x22, 0x8c, 0xe6, 0xf7, 0xe3, 0x62, - 0x94, 0xd4, 0x87, 0x7b, 0x60, 0xc9, 0x6f, 0xbb, 0x0f, 0x55, 0x87, 0x9a, 0xf6, 0x68, 0x4f, 0xd5, - 0x55, 0xca, 0x0b, 0x54, 0xb9, 0x5d, 0x1f, 0x9f, 0xaf, 0x2d, 0xa1, 0x0c, 0x39, 0xca, 0xb4, 0x62, - 0xb5, 0xd3, 0xc2, 0xae, 0x43, 0xfa, 0xbc, 0xe0, 0x54, 0xc2, 0xda, 0x79, 0xc8, 0x57, 0x91, 0x90, - 0xc2, 0x9f, 0xc4, 0xd2, 0xb4, 0x72, 0xb5, 0x34, 0xad, 0xe5, 0xa7, 0x28, 0x3c, 0x06, 0x2b, 0x96, - 0x6d, 0x0e, 0x6c, 0xe2, 0x38, 0x3b, 0x04, 0xf7, 0x35, 0xd5, 0x20, 0xfe, 0xc9, 0x54, 0xf9, 0x8e, - 0x3e, 0x1a, 0x9f, 0xaf, 0xad, 0x1c, 0x66, 0xab, 0xa0, 0x3c, 0xdb, 0xe6, 0x5f, 0x4a, 0xe0, 0x46, - 0xb2, 0xc7, 0xc1, 0x47, 0x00, 0x9a, 0x3d, 0x87, 0xd8, 0x43, 0xd2, 0x7f, 0xe0, 0x11, 0x37, 0xc6, - 0x6e, 0x24, 0xce, 0x6e, 0x82, 0x77, 0x7b, 0x90, 0xd2, 0x40, 0x19, 0x56, 0x1e, 0x3f, 0x12, 0x0f, - 0xa0, 0xc0, 0x03, 0x8d, 0xf0, 0xa3, 0xd4, 0x23, 0xd8, 0x06, 0xf3, 0xe2, 0xed, 0xfb, 0x42, 0x9e, - 0xac, 0x91, 0x7b, 0x3f, 0x8e, 0x8b, 0x51, 0x52, 0x1f, 0x3e, 0x00, 0x0b, 0x78, 0x88, 0x55, 0x0d, - 0xf7, 0x34, 0x12, 0x80, 0x94, 0x38, 0xc8, 0x87, 0x02, 0x64, 0x61, 0x3b, 0xa9, 0x80, 0xd2, 0x36, - 0x70, 0x1f, 0x2c, 0xba, 0x46, 0x1a, 0xca, 0xcb, 0xc3, 0x8f, 0x04, 0xd4, 0xe2, 0x71, 0x5a, 0x05, - 0x65, 0xd9, 0xc1, 0x67, 0x00, 0x28, 0x7e, 0x63, 0x76, 0xea, 0xd3, 0xbc, 0x92, 0x7e, 0x31, 0xc1, - 0x7b, 0x09, 0xba, 0x79, 0x58, 0xc5, 0x82, 0x25, 0x07, 0x45, 0x30, 0xe1, 0x5d, 0x30, 0x67, 0xb3, - 0x17, 0x10, 0x84, 0x3a, 0xc3, 0x43, 0xfd, 0x8e, 0x30, 0x9b, 0x43, 0x51, 0x21, 0x8a, 0xeb, 0xc2, - 0x3b, 0xa0, 0xa6, 0x98, 0x9a, 0xc6, 0x33, 0xbf, 0x63, 0xba, 0x06, 0xe5, 0xc9, 0x5b, 0x6c, 0x43, - 0xd6, 0x99, 0x3b, 0x31, 0x09, 0x4a, 0x68, 0x36, 0xff, 0x2c, 0x45, 0xdb, 0x8c, 0xff, 0x9c, 0xe1, - 0x9d, 0x18, 0xf5, 0xf9, 0x24, 0x41, 0x7d, 0x96, 0xd3, 0x16, 0x11, 0xe6, 0xa3, 0x82, 0x39, 0x96, - 0xfc, 0xaa, 0x31, 0xf0, 0x2e, 0x5c, 0x94, 0xc4, 0x2f, 0x2f, 0x7c, 0x4a, 0x81, 0x76, 0xa4, 0x31, - 0x2e, 0xf0, 0x9d, 0x47, 0x85, 0x28, 0x8e, 0xdc, 0xbc, 0x07, 0x6a, 0xf1, 0x77, 0x18, 0xe3, 0xf4, - 0xd2, 0xa5, 0x9c, 0xfe, 0x9d, 0x04, 0x56, 0x72, 0xbc, 0x43, 0x0d, 0xd4, 0x74, 0xfc, 0x32, 0x92, - 0x23, 0x97, 0x72, 0x63, 0x36, 0x35, 0xc9, 0xde, 0xd4, 0x24, 0xef, 0x1a, 0xf4, 0xc0, 0xee, 0x52, - 0x5b, 0x35, 0x06, 0xde, 0x3d, 0xec, 0xc7, 0xb0, 0x50, 0x02, 0x1b, 0x3e, 0x05, 0x15, 0x1d, 0xbf, - 0xec, 0xba, 0xf6, 0x20, 0xeb, 0xbc, 0x26, 0xf3, 0xc3, 0xfb, 0xc7, 0xbe, 0x40, 0x41, 0x01, 0x5e, - 0xf3, 0x00, 0xac, 0xc7, 0x36, 0xc9, 0x4a, 0x05, 0x79, 0xee, 0x6a, 0x5d, 0x12, 0x5e, 0xf8, 0xe7, - 0xa0, 0x6a, 0x61, 0x9b, 0xaa, 0x41, 0xb9, 0x28, 0xb7, 0xe7, 0xc6, 0xe7, 0x6b, 0xd5, 0x43, 0x7f, - 0x11, 0x85, 0xf2, 0xe6, 0xbf, 0x25, 0x50, 0xee, 0x2a, 0x58, 0x23, 0xd7, 0x30, 0x3a, 0xec, 0xc4, - 0x46, 0x87, 0x66, 0x6e, 0x12, 0xf1, 0x78, 0x72, 0xa7, 0x86, 0xbd, 0xc4, 0xd4, 0xf0, 0xf1, 0x25, - 0x38, 0x17, 0x0f, 0x0c, 0xb7, 0x41, 0x35, 0x70, 0x17, 0xab, 0x92, 0xd2, 0x65, 0x55, 0xb2, 0xf9, - 0xfb, 0x02, 0x98, 0x8d, 0xb8, 0xb8, 0x9a, 0x35, 0x3b, 0xee, 0x08, 0xd1, 0x60, 0x65, 0x68, 0x6b, - 0x92, 0x8d, 0xc8, 0x3e, 0xa9, 0xf0, 0xf8, 0x5b, 0xd8, 0xbd, 0xd3, 0x5c, 0xe3, 0x1e, 0xa8, 0x51, - 0x6c, 0x0f, 0x08, 0xf5, 0x65, 0xfc, 0xc0, 0xaa, 0x21, 0xd3, 0x3f, 0x8a, 0x49, 0x51, 0x42, 0x7b, - 0xf5, 0x2e, 0x98, 0x8b, 0x39, 0xbb, 0x12, 0x09, 0x7b, 0xc5, 0x0e, 0x27, 0x4c, 0xce, 0x6b, 0xc8, - 0xae, 0x47, 0xb1, 0xec, 0xda, 0xc8, 0x3f, 0xcc, 0xc8, 0x93, 0xc9, 0xcb, 0x31, 0x94, 0xc8, 0xb1, - 0xcf, 0x26, 0x42, 0xbb, 0x38, 0xd3, 0xfe, 0x24, 0x81, 0xf9, 0x88, 0xf6, 0x35, 0x4c, 0x30, 0xbb, - 0xf1, 0x09, 0xe6, 0xe3, 0x49, 0x36, 0x91, 0x33, 0xc2, 0xfc, 0xb5, 0x1c, 0x0b, 0xfe, 0xff, 0x9e, - 0x54, 0xff, 0x0a, 0x2c, 0x0d, 0x4d, 0xcd, 0xd5, 0x49, 0x47, 0xc3, 0xaa, 0xee, 0x2b, 0x30, 0x06, - 0x53, 0x4c, 0xfe, 0x51, 0x11, 0xc0, 0x13, 0xdb, 0x51, 0x1d, 0x4a, 0x0c, 0xfa, 0x24, 0xb4, 0x6c, - 0x7f, 0x57, 0x38, 0x59, 0x7a, 0x92, 0x01, 0x87, 0x32, 0x9d, 0xc0, 0x1f, 0x80, 0x59, 0x46, 0xe0, - 0x54, 0x85, 0xb0, 0x59, 0x50, 0x4c, 0xff, 0x8b, 0x02, 0x68, 0xb6, 0x1b, 0x8a, 0x50, 0x54, 0x0f, - 0x9e, 0x80, 0x45, 0xcb, 0xec, 0xef, 0x63, 0x03, 0x0f, 0x08, 0x6b, 0x7b, 0x87, 0xfc, 0x0f, 0x4d, - 0xce, 0xb4, 0xab, 0xed, 0x6f, 0x7c, 0xa6, 0x74, 0x98, 0x56, 0x79, 0xcf, 0x28, 0x6b, 0x7a, 0x99, - 0xf3, 0x80, 0x2c, 0x48, 0x68, 0x83, 0x9a, 0x2b, 0xda, 0x8f, 0x18, 0x3c, 0xbc, 0xf9, 0x7f, 0x6b, - 0x92, 0x0c, 0x3b, 0x8e, 0x59, 0x86, 0xd5, 0x28, 0xbe, 0x8e, 0x12, 0x1e, 0x72, 0x07, 0x89, 0xca, - 0x7f, 0x33, 0x48, 0x34, 0x7f, 0x53, 0x02, 0x0b, 0xa9, 0xa7, 0x0b, 0x7f, 0x74, 0x01, 0xe3, 0x5e, - 0xfe, 0x9f, 0xb1, 0xed, 0x14, 0x61, 0x2c, 0x5e, 0x81, 0x30, 0x6e, 0x83, 0x79, 0xc5, 0xb5, 0x6d, - 0x36, 0xeb, 0xc7, 0x59, 0x76, 0x40, 0xd5, 0x3b, 0x71, 0x31, 0x4a, 0xea, 0x67, 0xb1, 0xfd, 0xf2, - 0x15, 0xd9, 0x7e, 0x34, 0x0a, 0xc1, 0xd8, 0xbc, 0xb4, 0x4b, 0x47, 0x21, 0x88, 0x5b, 0x52, 0x9f, - 0x75, 0x2b, 0x0f, 0x35, 0x40, 0x98, 0x89, 0x77, 0xab, 0xe3, 0x98, 0x14, 0x25, 0xb4, 0x33, 0x98, - 0x73, 0x75, 0x62, 0xe6, 0xfc, 0x37, 0x09, 0x7c, 0x98, 0x9b, 0xa1, 0x70, 0x3b, 0x46, 0xa0, 0x37, - 0x13, 0x04, 0xfa, 0x7b, 0xb9, 0x86, 0x11, 0x1e, 0x6d, 0x67, 0xf3, 0xe8, 0xdb, 0x93, 0xf1, 0xe8, - 0x0c, 0x92, 0x77, 0x39, 0xa1, 0x6e, 0x6f, 0x9e, 0xbd, 0x6d, 0x4c, 0xbd, 0x7e, 0xdb, 0x98, 0x7a, - 0xf3, 0xb6, 0x31, 0xf5, 0xeb, 0x71, 0x43, 0x3a, 0x1b, 0x37, 0xa4, 0xd7, 0xe3, 0x86, 0xf4, 0x66, - 0xdc, 0x90, 0xfe, 0x31, 0x6e, 0x48, 0xbf, 0x7b, 0xd7, 0x98, 0x7a, 0x3a, 0x23, 0x3c, 0xfe, 0x27, - 0x00, 0x00, 0xff, 0xff, 0x72, 0x04, 0xd9, 0xa7, 0xb9, 0x19, 0x00, 0x00, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0x4b, 0x6f, 0x1b, 0xc9, + 0x11, 0xd6, 0x50, 0xa4, 0x44, 0xb6, 0x56, 0x94, 0xd5, 0x52, 0x24, 0xae, 0x36, 0xa1, 0x04, 0x66, + 0xb1, 0x2b, 0xef, 0xae, 0x86, 0x6b, 0xd9, 0x31, 0xfc, 0x00, 0x8c, 0x88, 0x94, 0x63, 0xcb, 0x90, + 0x22, 0xb9, 0x29, 0x39, 0x88, 0x93, 0x00, 0x6e, 0x0e, 0xdb, 0xd4, 0x58, 0xf3, 0xc2, 0x4c, 0x0f, + 0x63, 0x22, 0x97, 0xdc, 0x13, 0xc0, 0x39, 0xe7, 0x57, 0xe4, 0x18, 0x24, 0xb7, 0x9c, 0x74, 0x09, + 0x60, 0xe4, 0x12, 0x9f, 0x84, 0x98, 0xbe, 0xe6, 0x9a, 0x8b, 0x81, 0x00, 0x41, 0xf7, 0xf4, 0xbc, + 0x67, 0x24, 0x2a, 0x40, 0x74, 0xc8, 0x8d, 0xd3, 0x55, 0xf5, 0x55, 0x75, 0x77, 0x75, 0xd5, 0x57, + 0x04, 0x3f, 0x3c, 0xb9, 0xe3, 0xc8, 0xaa, 0xd9, 0x3c, 0x71, 0xbb, 0xc4, 0x36, 0x08, 0x25, 0x4e, + 0x73, 0x40, 0x8c, 0x9e, 0x69, 0x37, 0x85, 0x00, 0x5b, 0x6a, 0x13, 0x5b, 0x96, 0xd3, 0x1c, 0xdc, + 0xe8, 0x12, 0x8a, 0x6f, 0x34, 0xfb, 0xc4, 0x20, 0x36, 0xa6, 0xa4, 0x27, 0x5b, 0xb6, 0x49, 0x4d, + 0xb8, 0xec, 0x29, 0xca, 0xd8, 0x52, 0x65, 0xa6, 0x28, 0x0b, 0xc5, 0x95, 0x8d, 0xbe, 0x4a, 0x8f, + 0xdd, 0xae, 0xac, 0x98, 0x7a, 0xb3, 0x6f, 0xf6, 0xcd, 0x26, 0xd7, 0xef, 0xba, 0x2f, 0xf9, 0x17, + 0xff, 0xe0, 0xbf, 0x3c, 0x9c, 0x95, 0x46, 0xc4, 0xa1, 0x62, 0xda, 0xa4, 0x39, 0x48, 0xf9, 0x5a, + 0xb9, 0x1e, 0xd1, 0xb1, 0x4c, 0x4d, 0x55, 0x86, 0x79, 0x61, 0xad, 0xdc, 0x0a, 0x55, 0x75, 0xac, + 0x1c, 0xab, 0x06, 0xb1, 0x87, 0x4d, 0xeb, 0xa4, 0xcf, 0x16, 0x9c, 0xa6, 0x4e, 0x28, 0xce, 0x72, + 0xd0, 0xcc, 0xb3, 0xb2, 0x5d, 0x83, 0xaa, 0x3a, 0x49, 0x19, 0xdc, 0xbe, 0xc8, 0xc0, 0x51, 0x8e, + 0x89, 0x8e, 0x53, 0x76, 0x37, 0xf3, 0xec, 0x5c, 0xaa, 0x6a, 0x4d, 0xd5, 0xa0, 0x0e, 0xb5, 0x93, + 0x46, 0x8d, 0x7f, 0x49, 0x00, 0xb6, 0x4d, 0x83, 0xda, 0xa6, 0xa6, 0x11, 0x1b, 0x91, 0x81, 0xea, + 0xa8, 0xa6, 0x01, 0x5f, 0x80, 0x32, 0xdb, 0x4f, 0x0f, 0x53, 0x5c, 0x93, 0xd6, 0xa4, 0xf5, 0x99, + 0xcd, 0x6f, 0xe5, 0xf0, 0x52, 0x02, 0x78, 0xd9, 0x3a, 0xe9, 0xb3, 0x05, 0x47, 0x66, 0xda, 0xf2, + 0xe0, 0x86, 0xbc, 0xdf, 0x7d, 0x45, 0x14, 0xba, 0x47, 0x28, 0x6e, 0xc1, 0xd3, 0xb3, 0xd5, 0x89, + 0xd1, 0xd9, 0x2a, 0x08, 0xd7, 0x50, 0x80, 0x0a, 0xf7, 0x41, 0x91, 0xa3, 0x17, 0x38, 0xfa, 0x46, + 0x2e, 0xba, 0xd8, 0xb4, 0x8c, 0xf0, 0x2f, 0x1f, 0xbe, 0xa6, 0xc4, 0x60, 0xe1, 0xb5, 0x3e, 0x11, + 0xd0, 0xc5, 0x6d, 0x4c, 0x31, 0xe2, 0x40, 0xf0, 0x1b, 0x50, 0xb6, 0x45, 0xf8, 0xb5, 0xc9, 0x35, + 0x69, 0x7d, 0xb2, 0x75, 0x4d, 0x68, 0x95, 0xfd, 0x6d, 0xa1, 0x40, 0xa3, 0x71, 0x2a, 0x81, 0xa5, + 0xf4, 0xbe, 0x77, 0x55, 0x87, 0xc2, 0x9f, 0xa7, 0xf6, 0x2e, 0x8f, 0xb7, 0x77, 0x66, 0xcd, 0x77, + 0x1e, 0x38, 0xf6, 0x57, 0x22, 0xfb, 0x3e, 0x00, 0x25, 0x95, 0x12, 0xdd, 0xa9, 0x15, 0xd6, 0x26, + 0xd7, 0x67, 0x36, 0xbf, 0x96, 0x73, 0x72, 0x5d, 0x4e, 0x47, 0xd7, 0x9a, 0x15, 0xb8, 0xa5, 0x1d, + 0x86, 0x80, 0x3c, 0xa0, 0xc6, 0x6f, 0x0b, 0x00, 0x6c, 0x13, 0x4b, 0x33, 0x87, 0x3a, 0x31, 0xe8, + 0x15, 0x5c, 0xdd, 0x0e, 0x28, 0x3a, 0x16, 0x51, 0xc4, 0xd5, 0x7d, 0x99, 0xbb, 0x83, 0x30, 0xa8, + 0x8e, 0x45, 0x94, 0xf0, 0xd2, 0xd8, 0x17, 0xe2, 0x10, 0xf0, 0x29, 0x98, 0x72, 0x28, 0xa6, 0xae, + 0xc3, 0xaf, 0x6c, 0x66, 0xf3, 0xfa, 0x38, 0x60, 0xdc, 0xa0, 0x55, 0x15, 0x70, 0x53, 0xde, 0x37, + 0x12, 0x40, 0x8d, 0xbf, 0x4f, 0x82, 0x85, 0x50, 0xb9, 0x6d, 0x1a, 0x3d, 0x95, 0xb2, 0x94, 0xbe, + 0x0f, 0x8a, 0x74, 0x68, 0x11, 0x7e, 0x26, 0x95, 0xd6, 0x97, 0x7e, 0x30, 0x87, 0x43, 0x8b, 0x7c, + 0x3c, 0x5b, 0x5d, 0xce, 0x30, 0x61, 0x22, 0xc4, 0x8d, 0xe0, 0x6e, 0x10, 0x67, 0x81, 0x9b, 0xdf, + 0x8a, 0x3b, 0xff, 0x78, 0xb6, 0x9a, 0x51, 0x6b, 0xe4, 0x00, 0x29, 0x1e, 0x22, 0xfc, 0x02, 0x4c, + 0xd9, 0x04, 0x3b, 0xa6, 0x51, 0x2b, 0x72, 0xb4, 0x60, 0x2b, 0x88, 0xaf, 0x22, 0x21, 0x85, 0xd7, + 0xc1, 0xb4, 0x4e, 0x1c, 0x07, 0xf7, 0x49, 0xad, 0xc4, 0x15, 0xe7, 0x84, 0xe2, 0xf4, 0x9e, 0xb7, + 0x8c, 0x7c, 0x39, 0x7c, 0x05, 0xaa, 0x1a, 0x76, 0xe8, 0x91, 0xd5, 0xc3, 0x94, 0x1c, 0xaa, 0x3a, + 0xa9, 0x4d, 0xf1, 0x03, 0xfd, 0x6a, 0xbc, 0xbb, 0x67, 0x16, 0xad, 0x25, 0x81, 0x5e, 0xdd, 0x8d, + 0x21, 0xa1, 0x04, 0x32, 0x1c, 0x00, 0xc8, 0x56, 0x0e, 0x6d, 0x6c, 0x38, 0xde, 0x41, 0x31, 0x7f, + 0xd3, 0x97, 0xf6, 0xb7, 0x22, 0xfc, 0xc1, 0xdd, 0x14, 0x1a, 0xca, 0xf0, 0xd0, 0xf8, 0xa3, 0x04, + 0xaa, 0xe1, 0x35, 0x5d, 0xc1, 0x5b, 0x7d, 0x1c, 0x7f, 0xab, 0xdf, 0x1f, 0x23, 0x39, 0x73, 0xde, + 0xe8, 0x3f, 0x0b, 0x00, 0x86, 0x4a, 0xc8, 0xd4, 0xb4, 0x2e, 0x56, 0x4e, 0xe0, 0x1a, 0x28, 0x1a, + 0x58, 0xf7, 0x73, 0x32, 0x78, 0x20, 0x3f, 0xc6, 0x3a, 0x41, 0x5c, 0x02, 0xdf, 0x48, 0x00, 0xba, + 0xfc, 0xe8, 0x7b, 0x5b, 0x86, 0x61, 0x52, 0xcc, 0x4e, 0xc3, 0x0f, 0xa8, 0x3d, 0x46, 0x40, 0xbe, + 0x2f, 0xf9, 0x28, 0x85, 0xf2, 0xd0, 0xa0, 0xf6, 0x30, 0xbc, 0x85, 0xb4, 0x02, 0xca, 0x70, 0x0d, + 0x7f, 0x06, 0x80, 0x2d, 0x30, 0x0f, 0x4d, 0xf1, 0x6c, 0xf3, 0x6b, 0x80, 0xef, 0xbe, 0x6d, 0x1a, + 0x2f, 0xd5, 0x7e, 0x58, 0x58, 0x50, 0x00, 0x81, 0x22, 0x70, 0x2b, 0x0f, 0xc1, 0x72, 0x4e, 0x9c, + 0xf0, 0x1a, 0x98, 0x3c, 0x21, 0x43, 0xef, 0xa8, 0x10, 0xfb, 0x09, 0x17, 0x41, 0x69, 0x80, 0x35, + 0x97, 0x78, 0x6f, 0x12, 0x79, 0x1f, 0xf7, 0x0a, 0x77, 0xa4, 0xc6, 0x1f, 0x4a, 0xd1, 0x4c, 0x61, + 0xf5, 0x06, 0xae, 0xb3, 0xf6, 0x60, 0x69, 0xaa, 0x82, 0x1d, 0x8e, 0x51, 0x6a, 0x7d, 0xe2, 0xb5, + 0x06, 0x6f, 0x0d, 0x05, 0x52, 0xf8, 0x0b, 0x50, 0x76, 0x88, 0x46, 0x14, 0x6a, 0xda, 0xa2, 0xc4, + 0xdd, 0x1c, 0x33, 0xa7, 0x70, 0x97, 0x68, 0x1d, 0x61, 0xea, 0xc1, 0xfb, 0x5f, 0x28, 0x80, 0x84, + 0x4f, 0x41, 0x99, 0x12, 0xdd, 0xd2, 0x30, 0x25, 0xe2, 0xf4, 0x62, 0x79, 0xc5, 0x6a, 0x07, 0x03, + 0x3b, 0x30, 0x7b, 0x87, 0x42, 0x8d, 0x57, 0xcf, 0x20, 0x4f, 0xfd, 0x55, 0x14, 0xc0, 0xc0, 0x9f, + 0x82, 0xb2, 0x43, 0x59, 0x57, 0xef, 0x0f, 0x79, 0x45, 0x39, 0xaf, 0xad, 0x44, 0xeb, 0xa8, 0x67, + 0x12, 0x42, 0xfb, 0x2b, 0x28, 0x80, 0x83, 0x5b, 0x60, 0x4e, 0x57, 0x0d, 0x44, 0x70, 0x6f, 0xd8, + 0x21, 0x8a, 0x69, 0xf4, 0x1c, 0x5e, 0x8a, 0x4a, 0xad, 0x65, 0x61, 0x34, 0xb7, 0x17, 0x17, 0xa3, + 0xa4, 0x3e, 0xdc, 0x05, 0x8b, 0x7e, 0xdb, 0x7d, 0xac, 0x3a, 0xd4, 0xb4, 0x87, 0xbb, 0xaa, 0xae, + 0x52, 0x5e, 0xa0, 0x4a, 0xad, 0xda, 0xe8, 0x6c, 0x75, 0x11, 0x65, 0xc8, 0x51, 0xa6, 0x15, 0xab, + 0x9d, 0x16, 0x76, 0x1d, 0xd2, 0xe3, 0x05, 0xa7, 0x1c, 0xd6, 0xce, 0x03, 0xbe, 0x8a, 0x84, 0x14, + 0xfe, 0x24, 0x96, 0xa6, 0xe5, 0xcb, 0xa5, 0x69, 0x35, 0x3f, 0x45, 0xe1, 0x11, 0x58, 0xb6, 0x6c, + 0xb3, 0x6f, 0x13, 0xc7, 0xd9, 0x26, 0xb8, 0xa7, 0xa9, 0x06, 0xf1, 0x4f, 0xa6, 0xc2, 0x77, 0xf4, + 0xd9, 0xe8, 0x6c, 0x75, 0xf9, 0x20, 0x5b, 0x05, 0xe5, 0xd9, 0x36, 0xfe, 0x52, 0x04, 0xd7, 0x92, + 0x3d, 0x0e, 0x3e, 0x01, 0xd0, 0xec, 0x3a, 0xc4, 0x1e, 0x90, 0xde, 0x23, 0x8f, 0xb8, 0x31, 0x76, + 0x23, 0x71, 0x76, 0x13, 0xbc, 0xdb, 0xfd, 0x94, 0x06, 0xca, 0xb0, 0xf2, 0xf8, 0x91, 0x78, 0x00, + 0x05, 0x1e, 0x68, 0x84, 0x1f, 0xa5, 0x1e, 0xc1, 0x16, 0x98, 0x13, 0x6f, 0xdf, 0x17, 0xf2, 0x64, + 0x8d, 0xdc, 0xfb, 0x51, 0x5c, 0x8c, 0x92, 0xfa, 0xf0, 0x11, 0x98, 0xc7, 0x03, 0xac, 0x6a, 0xb8, + 0xab, 0x91, 0x00, 0xa4, 0xc8, 0x41, 0x3e, 0x15, 0x20, 0xf3, 0x5b, 0x49, 0x05, 0x94, 0xb6, 0x81, + 0x7b, 0x60, 0xc1, 0x35, 0xd2, 0x50, 0x5e, 0x1e, 0x7e, 0x26, 0xa0, 0x16, 0x8e, 0xd2, 0x2a, 0x28, + 0xcb, 0x0e, 0xbe, 0x00, 0x40, 0xf1, 0x1b, 0xb3, 0x53, 0x9b, 0xe2, 0x95, 0xf4, 0x9b, 0x31, 0xde, + 0x4b, 0xd0, 0xcd, 0xc3, 0x2a, 0x16, 0x2c, 0x39, 0x28, 0x82, 0x09, 0xef, 0x83, 0x59, 0x9b, 0xbd, + 0x80, 0x20, 0xd4, 0x69, 0x1e, 0xea, 0x77, 0x84, 0xd9, 0x2c, 0x8a, 0x0a, 0x51, 0x5c, 0x17, 0xde, + 0x03, 0x55, 0xc5, 0xd4, 0x34, 0x9e, 0xf9, 0x6d, 0xd3, 0x35, 0x28, 0x4f, 0xde, 0x52, 0x0b, 0xb2, + 0xce, 0xdc, 0x8e, 0x49, 0x50, 0x42, 0xb3, 0xf1, 0x67, 0x29, 0xda, 0x66, 0xfc, 0xe7, 0x0c, 0xef, + 0xc5, 0xa8, 0xcf, 0x17, 0x09, 0xea, 0xb3, 0x94, 0xb6, 0x88, 0x30, 0x1f, 0x15, 0xcc, 0xb2, 0xe4, + 0x57, 0x8d, 0xbe, 0x77, 0xe1, 0xa2, 0x24, 0x7e, 0x7b, 0xee, 0x53, 0x0a, 0xb4, 0x23, 0x8d, 0x71, + 0x9e, 0xef, 0x3c, 0x2a, 0x44, 0x71, 0xe4, 0xc6, 0x03, 0x50, 0x8d, 0xbf, 0xc3, 0x18, 0xa7, 0x97, + 0x2e, 0xe4, 0xf4, 0x1f, 0x24, 0xb0, 0x9c, 0xe3, 0x1d, 0x6a, 0xa0, 0xaa, 0xe3, 0xd7, 0x91, 0x1c, + 0xb9, 0x90, 0x1b, 0xb3, 0xa9, 0x49, 0xf6, 0xa6, 0x26, 0x79, 0xc7, 0xa0, 0xfb, 0x76, 0x87, 0xda, + 0xaa, 0xd1, 0xf7, 0xee, 0x61, 0x2f, 0x86, 0x85, 0x12, 0xd8, 0xf0, 0x39, 0x28, 0xeb, 0xf8, 0x75, + 0xc7, 0xb5, 0xfb, 0x59, 0xe7, 0x35, 0x9e, 0x1f, 0xde, 0x3f, 0xf6, 0x04, 0x0a, 0x0a, 0xf0, 0x1a, + 0xfb, 0x60, 0x2d, 0xb6, 0x49, 0x56, 0x2a, 0xc8, 0x4b, 0x57, 0xeb, 0x90, 0xf0, 0xc2, 0xbf, 0x06, + 0x15, 0x0b, 0xdb, 0x54, 0x0d, 0xca, 0x45, 0xa9, 0x35, 0x3b, 0x3a, 0x5b, 0xad, 0x1c, 0xf8, 0x8b, + 0x28, 0x94, 0x37, 0xfe, 0x2d, 0x81, 0x52, 0x47, 0xc1, 0x1a, 0xb9, 0x82, 0xd1, 0x61, 0x3b, 0x36, + 0x3a, 0x34, 0x72, 0x93, 0x88, 0xc7, 0x93, 0x3b, 0x35, 0xec, 0x26, 0xa6, 0x86, 0xcf, 0x2f, 0xc0, + 0x39, 0x7f, 0x60, 0xb8, 0x0b, 0x2a, 0x81, 0xbb, 0x58, 0x95, 0x94, 0x2e, 0xaa, 0x92, 0x8d, 0xdf, + 0x17, 0xc0, 0x4c, 0xc4, 0xc5, 0xe5, 0xac, 0xd9, 0x71, 0x47, 0x88, 0x06, 0x2b, 0x43, 0x9b, 0xe3, + 0x6c, 0x44, 0xf6, 0x49, 0x85, 0xc7, 0xdf, 0xc2, 0xee, 0x9d, 0xe6, 0x1a, 0x0f, 0x40, 0x95, 0x62, + 0xbb, 0x4f, 0xa8, 0x2f, 0xe3, 0x07, 0x56, 0x09, 0x99, 0xfe, 0x61, 0x4c, 0x8a, 0x12, 0xda, 0x2b, + 0xf7, 0xc1, 0x6c, 0xcc, 0xd9, 0xa5, 0x48, 0xd8, 0x1b, 0x76, 0x38, 0x61, 0x72, 0x5e, 0x41, 0x76, + 0x3d, 0x89, 0x65, 0xd7, 0x7a, 0xfe, 0x61, 0x46, 0x9e, 0x4c, 0x5e, 0x8e, 0xa1, 0x44, 0x8e, 0x7d, + 0x35, 0x16, 0xda, 0xf9, 0x99, 0xf6, 0x27, 0x09, 0xcc, 0x45, 0xb4, 0xaf, 0x60, 0x82, 0xd9, 0x89, + 0x4f, 0x30, 0x9f, 0x8f, 0xb3, 0x89, 0x9c, 0x11, 0xe6, 0xaf, 0xa5, 0x58, 0xf0, 0xff, 0xf7, 0xa4, + 0xfa, 0x57, 0x60, 0x71, 0x60, 0x6a, 0xae, 0x4e, 0xda, 0x1a, 0x56, 0x75, 0x5f, 0x81, 0x31, 0x98, + 0xc9, 0xe4, 0x1f, 0x15, 0x01, 0x3c, 0xb1, 0x1d, 0xd5, 0xa1, 0xc4, 0xa0, 0xcf, 0x42, 0xcb, 0xd6, + 0x77, 0x85, 0x93, 0xc5, 0x67, 0x19, 0x70, 0x28, 0xd3, 0x09, 0xfc, 0x01, 0x98, 0x61, 0x04, 0x4e, + 0x55, 0x08, 0x9b, 0x05, 0xc5, 0xf4, 0xbf, 0x20, 0x80, 0x66, 0x3a, 0xa1, 0x08, 0x45, 0xf5, 0xe0, + 0x31, 0x58, 0xb0, 0xcc, 0xde, 0x1e, 0x36, 0x70, 0x9f, 0xb0, 0xb6, 0x77, 0xc0, 0xff, 0xd0, 0xe4, + 0x4c, 0xbb, 0xd2, 0xba, 0xed, 0x33, 0xa5, 0x83, 0xb4, 0xca, 0x47, 0x46, 0x59, 0xd3, 0xcb, 0x9c, + 0x07, 0x64, 0x41, 0x42, 0x1b, 0x54, 0x5d, 0xd1, 0x7e, 0xc4, 0xe0, 0xe1, 0xcd, 0xff, 0x9b, 0xe3, + 0x64, 0xd8, 0x51, 0xcc, 0x32, 0xac, 0x46, 0xf1, 0x75, 0x94, 0xf0, 0x90, 0x3b, 0x48, 0x94, 0xff, + 0x9b, 0x41, 0xa2, 0xf1, 0x9b, 0x22, 0x98, 0x4f, 0x3d, 0x5d, 0xf8, 0xa3, 0x73, 0x18, 0xf7, 0xd2, + 0xff, 0x8c, 0x6d, 0xa7, 0x08, 0xe3, 0xe4, 0x25, 0x08, 0xe3, 0x16, 0x98, 0x53, 0x5c, 0xdb, 0x66, + 0xb3, 0x7e, 0x9c, 0x65, 0x07, 0x54, 0xbd, 0x1d, 0x17, 0xa3, 0xa4, 0x7e, 0x16, 0xdb, 0x2f, 0x5d, + 0x92, 0xed, 0x47, 0xa3, 0x10, 0x8c, 0xcd, 0x4b, 0xbb, 0x74, 0x14, 0x82, 0xb8, 0x25, 0xf5, 0x59, + 0xb7, 0xf2, 0x50, 0x03, 0x84, 0xe9, 0x78, 0xb7, 0x3a, 0x8a, 0x49, 0x51, 0x42, 0x3b, 0x83, 0x39, + 0x57, 0xc6, 0x66, 0xce, 0x7f, 0x93, 0xc0, 0xa7, 0xb9, 0x19, 0x0a, 0xb7, 0x62, 0x04, 0x7a, 0x23, + 0x41, 0xa0, 0xbf, 0x97, 0x6b, 0x18, 0xe1, 0xd1, 0x76, 0x36, 0x8f, 0xbe, 0x3b, 0x1e, 0x8f, 0xce, + 0x20, 0x79, 0x17, 0x13, 0xea, 0xd6, 0xc6, 0xe9, 0xfb, 0xfa, 0xc4, 0xdb, 0xf7, 0xf5, 0x89, 0x77, + 0xef, 0xeb, 0x13, 0xbf, 0x1e, 0xd5, 0xa5, 0xd3, 0x51, 0x5d, 0x7a, 0x3b, 0xaa, 0x4b, 0xef, 0x46, + 0x75, 0xe9, 0x1f, 0xa3, 0xba, 0xf4, 0xbb, 0x0f, 0xf5, 0x89, 0xe7, 0xd3, 0xc2, 0xe3, 0x7f, 0x02, + 0x00, 0x00, 0xff, 0xff, 0x3a, 0x2e, 0x29, 0xcd, 0xb9, 0x19, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/apps/v1beta1/generated.proto b/staging/src/k8s.io/api/apps/v1beta1/generated.proto index 79f49feac03f3..3d2c9dbf0bdde 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/generated.proto +++ b/staging/src/k8s.io/api/apps/v1beta1/generated.proto @@ -208,7 +208,7 @@ message DeploymentStatus { // field as a collision avoidance mechanism when it needs to create the name for the // newest ReplicaSet. // +optional - optional int64 collisionCount = 8; + optional int32 collisionCount = 8; } // DeploymentStrategy describes how to replace existing pods with new ones. @@ -434,7 +434,7 @@ message StatefulSetStatus { // uses this field as a collision avoidance mechanism when it needs to create the name for the // newest ControllerRevision. // +optional - optional int64 collisionCount = 9; + optional int32 collisionCount = 9; } // StatefulSetUpdateStrategy indicates the strategy that the StatefulSet diff --git a/staging/src/k8s.io/api/apps/v1beta1/types.generated.go b/staging/src/k8s.io/api/apps/v1beta1/types.generated.go index faf2b8e711fa1..e687ac9521dec 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/types.generated.go +++ b/staging/src/k8s.io/api/apps/v1beta1/types.generated.go @@ -2812,13 +2812,13 @@ func (x *StatefulSetStatus) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) } } else { if x.CollisionCount == nil { - x.CollisionCount = new(int64) + x.CollisionCount = new(int32) } yym19 := z.DecBinary() _ = yym19 if false { } else { - *((*int64)(x.CollisionCount)) = int64(r.DecodeInt(64)) + *((*int32)(x.CollisionCount)) = int32(r.DecodeInt(32)) } } default: @@ -3010,13 +3010,13 @@ func (x *StatefulSetStatus) codecDecodeSelfFromArray(l int, d *codec1978.Decoder } } else { if x.CollisionCount == nil { - x.CollisionCount = new(int64) + x.CollisionCount = new(int32) } yym36 := z.DecBinary() _ = yym36 if false { } else { - *((*int64)(x.CollisionCount)) = int64(r.DecodeInt(64)) + *((*int32)(x.CollisionCount)) = int32(r.DecodeInt(32)) } } for { @@ -6008,13 +6008,13 @@ func (x *DeploymentStatus) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { } } else { if x.CollisionCount == nil { - x.CollisionCount = new(int64) + x.CollisionCount = new(int32) } yym19 := z.DecBinary() _ = yym19 if false { } else { - *((*int64)(x.CollisionCount)) = int64(r.DecodeInt(64)) + *((*int32)(x.CollisionCount)) = int32(r.DecodeInt(32)) } } default: @@ -6202,13 +6202,13 @@ func (x *DeploymentStatus) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) } } else { if x.CollisionCount == nil { - x.CollisionCount = new(int64) + x.CollisionCount = new(int32) } yym36 := z.DecBinary() _ = yym36 if false { } else { - *((*int64)(x.CollisionCount)) = int64(r.DecodeInt(64)) + *((*int32)(x.CollisionCount)) = int32(r.DecodeInt(32)) } } for { diff --git a/staging/src/k8s.io/api/apps/v1beta1/types.go b/staging/src/k8s.io/api/apps/v1beta1/types.go index 6ff3f8fdee194..36ac1e4229626 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/types.go +++ b/staging/src/k8s.io/api/apps/v1beta1/types.go @@ -244,7 +244,7 @@ type StatefulSetStatus struct { // uses this field as a collision avoidance mechanism when it needs to create the name for the // newest ControllerRevision. // +optional - CollisionCount *int64 `json:"collisionCount,omitempty" protobuf:"varint,9,opt,name=collisionCount"` + CollisionCount *int32 `json:"collisionCount,omitempty" protobuf:"varint,9,opt,name=collisionCount"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -445,7 +445,7 @@ type DeploymentStatus struct { // field as a collision avoidance mechanism when it needs to create the name for the // newest ReplicaSet. // +optional - CollisionCount *int64 `json:"collisionCount,omitempty" protobuf:"varint,8,opt,name=collisionCount"` + CollisionCount *int32 `json:"collisionCount,omitempty" protobuf:"varint,8,opt,name=collisionCount"` } type DeploymentConditionType string diff --git a/staging/src/k8s.io/api/apps/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/apps/v1beta1/zz_generated.deepcopy.go index 4ee37fe96369c..2f046b02057dd 100644 --- a/staging/src/k8s.io/api/apps/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/apps/v1beta1/zz_generated.deepcopy.go @@ -377,7 +377,7 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } @@ -694,7 +694,7 @@ func (in *StatefulSetStatus) DeepCopyInto(out *StatefulSetStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } diff --git a/staging/src/k8s.io/api/apps/v1beta2/generated.pb.go b/staging/src/k8s.io/api/apps/v1beta2/generated.pb.go index bc561c69f2711..07d95d8aca5c7 100644 --- a/staging/src/k8s.io/api/apps/v1beta2/generated.pb.go +++ b/staging/src/k8s.io/api/apps/v1beta2/generated.pb.go @@ -3167,7 +3167,7 @@ func (m *DaemonSetStatus) Unmarshal(dAtA []byte) error { if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CollisionCount", wireType) } - var v int64 + var v int32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -3177,7 +3177,7 @@ func (m *DaemonSetStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= (int32(b) & 0x7F) << shift if b < 0x80 { break } @@ -4213,7 +4213,7 @@ func (m *DeploymentStatus) Unmarshal(dAtA []byte) error { if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CollisionCount", wireType) } - var v int64 + var v int32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -4223,7 +4223,7 @@ func (m *DeploymentStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= (int32(b) & 0x7F) << shift if b < 0x80 { break } @@ -6538,7 +6538,7 @@ func (m *StatefulSetStatus) Unmarshal(dAtA []byte) error { if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CollisionCount", wireType) } - var v int64 + var v int32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -6548,7 +6548,7 @@ func (m *StatefulSetStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= (int32(b) & 0x7F) << shift if b < 0x80 { break } @@ -6804,131 +6804,131 @@ var fileDescriptorGenerated = []byte{ 0x28, 0xc9, 0x40, 0x83, 0x16, 0x30, 0xb5, 0x4b, 0xaf, 0x26, 0x9a, 0x17, 0x66, 0x38, 0x5b, 0x2f, 0x7a, 0xe9, 0xa9, 0x40, 0x81, 0x02, 0xe9, 0xb9, 0xff, 0x44, 0x7b, 0x2a, 0x8a, 0xf6, 0x56, 0x04, 0x85, 0x2f, 0x05, 0x82, 0x5e, 0x92, 0x93, 0x50, 0x6f, 0x8e, 0x3d, 0xf7, 0x12, 0xa0, 0x40, 0x41, - 0x0e, 0xe7, 0xc1, 0x79, 0x58, 0x23, 0x25, 0xde, 0x14, 0xb9, 0xed, 0x90, 0xbf, 0xef, 0xc7, 0x8f, - 0xe4, 0xc7, 0xef, 0xfb, 0x0d, 0x67, 0xc1, 0x8f, 0x4f, 0xde, 0x75, 0x55, 0xcd, 0x6a, 0x9d, 0x78, - 0x47, 0xc4, 0x31, 0x09, 0x25, 0x6e, 0xab, 0x4f, 0xcc, 0xae, 0xe5, 0xb4, 0x44, 0x07, 0xb6, 0xb5, - 0x16, 0xb6, 0x6d, 0xb7, 0xd5, 0xbf, 0x73, 0x44, 0x28, 0x5e, 0x6b, 0xf5, 0x88, 0x49, 0x1c, 0x4c, - 0x49, 0x57, 0xb5, 0x1d, 0x8b, 0x5a, 0x70, 0xc1, 0x07, 0xaa, 0xd8, 0xd6, 0x54, 0x06, 0x54, 0x05, - 0x70, 0xe9, 0x56, 0x4f, 0xa3, 0xc7, 0xde, 0x91, 0xda, 0xb1, 0x8c, 0x56, 0xcf, 0xea, 0x59, 0x2d, - 0x8e, 0x3f, 0xf2, 0x9e, 0xf0, 0x27, 0xfe, 0xc0, 0x7f, 0xf9, 0x3c, 0x4b, 0xcd, 0xd8, 0x80, 0x1d, - 0xcb, 0x21, 0xad, 0xfe, 0x9d, 0xe4, 0x58, 0x4b, 0xd7, 0x63, 0x18, 0xdb, 0xd2, 0xb5, 0xce, 0x40, - 0xb8, 0x95, 0x86, 0xbe, 0x15, 0x41, 0x0d, 0xdc, 0x39, 0xd6, 0x4c, 0xe2, 0x0c, 0x5a, 0xf6, 0x49, - 0x8f, 0x35, 0xb8, 0x2d, 0x83, 0x50, 0x9c, 0x35, 0x40, 0x2b, 0xcf, 0xca, 0xf1, 0x4c, 0xaa, 0x19, - 0x24, 0x65, 0xf0, 0xce, 0x59, 0x06, 0x6e, 0xe7, 0x98, 0x18, 0x38, 0x65, 0xf7, 0x66, 0x9e, 0x9d, - 0x47, 0x35, 0xbd, 0xa5, 0x99, 0xd4, 0xa5, 0x4e, 0xd2, 0xa8, 0xf9, 0x1f, 0x05, 0xc0, 0x0d, 0xcb, - 0xa4, 0x8e, 0xa5, 0xeb, 0xc4, 0x41, 0xa4, 0xaf, 0xb9, 0x9a, 0x65, 0xc2, 0xc7, 0xa0, 0xc6, 0xe6, - 0xd3, 0xc5, 0x14, 0x2f, 0x2a, 0x2b, 0xca, 0xea, 0xe4, 0xda, 0x6d, 0x35, 0xda, 0x94, 0x90, 0x5e, - 0xb5, 0x4f, 0x7a, 0xac, 0xc1, 0x55, 0x19, 0x5a, 0xed, 0xdf, 0x51, 0x77, 0x8f, 0x3e, 0x26, 0x1d, - 0xba, 0x4d, 0x28, 0x6e, 0xc3, 0x67, 0xa7, 0xcb, 0x63, 0xc3, 0xd3, 0x65, 0x10, 0xb5, 0xa1, 0x90, - 0x15, 0xee, 0x82, 0x0a, 0x67, 0x2f, 0x71, 0xf6, 0x5b, 0xb9, 0xec, 0x62, 0xd2, 0x2a, 0xc2, 0xbf, - 0x78, 0xff, 0x29, 0x25, 0x26, 0x73, 0xaf, 0x7d, 0x49, 0x50, 0x57, 0x36, 0x31, 0xc5, 0x88, 0x13, - 0xc1, 0x9b, 0xa0, 0xe6, 0x08, 0xf7, 0x17, 0xcb, 0x2b, 0xca, 0x6a, 0xb9, 0x7d, 0x59, 0xa0, 0x6a, - 0xc1, 0xb4, 0x50, 0x88, 0x68, 0x3e, 0x53, 0xc0, 0x7c, 0x7a, 0xde, 0x5b, 0x9a, 0x4b, 0xe1, 0xcf, - 0x52, 0x73, 0x57, 0x8b, 0xcd, 0x9d, 0x59, 0xf3, 0x99, 0x87, 0x03, 0x07, 0x2d, 0xb1, 0x79, 0xef, - 0x81, 0xaa, 0x46, 0x89, 0xe1, 0x2e, 0x96, 0x56, 0xca, 0xab, 0x93, 0x6b, 0x37, 0xd4, 0x9c, 0x58, - 0x57, 0xd3, 0xde, 0xb5, 0xa7, 0x04, 0x6f, 0xf5, 0x21, 0x63, 0x40, 0x3e, 0x51, 0xf3, 0x37, 0x25, - 0x50, 0xdf, 0xc4, 0xc4, 0xb0, 0xcc, 0x7d, 0x42, 0x47, 0xb0, 0x73, 0x0f, 0x40, 0xc5, 0xb5, 0x49, - 0x47, 0xec, 0xdc, 0xb5, 0xdc, 0x09, 0x84, 0x3e, 0xed, 0xdb, 0xa4, 0x13, 0x6d, 0x19, 0x7b, 0x42, - 0x9c, 0x01, 0xee, 0x81, 0x71, 0x97, 0x62, 0xea, 0xb9, 0x7c, 0xc3, 0x26, 0xd7, 0x56, 0x0b, 0x70, - 0x71, 0x7c, 0x7b, 0x5a, 0xb0, 0x8d, 0xfb, 0xcf, 0x48, 0xf0, 0x34, 0xff, 0xa4, 0x80, 0xa9, 0x10, - 0x3b, 0x82, 0xdd, 0xbc, 0x2f, 0xef, 0x66, 0xf3, 0xec, 0x09, 0xe4, 0x6c, 0xe2, 0xa7, 0xe5, 0x98, - 0xe3, 0x6c, 0x89, 0xe0, 0xcf, 0x41, 0xcd, 0x25, 0x3a, 0xe9, 0x50, 0xcb, 0x11, 0x8e, 0xbf, 0x59, - 0xd0, 0x71, 0x7c, 0x44, 0xf4, 0x7d, 0x61, 0xda, 0xbe, 0xc4, 0x3c, 0x0f, 0x9e, 0x50, 0x48, 0x09, - 0x3f, 0x04, 0x35, 0x4a, 0x0c, 0x5b, 0xc7, 0x94, 0x88, 0x9d, 0x7c, 0x35, 0xee, 0x3c, 0x4b, 0x97, - 0x8c, 0x6c, 0xcf, 0xea, 0x1e, 0x08, 0x18, 0xdf, 0xc6, 0x70, 0x31, 0x82, 0x56, 0x14, 0xd2, 0x40, - 0x1b, 0x4c, 0x7b, 0x76, 0x97, 0x21, 0x29, 0x4b, 0x31, 0xbd, 0x81, 0xd8, 0xd6, 0xdb, 0x67, 0xaf, - 0xca, 0xa1, 0x64, 0xd7, 0x9e, 0x17, 0xa3, 0x4c, 0xcb, 0xed, 0x28, 0xc1, 0x0f, 0xd7, 0xc1, 0x8c, - 0xa1, 0x99, 0x88, 0xe0, 0xee, 0x60, 0x9f, 0x74, 0x2c, 0xb3, 0xeb, 0x2e, 0x56, 0x56, 0x94, 0xd5, - 0x6a, 0x7b, 0x41, 0x10, 0xcc, 0x6c, 0xcb, 0xdd, 0x28, 0x89, 0x87, 0x5b, 0x60, 0x2e, 0x48, 0x0a, - 0x0f, 0x34, 0x97, 0x5a, 0xce, 0x60, 0x4b, 0x33, 0x34, 0xba, 0x38, 0xce, 0x79, 0x16, 0x87, 0xa7, - 0xcb, 0x73, 0x28, 0xa3, 0x1f, 0x65, 0x5a, 0x35, 0xff, 0x58, 0x05, 0x33, 0x89, 0x58, 0x85, 0x8f, - 0xc0, 0x7c, 0xc7, 0x73, 0x1c, 0x62, 0xd2, 0x1d, 0xcf, 0x38, 0x22, 0xce, 0x7e, 0xe7, 0x98, 0x74, - 0x3d, 0x9d, 0x74, 0xf9, 0xb6, 0x56, 0xdb, 0x0d, 0xe1, 0xeb, 0xfc, 0x46, 0x26, 0x0a, 0xe5, 0x58, - 0xc3, 0x0f, 0x00, 0x34, 0x79, 0xd3, 0xb6, 0xe6, 0xba, 0x21, 0x67, 0x89, 0x73, 0x2e, 0x09, 0x4e, - 0xb8, 0x93, 0x42, 0xa0, 0x0c, 0x2b, 0xe6, 0x63, 0x97, 0xb8, 0x9a, 0x43, 0xba, 0x49, 0x1f, 0xcb, - 0xb2, 0x8f, 0x9b, 0x99, 0x28, 0x94, 0x63, 0x0d, 0xdf, 0x06, 0x93, 0xfe, 0x68, 0x7c, 0xcd, 0xc5, - 0xe6, 0xcc, 0x0a, 0xb2, 0xc9, 0x9d, 0xa8, 0x0b, 0xc5, 0x71, 0x6c, 0x6a, 0xd6, 0x91, 0x4b, 0x9c, - 0x3e, 0xe9, 0xde, 0xf7, 0x0b, 0x16, 0xcb, 0xea, 0x55, 0x9e, 0xd5, 0xc3, 0xa9, 0xed, 0xa6, 0x10, - 0x28, 0xc3, 0x8a, 0x4d, 0xcd, 0x8f, 0x9a, 0xd4, 0xd4, 0xc6, 0xe5, 0xa9, 0x1d, 0x66, 0xa2, 0x50, - 0x8e, 0x35, 0x8b, 0x3d, 0xdf, 0xe5, 0xf5, 0x3e, 0xd6, 0x74, 0x7c, 0xa4, 0x93, 0xc5, 0x09, 0x39, - 0xf6, 0x76, 0xe4, 0x6e, 0x94, 0xc4, 0xc3, 0xfb, 0xe0, 0x8a, 0xdf, 0x74, 0x68, 0xe2, 0x90, 0xa4, - 0xc6, 0x49, 0x5e, 0x11, 0x24, 0x57, 0x76, 0x92, 0x00, 0x94, 0xb6, 0x81, 0x77, 0xc1, 0x74, 0xc7, - 0xd2, 0x75, 0x1e, 0x8f, 0x1b, 0x96, 0x67, 0xd2, 0xc5, 0x3a, 0x5f, 0x2b, 0xc8, 0xce, 0xd0, 0x86, - 0xd4, 0x83, 0x12, 0xc8, 0xe6, 0xa7, 0x0a, 0x58, 0xc8, 0x39, 0x87, 0xf0, 0x47, 0xa0, 0x42, 0x07, - 0x36, 0xe1, 0x81, 0x5a, 0x6f, 0xdf, 0x08, 0x52, 0xf8, 0xc1, 0xc0, 0x26, 0x5f, 0x9d, 0x2e, 0x5f, - 0xcd, 0x31, 0x63, 0xdd, 0x88, 0x1b, 0xc2, 0x63, 0x30, 0xc5, 0x6a, 0x98, 0x66, 0xf6, 0x7c, 0x88, - 0x48, 0x35, 0xad, 0xdc, 0x8c, 0x80, 0xe2, 0xe8, 0x28, 0x69, 0x5e, 0x19, 0x9e, 0x2e, 0x4f, 0x49, - 0x7d, 0x48, 0x26, 0x6e, 0xfe, 0xb6, 0x04, 0xc0, 0x26, 0xb1, 0x75, 0x6b, 0x60, 0x10, 0x73, 0x14, - 0x65, 0xf0, 0xa1, 0x54, 0x06, 0x5f, 0xcf, 0xcf, 0x71, 0xa1, 0x53, 0xb9, 0x75, 0xf0, 0xc3, 0x44, - 0x1d, 0xbc, 0x5e, 0x84, 0xec, 0xc5, 0x85, 0xf0, 0xf3, 0x32, 0x98, 0x8d, 0xc0, 0x1b, 0x96, 0xd9, - 0xd5, 0xf8, 0x69, 0xb8, 0x27, 0xed, 0xe8, 0xeb, 0x89, 0x1d, 0x5d, 0xc8, 0x30, 0x89, 0xed, 0xe6, - 0x56, 0xe8, 0x67, 0x89, 0x9b, 0xbf, 0x25, 0x0f, 0xfe, 0xd5, 0xe9, 0x72, 0x86, 0xe2, 0x56, 0x43, - 0x26, 0xd9, 0x45, 0x78, 0x0d, 0x8c, 0x3b, 0x04, 0xbb, 0x96, 0xc9, 0xd3, 0x42, 0x3d, 0x9a, 0x0a, - 0xe2, 0xad, 0x48, 0xf4, 0xc2, 0xeb, 0x60, 0xc2, 0x20, 0xae, 0x8b, 0x7b, 0x84, 0x67, 0x80, 0x7a, - 0x7b, 0x46, 0x00, 0x27, 0xb6, 0xfd, 0x66, 0x14, 0xf4, 0xc3, 0x8f, 0xc1, 0xb4, 0x8e, 0x5d, 0x11, - 0x8e, 0x07, 0x9a, 0x41, 0xf8, 0x19, 0x9f, 0x5c, 0x7b, 0xa3, 0xd8, 0xde, 0x33, 0x8b, 0xa8, 0xf6, - 0x6c, 0x49, 0x4c, 0x28, 0xc1, 0x0c, 0xfb, 0x00, 0xb2, 0x96, 0x03, 0x07, 0x9b, 0xae, 0xbf, 0x50, - 0x6c, 0xbc, 0x89, 0x73, 0x8f, 0x17, 0xe6, 0xb3, 0xad, 0x14, 0x1b, 0xca, 0x18, 0xa1, 0xf9, 0x67, - 0x05, 0x4c, 0x47, 0xdb, 0x34, 0x02, 0x8d, 0xf3, 0x40, 0xd6, 0x38, 0xaf, 0x16, 0x08, 0xce, 0x1c, - 0x91, 0xf3, 0x79, 0x25, 0xee, 0x3a, 0x57, 0x39, 0xab, 0x4c, 0xb5, 0xdb, 0xba, 0xd6, 0xc1, 0xae, - 0x28, 0x87, 0x97, 0x7c, 0xc5, 0xee, 0xb7, 0xa1, 0xb0, 0x57, 0xd2, 0x43, 0xa5, 0x97, 0xab, 0x87, - 0xca, 0xdf, 0x8c, 0x1e, 0xfa, 0x29, 0xa8, 0xb9, 0x81, 0x12, 0xaa, 0x70, 0xca, 0x1b, 0x85, 0x0e, - 0xb6, 0x10, 0x41, 0x21, 0x75, 0x28, 0x7f, 0x42, 0xba, 0x2c, 0xe1, 0x53, 0xfd, 0x36, 0x85, 0x0f, - 0x3b, 0xcc, 0x36, 0xf6, 0x5c, 0xd2, 0xe5, 0x27, 0xa0, 0x16, 0x1d, 0xe6, 0x3d, 0xde, 0x8a, 0x44, - 0x2f, 0x3c, 0x04, 0x0b, 0xb6, 0x63, 0xf5, 0x1c, 0xe2, 0xba, 0x9b, 0x04, 0x77, 0x75, 0xcd, 0x24, - 0xc1, 0x04, 0xea, 0x7c, 0xe0, 0xab, 0xc3, 0xd3, 0xe5, 0x85, 0xbd, 0x6c, 0x08, 0xca, 0xb3, 0x6d, - 0xfe, 0xad, 0x02, 0x2e, 0x27, 0x73, 0x63, 0x8e, 0x8a, 0x50, 0x2e, 0xa4, 0x22, 0x6e, 0xc6, 0xe2, - 0xd4, 0x97, 0x58, 0xb1, 0xb7, 0xcb, 0x54, 0xac, 0xae, 0x83, 0x19, 0xa1, 0x1a, 0x82, 0x4e, 0xa1, - 0xa3, 0xc2, 0xed, 0x39, 0x94, 0xbb, 0x51, 0x12, 0xcf, 0xb4, 0x41, 0x54, 0xf2, 0x03, 0x92, 0x8a, - 0xac, 0x0d, 0xd6, 0x93, 0x00, 0x94, 0xb6, 0x81, 0xdb, 0x60, 0xd6, 0x33, 0xd3, 0x54, 0x7e, 0xb8, - 0x5c, 0x15, 0x54, 0xb3, 0x87, 0x69, 0x08, 0xca, 0xb2, 0x83, 0x8f, 0x01, 0xe8, 0x04, 0x09, 0xdd, - 0x5d, 0x1c, 0xe7, 0x29, 0xe1, 0x66, 0x81, 0xb0, 0x0e, 0xab, 0x40, 0x54, 0x56, 0xc3, 0x26, 0x17, - 0xc5, 0x38, 0xe1, 0x3d, 0x30, 0xe5, 0x70, 0x49, 0x18, 0xb8, 0xea, 0xcb, 0xaa, 0xef, 0x09, 0xb3, - 0x29, 0x14, 0xef, 0x44, 0x32, 0x36, 0x43, 0x09, 0xd5, 0x0a, 0x2b, 0xa1, 0xbf, 0x2a, 0x00, 0xa6, - 0xcf, 0x21, 0xbc, 0x2b, 0x95, 0xcc, 0x6b, 0x89, 0x92, 0x39, 0x9f, 0xb6, 0x88, 0x55, 0x4c, 0x2d, - 0x5b, 0xff, 0xdc, 0x2e, 0xa8, 0x7f, 0xa2, 0x84, 0x5a, 0x4c, 0x00, 0x89, 0x65, 0x18, 0xcd, 0x3d, - 0x40, 0x51, 0x01, 0x14, 0x39, 0xf5, 0x0d, 0x08, 0xa0, 0x18, 0xd9, 0x8b, 0x05, 0xd0, 0xbf, 0x4b, - 0x60, 0x36, 0x02, 0x17, 0x16, 0x40, 0x19, 0x26, 0x2f, 0x4d, 0x00, 0x65, 0x2b, 0x88, 0xf2, 0xcb, - 0x56, 0x10, 0x2f, 0x41, 0x78, 0x71, 0x51, 0x12, 0x2d, 0xdd, 0xff, 0x93, 0x28, 0x89, 0xbc, 0xca, - 0x11, 0x25, 0x7f, 0x28, 0xc5, 0x5d, 0xff, 0xce, 0x8b, 0x92, 0xaf, 0x7f, 0x65, 0xd2, 0xfc, 0x7b, - 0x19, 0x5c, 0x4e, 0x9e, 0x43, 0xa9, 0x40, 0x2a, 0x67, 0x16, 0xc8, 0x3d, 0x30, 0xf7, 0xc4, 0xd3, - 0xf5, 0x01, 0x5f, 0x86, 0x58, 0x95, 0xf4, 0x4b, 0xeb, 0xf7, 0x85, 0xe5, 0xdc, 0x4f, 0x32, 0x30, - 0x28, 0xd3, 0x32, 0xa7, 0xd8, 0x97, 0x2f, 0x54, 0xec, 0x53, 0x15, 0xa8, 0x72, 0x8e, 0x0a, 0x94, - 0x59, 0xb8, 0xab, 0x17, 0x28, 0xdc, 0xe7, 0xab, 0xb4, 0x19, 0x89, 0xeb, 0xac, 0x4a, 0xdb, 0xfc, - 0xb5, 0x02, 0xe6, 0xb3, 0x5f, 0xb8, 0xa1, 0x0e, 0xa6, 0x0d, 0xfc, 0x34, 0x7e, 0x2f, 0x71, 0x56, - 0x11, 0xf1, 0xa8, 0xa6, 0xab, 0xfe, 0x57, 0x06, 0xf5, 0xa1, 0x49, 0x77, 0x9d, 0x7d, 0xea, 0x68, - 0x66, 0xcf, 0xaf, 0xbc, 0xdb, 0x12, 0x17, 0x4a, 0x70, 0x37, 0xbf, 0x54, 0xc0, 0x42, 0x4e, 0xe5, - 0x1b, 0xad, 0x27, 0xf0, 0x23, 0x50, 0x33, 0xf0, 0xd3, 0x7d, 0xcf, 0xe9, 0x65, 0xd5, 0xea, 0x62, - 0xe3, 0xf0, 0xd3, 0xbc, 0x2d, 0x58, 0x50, 0xc8, 0xd7, 0xdc, 0x05, 0x2b, 0xd2, 0x24, 0xd9, 0xc9, - 0x21, 0x4f, 0x3c, 0x9d, 0x1f, 0x22, 0x21, 0x36, 0x6e, 0x80, 0xba, 0x8d, 0x1d, 0xaa, 0x85, 0x52, - 0xb5, 0xda, 0x9e, 0x1a, 0x9e, 0x2e, 0xd7, 0xf7, 0x82, 0x46, 0x14, 0xf5, 0x37, 0xff, 0xab, 0x80, - 0xea, 0x7e, 0x07, 0xeb, 0x64, 0x04, 0xd5, 0x7e, 0x53, 0xaa, 0xf6, 0xf9, 0x17, 0xdd, 0xdc, 0x9f, - 0xdc, 0x42, 0xbf, 0x95, 0x28, 0xf4, 0xaf, 0x9d, 0xc1, 0xf3, 0xe2, 0x1a, 0xff, 0x1e, 0xa8, 0x87, - 0xc3, 0x9d, 0x2f, 0x01, 0x35, 0x7f, 0x5f, 0x02, 0x93, 0xb1, 0x21, 0xce, 0x99, 0xbe, 0x1e, 0x4b, - 0x69, 0x9f, 0x1d, 0xcc, 0xb5, 0x22, 0x13, 0x51, 0x83, 0x14, 0xff, 0xbe, 0x49, 0x9d, 0xf8, 0x0b, - 0x5e, 0x3a, 0xf3, 0xff, 0x10, 0x4c, 0x53, 0xec, 0xf4, 0x08, 0x0d, 0xfa, 0xf8, 0x82, 0xd5, 0xa3, - 0xdb, 0x89, 0x03, 0xa9, 0x17, 0x25, 0xd0, 0x4b, 0xf7, 0xc0, 0x94, 0x34, 0x18, 0xbc, 0x0c, 0xca, - 0x27, 0x64, 0xe0, 0xcb, 0x1e, 0xc4, 0x7e, 0xc2, 0x39, 0x50, 0xed, 0x63, 0xdd, 0xf3, 0xe3, 0xbc, - 0x8e, 0xfc, 0x87, 0xbb, 0xa5, 0x77, 0x95, 0xe6, 0x27, 0x6c, 0x71, 0xa2, 0xe0, 0x1c, 0x41, 0x74, - 0x7d, 0x20, 0x45, 0x57, 0xfe, 0x77, 0xa0, 0xf8, 0x91, 0xc9, 0x8b, 0x31, 0x94, 0x88, 0xb1, 0x37, - 0x0a, 0xb1, 0xbd, 0x38, 0xd2, 0xfe, 0xa2, 0x80, 0x99, 0x18, 0x7a, 0x04, 0x02, 0xe7, 0xa1, 0x2c, - 0x70, 0x5e, 0x2b, 0x32, 0x89, 0x1c, 0x85, 0xf3, 0x8f, 0xaa, 0xe4, 0xfc, 0x77, 0x5e, 0xe2, 0xfc, - 0x12, 0xcc, 0xf5, 0x2d, 0xdd, 0x33, 0xc8, 0x86, 0x8e, 0x35, 0x23, 0x00, 0xb0, 0x2a, 0x5e, 0x4e, - 0xbe, 0x5b, 0x84, 0xf4, 0xc4, 0x71, 0x35, 0x97, 0x12, 0x93, 0x3e, 0x8a, 0x2c, 0x23, 0x1d, 0xf2, - 0x28, 0x83, 0x0e, 0x65, 0x0e, 0x02, 0xdf, 0x06, 0x93, 0x4c, 0x4f, 0x68, 0x1d, 0xb2, 0x83, 0x8d, - 0x40, 0x38, 0x87, 0x5f, 0x3c, 0xf6, 0xa3, 0x2e, 0x14, 0xc7, 0xc1, 0x63, 0x30, 0x6b, 0x5b, 0xdd, - 0x6d, 0x6c, 0xe2, 0x1e, 0x61, 0x65, 0x6f, 0x8f, 0xff, 0x15, 0x81, 0x5f, 0xc6, 0xd4, 0xdb, 0xef, - 0x04, 0x6f, 0xe9, 0x7b, 0x69, 0x08, 0x7b, 0x69, 0xc9, 0x68, 0xe6, 0x2f, 0x2d, 0x59, 0x94, 0xd0, - 0x49, 0x7d, 0xa5, 0xf3, 0xef, 0x2c, 0xd7, 0x8a, 0x44, 0xd8, 0x05, 0xbf, 0xd3, 0xe5, 0xdd, 0x35, - 0xd5, 0x2e, 0xf4, 0x91, 0xed, 0x93, 0x0a, 0xb8, 0x92, 0x3a, 0xba, 0xdf, 0xe2, 0x6d, 0x4f, 0x4a, - 0x2e, 0x96, 0xcf, 0x21, 0x17, 0xd7, 0xc1, 0x8c, 0xf8, 0xbe, 0x97, 0x50, 0x9b, 0xa1, 0x1e, 0xdf, - 0x90, 0xbb, 0x51, 0x12, 0x9f, 0x75, 0xdb, 0x54, 0x3d, 0xe7, 0x6d, 0x53, 0xdc, 0x0b, 0xf1, 0x1f, - 0x0a, 0x3f, 0xf4, 0xd2, 0x5e, 0x88, 0xbf, 0x52, 0x24, 0xf1, 0xac, 0x62, 0xf9, 0xac, 0x21, 0xc3, - 0x84, 0x5c, 0xb1, 0x0e, 0xa5, 0x5e, 0x94, 0x40, 0x7f, 0xad, 0x6f, 0x58, 0xff, 0x54, 0xc0, 0x2b, - 0xb9, 0x51, 0x0a, 0xd7, 0xa5, 0x57, 0xfe, 0x5b, 0x89, 0x57, 0xfe, 0x1f, 0xe4, 0x1a, 0xc6, 0x5e, - 0xfc, 0x9d, 0xec, 0x7b, 0x9c, 0xf7, 0x8a, 0xdd, 0xe3, 0x64, 0x08, 0xbd, 0xb3, 0x2f, 0x74, 0xda, - 0xb7, 0x9e, 0x3d, 0x6f, 0x8c, 0x7d, 0xf6, 0xbc, 0x31, 0xf6, 0xc5, 0xf3, 0xc6, 0xd8, 0xaf, 0x86, - 0x0d, 0xe5, 0xd9, 0xb0, 0xa1, 0x7c, 0x36, 0x6c, 0x28, 0x5f, 0x0c, 0x1b, 0xca, 0xbf, 0x86, 0x0d, - 0xe5, 0x77, 0x5f, 0x36, 0xc6, 0x3e, 0x9a, 0x10, 0x23, 0xfe, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xc2, - 0x59, 0x7e, 0xf2, 0x77, 0x25, 0x00, 0x00, + 0x0e, 0xe7, 0xc1, 0x79, 0x58, 0x23, 0x25, 0xde, 0x14, 0xb9, 0x69, 0xf9, 0xfd, 0xbe, 0x1f, 0x3f, + 0x0e, 0x3f, 0x7e, 0xdf, 0x6f, 0x38, 0x02, 0x3f, 0x3e, 0x79, 0xd7, 0x55, 0x35, 0xab, 0x75, 0xe2, + 0x1d, 0x11, 0xc7, 0x24, 0x94, 0xb8, 0xad, 0x3e, 0x31, 0xbb, 0x96, 0xd3, 0x12, 0x06, 0x6c, 0x6b, + 0x2d, 0x6c, 0xdb, 0x6e, 0xab, 0x7f, 0xe7, 0x88, 0x50, 0xbc, 0xd6, 0xea, 0x11, 0x93, 0x38, 0x98, + 0x92, 0xae, 0x6a, 0x3b, 0x16, 0xb5, 0xe0, 0x82, 0x0f, 0x54, 0xb1, 0xad, 0xa9, 0x0c, 0xa8, 0x0a, + 0xe0, 0xd2, 0xad, 0x9e, 0x46, 0x8f, 0xbd, 0x23, 0xb5, 0x63, 0x19, 0xad, 0x9e, 0xd5, 0xb3, 0x5a, + 0x1c, 0x7f, 0xe4, 0x3d, 0xe1, 0xbf, 0xf8, 0x0f, 0xfe, 0x97, 0xcf, 0xb3, 0xd4, 0x8c, 0x4d, 0xd8, + 0xb1, 0x1c, 0xd2, 0xea, 0xdf, 0x49, 0xce, 0xb5, 0x74, 0x3d, 0x86, 0xb1, 0x2d, 0x5d, 0xeb, 0x0c, + 0x44, 0x58, 0x69, 0xe8, 0x5b, 0x11, 0xd4, 0xc0, 0x9d, 0x63, 0xcd, 0x24, 0xce, 0xa0, 0x65, 0x9f, + 0xf4, 0xd8, 0x80, 0xdb, 0x32, 0x08, 0xc5, 0x59, 0x13, 0xb4, 0xf2, 0xbc, 0x1c, 0xcf, 0xa4, 0x9a, + 0x41, 0x52, 0x0e, 0xef, 0x9c, 0xe5, 0xe0, 0x76, 0x8e, 0x89, 0x81, 0x53, 0x7e, 0x6f, 0xe6, 0xf9, + 0x79, 0x54, 0xd3, 0x5b, 0x9a, 0x49, 0x5d, 0xea, 0x24, 0x9d, 0x9a, 0xff, 0x51, 0x00, 0xdc, 0xb0, + 0x4c, 0xea, 0x58, 0xba, 0x4e, 0x1c, 0x44, 0xfa, 0x9a, 0xab, 0x59, 0x26, 0x7c, 0x0c, 0x6a, 0x6c, + 0x3d, 0x5d, 0x4c, 0xf1, 0xa2, 0xb2, 0xa2, 0xac, 0x4e, 0xae, 0xdd, 0x56, 0xa3, 0x4d, 0x09, 0xe9, + 0x55, 0xfb, 0xa4, 0xc7, 0x06, 0x5c, 0x95, 0xa1, 0xd5, 0xfe, 0x1d, 0x75, 0xf7, 0xe8, 0x63, 0xd2, + 0xa1, 0xdb, 0x84, 0xe2, 0x36, 0x7c, 0x76, 0xba, 0x3c, 0x36, 0x3c, 0x5d, 0x06, 0xd1, 0x18, 0x0a, + 0x59, 0xe1, 0x2e, 0xa8, 0x70, 0xf6, 0x12, 0x67, 0xbf, 0x95, 0xcb, 0x2e, 0x16, 0xad, 0x22, 0xfc, + 0x8b, 0xf7, 0x9f, 0x52, 0x62, 0xb2, 0xf0, 0xda, 0x97, 0x04, 0x75, 0x65, 0x13, 0x53, 0x8c, 0x38, + 0x11, 0xbc, 0x09, 0x6a, 0x8e, 0x08, 0x7f, 0xb1, 0xbc, 0xa2, 0xac, 0x96, 0xdb, 0x97, 0x05, 0xaa, + 0x16, 0x2c, 0x0b, 0x85, 0x88, 0xe6, 0x33, 0x05, 0xcc, 0xa7, 0xd7, 0xbd, 0xa5, 0xb9, 0x14, 0xfe, + 0x2c, 0xb5, 0x76, 0xb5, 0xd8, 0xda, 0x99, 0x37, 0x5f, 0x79, 0x38, 0x71, 0x30, 0x12, 0x5b, 0xf7, + 0x1e, 0xa8, 0x6a, 0x94, 0x18, 0xee, 0x62, 0x69, 0xa5, 0xbc, 0x3a, 0xb9, 0x76, 0x43, 0xcd, 0xc9, + 0x75, 0x35, 0x1d, 0x5d, 0x7b, 0x4a, 0xf0, 0x56, 0x1f, 0x32, 0x06, 0xe4, 0x13, 0x35, 0x7f, 0x53, + 0x02, 0xf5, 0x4d, 0x4c, 0x0c, 0xcb, 0xdc, 0x27, 0x74, 0x04, 0x3b, 0xf7, 0x00, 0x54, 0x5c, 0x9b, + 0x74, 0xc4, 0xce, 0x5d, 0xcb, 0x5d, 0x40, 0x18, 0xd3, 0xbe, 0x4d, 0x3a, 0xd1, 0x96, 0xb1, 0x5f, + 0x88, 0x33, 0xc0, 0x3d, 0x30, 0xee, 0x52, 0x4c, 0x3d, 0x97, 0x6f, 0xd8, 0xe4, 0xda, 0x6a, 0x01, + 0x2e, 0x8e, 0x6f, 0x4f, 0x0b, 0xb6, 0x71, 0xff, 0x37, 0x12, 0x3c, 0xcd, 0x3f, 0x29, 0x60, 0x2a, + 0xc4, 0x8e, 0x60, 0x37, 0xef, 0xcb, 0xbb, 0xd9, 0x3c, 0x7b, 0x01, 0x39, 0x9b, 0xf8, 0x69, 0x39, + 0x16, 0x38, 0x7b, 0x44, 0xf0, 0xe7, 0xa0, 0xe6, 0x12, 0x9d, 0x74, 0xa8, 0xe5, 0x88, 0xc0, 0xdf, + 0x2c, 0x18, 0x38, 0x3e, 0x22, 0xfa, 0xbe, 0x70, 0x6d, 0x5f, 0x62, 0x91, 0x07, 0xbf, 0x50, 0x48, + 0x09, 0x3f, 0x04, 0x35, 0x4a, 0x0c, 0x5b, 0xc7, 0x94, 0x88, 0x9d, 0x7c, 0x35, 0x1e, 0x3c, 0x2b, + 0x97, 0x8c, 0x6c, 0xcf, 0xea, 0x1e, 0x08, 0x18, 0xdf, 0xc6, 0xf0, 0x61, 0x04, 0xa3, 0x28, 0xa4, + 0x81, 0x36, 0x98, 0xf6, 0xec, 0x2e, 0x43, 0x52, 0x56, 0x62, 0x7a, 0x03, 0xb1, 0xad, 0xb7, 0xcf, + 0x7e, 0x2a, 0x87, 0x92, 0x5f, 0x7b, 0x5e, 0xcc, 0x32, 0x2d, 0x8f, 0xa3, 0x04, 0x3f, 0x5c, 0x07, + 0x33, 0x86, 0x66, 0x22, 0x82, 0xbb, 0x83, 0x7d, 0xd2, 0xb1, 0xcc, 0xae, 0xbb, 0x58, 0x59, 0x51, + 0x56, 0xab, 0xed, 0x05, 0x41, 0x30, 0xb3, 0x2d, 0x9b, 0x51, 0x12, 0x0f, 0xb7, 0xc0, 0x5c, 0x50, + 0x14, 0x1e, 0x68, 0x2e, 0xb5, 0x9c, 0xc1, 0x96, 0x66, 0x68, 0x74, 0x71, 0x9c, 0xf3, 0x2c, 0x0e, + 0x4f, 0x97, 0xe7, 0x50, 0x86, 0x1d, 0x65, 0x7a, 0x35, 0xff, 0x58, 0x05, 0x33, 0x89, 0x5c, 0x85, + 0x8f, 0xc0, 0x7c, 0xc7, 0x73, 0x1c, 0x62, 0xd2, 0x1d, 0xcf, 0x38, 0x22, 0xce, 0x7e, 0xe7, 0x98, + 0x74, 0x3d, 0x9d, 0x74, 0xf9, 0xb6, 0x56, 0xdb, 0x0d, 0x11, 0xeb, 0xfc, 0x46, 0x26, 0x0a, 0xe5, + 0x78, 0xc3, 0x0f, 0x00, 0x34, 0xf9, 0xd0, 0xb6, 0xe6, 0xba, 0x21, 0x67, 0x89, 0x73, 0x2e, 0x09, + 0x4e, 0xb8, 0x93, 0x42, 0xa0, 0x0c, 0x2f, 0x16, 0x63, 0x97, 0xb8, 0x9a, 0x43, 0xba, 0xc9, 0x18, + 0xcb, 0x72, 0x8c, 0x9b, 0x99, 0x28, 0x94, 0xe3, 0x0d, 0xdf, 0x06, 0x93, 0xfe, 0x6c, 0xfc, 0x99, + 0x8b, 0xcd, 0x99, 0x15, 0x64, 0x93, 0x3b, 0x91, 0x09, 0xc5, 0x71, 0x6c, 0x69, 0xd6, 0x91, 0x4b, + 0x9c, 0x3e, 0xe9, 0xde, 0xf7, 0x1b, 0x16, 0xab, 0xea, 0x55, 0x5e, 0xd5, 0xc3, 0xa5, 0xed, 0xa6, + 0x10, 0x28, 0xc3, 0x8b, 0x2d, 0xcd, 0xcf, 0x9a, 0xd4, 0xd2, 0xc6, 0xe5, 0xa5, 0x1d, 0x66, 0xa2, + 0x50, 0x8e, 0x37, 0xcb, 0x3d, 0x3f, 0xe4, 0xf5, 0x3e, 0xd6, 0x74, 0x7c, 0xa4, 0x93, 0xc5, 0x09, + 0x39, 0xf7, 0x76, 0x64, 0x33, 0x4a, 0xe2, 0xe1, 0x7d, 0x70, 0xc5, 0x1f, 0x3a, 0x34, 0x71, 0x48, + 0x52, 0xe3, 0x24, 0xaf, 0x08, 0x92, 0x2b, 0x3b, 0x49, 0x00, 0x4a, 0xfb, 0xc0, 0xbb, 0x60, 0xba, + 0x63, 0xe9, 0x3a, 0xcf, 0xc7, 0x0d, 0xcb, 0x33, 0xe9, 0x62, 0x9d, 0xb3, 0x40, 0x76, 0x86, 0x36, + 0x24, 0x0b, 0x4a, 0x20, 0x9b, 0x9f, 0x2a, 0x60, 0x21, 0xe7, 0x1c, 0xc2, 0x1f, 0x81, 0x0a, 0x1d, + 0xd8, 0x84, 0x27, 0x6a, 0xbd, 0x7d, 0x23, 0x28, 0xe1, 0x07, 0x03, 0x9b, 0x7c, 0x75, 0xba, 0x7c, + 0x35, 0xc7, 0x8d, 0x99, 0x11, 0x77, 0x84, 0xc7, 0x60, 0x8a, 0xf5, 0x30, 0xcd, 0xec, 0xf9, 0x10, + 0x51, 0x6a, 0x5a, 0xb9, 0x15, 0x01, 0xc5, 0xd1, 0x51, 0xd1, 0xbc, 0x32, 0x3c, 0x5d, 0x9e, 0x92, + 0x6c, 0x48, 0x26, 0x6e, 0xfe, 0xb6, 0x04, 0xc0, 0x26, 0xb1, 0x75, 0x6b, 0x60, 0x10, 0x73, 0x14, + 0x6d, 0xf0, 0xa1, 0xd4, 0x06, 0x5f, 0xcf, 0xaf, 0x71, 0x61, 0x50, 0xb9, 0x7d, 0xf0, 0xc3, 0x44, + 0x1f, 0xbc, 0x5e, 0x84, 0xec, 0xc5, 0x8d, 0xf0, 0xf3, 0x32, 0x98, 0x8d, 0xc0, 0x1b, 0x96, 0xd9, + 0xd5, 0xf8, 0x69, 0xb8, 0x27, 0xed, 0xe8, 0xeb, 0x89, 0x1d, 0x5d, 0xc8, 0x70, 0x89, 0xed, 0xe6, + 0x56, 0x18, 0x67, 0x89, 0xbb, 0xbf, 0x25, 0x4f, 0xfe, 0xd5, 0xe9, 0x72, 0x86, 0xe2, 0x56, 0x43, + 0x26, 0x39, 0x44, 0x78, 0x0d, 0x8c, 0x3b, 0x04, 0xbb, 0x96, 0xc9, 0xcb, 0x42, 0x3d, 0x5a, 0x0a, + 0xe2, 0xa3, 0x48, 0x58, 0xe1, 0x75, 0x30, 0x61, 0x10, 0xd7, 0xc5, 0x3d, 0xc2, 0x2b, 0x40, 0xbd, + 0x3d, 0x23, 0x80, 0x13, 0xdb, 0xfe, 0x30, 0x0a, 0xec, 0xf0, 0x63, 0x30, 0xad, 0x63, 0x57, 0xa4, + 0xe3, 0x81, 0x66, 0x10, 0x7e, 0xc6, 0x27, 0xd7, 0xde, 0x28, 0xb6, 0xf7, 0xcc, 0x23, 0xea, 0x3d, + 0x5b, 0x12, 0x13, 0x4a, 0x30, 0xc3, 0x3e, 0x80, 0x6c, 0xe4, 0xc0, 0xc1, 0xa6, 0xeb, 0x3f, 0x28, + 0x36, 0xdf, 0xc4, 0xb9, 0xe7, 0x0b, 0xeb, 0xd9, 0x56, 0x8a, 0x0d, 0x65, 0xcc, 0xd0, 0xfc, 0xb3, + 0x02, 0xa6, 0xa3, 0x6d, 0x1a, 0x81, 0xc6, 0x79, 0x20, 0x6b, 0x9c, 0x57, 0x0b, 0x24, 0x67, 0x8e, + 0xc8, 0xf9, 0xbc, 0x12, 0x0f, 0x9d, 0xab, 0x9c, 0x55, 0xa6, 0xda, 0x6d, 0x5d, 0xeb, 0x60, 0x57, + 0xb4, 0xc3, 0x4b, 0xbe, 0x62, 0xf7, 0xc7, 0x50, 0x68, 0x95, 0xf4, 0x50, 0xe9, 0xe5, 0xea, 0xa1, + 0xf2, 0x37, 0xa3, 0x87, 0x7e, 0x0a, 0x6a, 0x6e, 0xa0, 0x84, 0x2a, 0x9c, 0xf2, 0x46, 0xa1, 0x83, + 0x2d, 0x44, 0x50, 0x48, 0x1d, 0xca, 0x9f, 0x90, 0x2e, 0x4b, 0xf8, 0x54, 0xbf, 0x4d, 0xe1, 0xc3, + 0x0e, 0xb3, 0x8d, 0x3d, 0x97, 0x74, 0xf9, 0x09, 0xa8, 0x45, 0x87, 0x79, 0x8f, 0x8f, 0x22, 0x61, + 0x85, 0x87, 0x60, 0xc1, 0x76, 0xac, 0x9e, 0x43, 0x5c, 0x77, 0x93, 0xe0, 0xae, 0xae, 0x99, 0x24, + 0x58, 0x80, 0xdf, 0xb2, 0xae, 0x0e, 0x4f, 0x97, 0x17, 0xf6, 0xb2, 0x21, 0x28, 0xcf, 0xb7, 0xf9, + 0xb7, 0x0a, 0xb8, 0x9c, 0xac, 0x8d, 0x39, 0x2a, 0x42, 0xb9, 0x90, 0x8a, 0xb8, 0x19, 0xcb, 0x53, + 0x5f, 0x62, 0xc5, 0xde, 0x2e, 0x53, 0xb9, 0xba, 0x0e, 0x66, 0x84, 0x6a, 0x08, 0x8c, 0x42, 0x47, + 0x85, 0xdb, 0x73, 0x28, 0x9b, 0x51, 0x12, 0xcf, 0xb4, 0x41, 0xd4, 0xf2, 0x03, 0x92, 0x8a, 0xac, + 0x0d, 0xd6, 0x93, 0x00, 0x94, 0xf6, 0x81, 0xdb, 0x60, 0xd6, 0x33, 0xd3, 0x54, 0x7e, 0xba, 0x5c, + 0x15, 0x54, 0xb3, 0x87, 0x69, 0x08, 0xca, 0xf2, 0x83, 0x8f, 0x01, 0xe8, 0x04, 0x05, 0xdd, 0x5d, + 0x1c, 0xe7, 0x25, 0xe1, 0x66, 0x81, 0xb4, 0x0e, 0xbb, 0x40, 0xd4, 0x56, 0xc3, 0x21, 0x17, 0xc5, + 0x38, 0xe1, 0x3d, 0x30, 0xe5, 0x70, 0x49, 0x18, 0x84, 0xea, 0xcb, 0xaa, 0xef, 0x09, 0xb7, 0x29, + 0x14, 0x37, 0x22, 0x19, 0x9b, 0xa1, 0x84, 0x6a, 0x85, 0x95, 0xd0, 0x5f, 0x15, 0x00, 0xd3, 0xe7, + 0x10, 0xde, 0x95, 0x5a, 0xe6, 0xb5, 0x44, 0xcb, 0x9c, 0x4f, 0x7b, 0xc4, 0x3a, 0xa6, 0x96, 0xad, + 0x7f, 0x6e, 0x17, 0xd4, 0x3f, 0x51, 0x41, 0x2d, 0x26, 0x80, 0xc4, 0x63, 0x18, 0xcd, 0x3d, 0x40, + 0x51, 0x01, 0x14, 0x05, 0xf5, 0x0d, 0x08, 0xa0, 0x18, 0xd9, 0x8b, 0x05, 0xd0, 0xbf, 0x4b, 0x60, + 0x36, 0x02, 0x17, 0x16, 0x40, 0x19, 0x2e, 0x2f, 0x4d, 0x00, 0x65, 0x2b, 0x88, 0xf2, 0xcb, 0x56, + 0x10, 0x2f, 0x41, 0x78, 0x71, 0x51, 0x12, 0x3d, 0xba, 0xff, 0x27, 0x51, 0x12, 0x45, 0x95, 0x23, + 0x4a, 0xfe, 0x50, 0x8a, 0x87, 0xfe, 0x9d, 0x17, 0x25, 0x5f, 0xff, 0xca, 0xa4, 0xf9, 0xf7, 0x32, + 0xb8, 0x9c, 0x3c, 0x87, 0x52, 0x83, 0x54, 0xce, 0x6c, 0x90, 0x7b, 0x60, 0xee, 0x89, 0xa7, 0xeb, + 0x03, 0xfe, 0x18, 0x62, 0x5d, 0xd2, 0x6f, 0xad, 0xdf, 0x17, 0x9e, 0x73, 0x3f, 0xc9, 0xc0, 0xa0, + 0x4c, 0xcf, 0x9c, 0x66, 0x5f, 0xbe, 0x50, 0xb3, 0x4f, 0x75, 0xa0, 0xca, 0x39, 0x3a, 0x50, 0x66, + 0xe3, 0xae, 0x5e, 0xa0, 0x71, 0x9f, 0xaf, 0xd3, 0x66, 0x14, 0xae, 0xb3, 0x3a, 0x6d, 0xf3, 0xd7, + 0x0a, 0x98, 0xcf, 0x7e, 0xe1, 0x86, 0x3a, 0x98, 0x36, 0xf0, 0xd3, 0xf8, 0xbd, 0xc4, 0x59, 0x4d, + 0xc4, 0xa3, 0x9a, 0xae, 0xfa, 0x5f, 0x19, 0xd4, 0x87, 0x26, 0xdd, 0x75, 0xf6, 0xa9, 0xa3, 0x99, + 0x3d, 0xbf, 0xf3, 0x6e, 0x4b, 0x5c, 0x28, 0xc1, 0xdd, 0xfc, 0x52, 0x01, 0x0b, 0x39, 0x9d, 0x6f, + 0xb4, 0x91, 0xc0, 0x8f, 0x40, 0xcd, 0xc0, 0x4f, 0xf7, 0x3d, 0xa7, 0x97, 0xd5, 0xab, 0x8b, 0xcd, + 0xc3, 0x4f, 0xf3, 0xb6, 0x60, 0x41, 0x21, 0x5f, 0x73, 0x17, 0xac, 0x48, 0x8b, 0x64, 0x27, 0x87, + 0x3c, 0xf1, 0x74, 0x7e, 0x88, 0x84, 0xd8, 0xb8, 0x01, 0xea, 0x36, 0x76, 0xa8, 0x16, 0x4a, 0xd5, + 0x6a, 0x7b, 0x6a, 0x78, 0xba, 0x5c, 0xdf, 0x0b, 0x06, 0x51, 0x64, 0x6f, 0xfe, 0x57, 0x01, 0xd5, + 0xfd, 0x0e, 0xd6, 0xc9, 0x08, 0xba, 0xfd, 0xa6, 0xd4, 0xed, 0xf3, 0x2f, 0xba, 0x79, 0x3c, 0xb9, + 0x8d, 0x7e, 0x2b, 0xd1, 0xe8, 0x5f, 0x3b, 0x83, 0xe7, 0xc5, 0x3d, 0xfe, 0x3d, 0x50, 0x0f, 0xa7, + 0x3b, 0x5f, 0x01, 0x6a, 0xfe, 0xbe, 0x04, 0x26, 0x63, 0x53, 0x9c, 0xb3, 0x7c, 0x3d, 0x96, 0xca, + 0x3e, 0x3b, 0x98, 0x6b, 0x45, 0x16, 0xa2, 0x06, 0x25, 0xfe, 0x7d, 0x93, 0x3a, 0xf1, 0x17, 0xbc, + 0x74, 0xe5, 0xff, 0x21, 0x98, 0xa6, 0xd8, 0xe9, 0x11, 0x1a, 0xd8, 0xf8, 0x03, 0xab, 0x47, 0xb7, + 0x13, 0x07, 0x92, 0x15, 0x25, 0xd0, 0x4b, 0xf7, 0xc0, 0x94, 0x34, 0x19, 0xbc, 0x0c, 0xca, 0x27, + 0x64, 0xe0, 0xcb, 0x1e, 0xc4, 0xfe, 0x84, 0x73, 0xa0, 0xda, 0xc7, 0xba, 0xe7, 0xe7, 0x79, 0x1d, + 0xf9, 0x3f, 0xee, 0x96, 0xde, 0x55, 0x9a, 0x9f, 0xb0, 0x87, 0x13, 0x25, 0xe7, 0x08, 0xb2, 0xeb, + 0x03, 0x29, 0xbb, 0xf2, 0xbf, 0x03, 0xc5, 0x8f, 0x4c, 0x5e, 0x8e, 0xa1, 0x44, 0x8e, 0xbd, 0x51, + 0x88, 0xed, 0xc5, 0x99, 0xf6, 0x17, 0x05, 0xcc, 0xc4, 0xd0, 0x23, 0x10, 0x38, 0x0f, 0x65, 0x81, + 0xf3, 0x5a, 0x91, 0x45, 0xe4, 0x28, 0x9c, 0x7f, 0x54, 0xa5, 0xe0, 0xbf, 0xf3, 0x12, 0xe7, 0x97, + 0x60, 0xae, 0x6f, 0xe9, 0x9e, 0x41, 0x36, 0x74, 0xac, 0x19, 0x01, 0x80, 0x75, 0xf1, 0x72, 0xf2, + 0xdd, 0x22, 0xa4, 0x27, 0x8e, 0xab, 0xb9, 0x94, 0x98, 0xf4, 0x51, 0xe4, 0x19, 0xe9, 0x90, 0x47, + 0x19, 0x74, 0x28, 0x73, 0x12, 0xf8, 0x36, 0x98, 0x64, 0x7a, 0x42, 0xeb, 0x90, 0x1d, 0x6c, 0x04, + 0xc2, 0x39, 0xfc, 0xe2, 0xb1, 0x1f, 0x99, 0x50, 0x1c, 0x07, 0x8f, 0xc1, 0xac, 0x6d, 0x75, 0xb7, + 0xb1, 0x89, 0x7b, 0x84, 0xb5, 0xbd, 0x3d, 0xfe, 0xaf, 0x08, 0xfc, 0x32, 0xa6, 0xde, 0x7e, 0x27, + 0x78, 0x4b, 0xdf, 0x4b, 0x43, 0xd8, 0x4b, 0x4b, 0xc6, 0x30, 0x7f, 0x69, 0xc9, 0xa2, 0x84, 0x4e, + 0xea, 0x2b, 0x9d, 0x7f, 0x67, 0xb9, 0x56, 0x24, 0xc3, 0x2e, 0xf8, 0x9d, 0x2e, 0xef, 0xae, 0xa9, + 0x76, 0xa1, 0x8f, 0x6c, 0x9f, 0x54, 0xc0, 0x95, 0xd4, 0xd1, 0xfd, 0x16, 0x6f, 0x7b, 0x52, 0x72, + 0xb1, 0x7c, 0x0e, 0xb9, 0xb8, 0x0e, 0x66, 0xc4, 0xf7, 0xbd, 0x84, 0xda, 0x0c, 0xf5, 0xf8, 0x86, + 0x6c, 0x46, 0x49, 0x7c, 0xd6, 0x6d, 0x53, 0xf5, 0x9c, 0xb7, 0x4d, 0xf1, 0x28, 0xc4, 0xff, 0x50, + 0xf8, 0xa9, 0x97, 0x8e, 0x42, 0xfc, 0x2b, 0x45, 0x12, 0xcf, 0x3a, 0x96, 0xcf, 0x1a, 0x32, 0x4c, + 0xc8, 0x1d, 0xeb, 0x50, 0xb2, 0xa2, 0x04, 0xfa, 0x6b, 0x7d, 0xc3, 0xfa, 0xa7, 0x02, 0x5e, 0xc9, + 0xcd, 0x52, 0xb8, 0x2e, 0xbd, 0xf2, 0xdf, 0x4a, 0xbc, 0xf2, 0xff, 0x20, 0xd7, 0x31, 0xf6, 0xe2, + 0xef, 0x64, 0xdf, 0xe3, 0xbc, 0x57, 0xec, 0x1e, 0x27, 0x43, 0xe8, 0x9d, 0x7d, 0xa1, 0xd3, 0xbe, + 0xf5, 0xec, 0x79, 0x63, 0xec, 0xb3, 0xe7, 0x8d, 0xb1, 0x2f, 0x9e, 0x37, 0xc6, 0x7e, 0x35, 0x6c, + 0x28, 0xcf, 0x86, 0x0d, 0xe5, 0xb3, 0x61, 0x43, 0xf9, 0x62, 0xd8, 0x50, 0xfe, 0x35, 0x6c, 0x28, + 0xbf, 0xfb, 0xb2, 0x31, 0xf6, 0xd1, 0x84, 0x98, 0xf1, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe8, + 0x1d, 0x40, 0xdd, 0x77, 0x25, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/apps/v1beta2/generated.proto b/staging/src/k8s.io/api/apps/v1beta2/generated.proto index 432e4225017c8..3c29c96d36a4c 100644 --- a/staging/src/k8s.io/api/apps/v1beta2/generated.proto +++ b/staging/src/k8s.io/api/apps/v1beta2/generated.proto @@ -174,7 +174,7 @@ message DaemonSetStatus { // uses this field as a collision avoidance mechanism when it needs to // create the name for the newest ControllerRevision. // +optional - optional int64 collisionCount = 9; + optional int32 collisionCount = 9; } // DaemonSetUpdateStrategy is a struct used to control the update strategy for a DaemonSet. @@ -318,7 +318,7 @@ message DeploymentStatus { // field as a collision avoidance mechanism when it needs to create the name for the // newest ReplicaSet. // +optional - optional int64 collisionCount = 8; + optional int32 collisionCount = 8; } // DeploymentStrategy describes how to replace existing pods with new ones. @@ -672,7 +672,7 @@ message StatefulSetStatus { // uses this field as a collision avoidance mechanism when it needs to create the name for the // newest ControllerRevision. // +optional - optional int64 collisionCount = 9; + optional int32 collisionCount = 9; } // StatefulSetUpdateStrategy indicates the strategy that the StatefulSet diff --git a/staging/src/k8s.io/api/apps/v1beta2/types.go b/staging/src/k8s.io/api/apps/v1beta2/types.go index d040d3de2a005..f1bebdccfa7d0 100644 --- a/staging/src/k8s.io/api/apps/v1beta2/types.go +++ b/staging/src/k8s.io/api/apps/v1beta2/types.go @@ -251,7 +251,7 @@ type StatefulSetStatus struct { // uses this field as a collision avoidance mechanism when it needs to create the name for the // newest ControllerRevision. // +optional - CollisionCount *int64 `json:"collisionCount,omitempty" protobuf:"varint,9,opt,name=collisionCount"` + CollisionCount *int32 `json:"collisionCount,omitempty" protobuf:"varint,9,opt,name=collisionCount"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -425,7 +425,7 @@ type DeploymentStatus struct { // field as a collision avoidance mechanism when it needs to create the name for the // newest ReplicaSet. // +optional - CollisionCount *int64 `json:"collisionCount,omitempty" protobuf:"varint,8,opt,name=collisionCount"` + CollisionCount *int32 `json:"collisionCount,omitempty" protobuf:"varint,8,opt,name=collisionCount"` } type DeploymentConditionType string @@ -598,7 +598,7 @@ type DaemonSetStatus struct { // uses this field as a collision avoidance mechanism when it needs to // create the name for the newest ControllerRevision. // +optional - CollisionCount *int64 `json:"collisionCount,omitempty" protobuf:"varint,9,opt,name=collisionCount"` + CollisionCount *int32 `json:"collisionCount,omitempty" protobuf:"varint,9,opt,name=collisionCount"` } // +genclient diff --git a/staging/src/k8s.io/api/apps/v1beta2/zz_generated.deepcopy.go b/staging/src/k8s.io/api/apps/v1beta2/zz_generated.deepcopy.go index b631371e7eca1..124f84a9d4ef8 100644 --- a/staging/src/k8s.io/api/apps/v1beta2/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/apps/v1beta2/zz_generated.deepcopy.go @@ -327,7 +327,7 @@ func (in *DaemonSetStatus) DeepCopyInto(out *DaemonSetStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } @@ -519,7 +519,7 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } @@ -974,7 +974,7 @@ func (in *StatefulSetStatus) DeepCopyInto(out *StatefulSetStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } diff --git a/staging/src/k8s.io/api/authentication/OWNERS b/staging/src/k8s.io/api/authentication/OWNERS index 4135522b21d5f..95235234107c9 100755 --- a/staging/src/k8s.io/api/authentication/OWNERS +++ b/staging/src/k8s.io/api/authentication/OWNERS @@ -7,3 +7,4 @@ reviewers: - timothysc - mbohlool - jianhuiz +- enj diff --git a/staging/src/k8s.io/api/authorization/OWNERS b/staging/src/k8s.io/api/authorization/OWNERS index 2fef50443b652..a68d7eef5775a 100755 --- a/staging/src/k8s.io/api/authorization/OWNERS +++ b/staging/src/k8s.io/api/authorization/OWNERS @@ -15,3 +15,4 @@ reviewers: - mbohlool - david-mcmahon - jianhuiz +- enj diff --git a/staging/src/k8s.io/api/certificates/OWNERS b/staging/src/k8s.io/api/certificates/OWNERS index c67bd1172a1b3..6066d2c1218f7 100755 --- a/staging/src/k8s.io/api/certificates/OWNERS +++ b/staging/src/k8s.io/api/certificates/OWNERS @@ -12,3 +12,4 @@ reviewers: - mbohlool - david-mcmahon - jianhuiz +- enj diff --git a/staging/src/k8s.io/api/core/v1/annotation_key_constants.go b/staging/src/k8s.io/api/core/v1/annotation_key_constants.go index 6c5deb65fd336..e623913fdd129 100644 --- a/staging/src/k8s.io/api/core/v1/annotation_key_constants.go +++ b/staging/src/k8s.io/api/core/v1/annotation_key_constants.go @@ -47,6 +47,8 @@ const ( // CreatedByAnnotation represents the key used to store the spec(json) // used to create the resource. + // This field is deprecated in favor of ControllerRef (see #44407). + // TODO(#50720): Remove this field in v1.9. CreatedByAnnotation = "kubernetes.io/created-by" // PreferAvoidPodsAnnotationKey represents the key of preferAvoidPods data (json serialized) diff --git a/staging/src/k8s.io/api/core/v1/generated.pb.go b/staging/src/k8s.io/api/core/v1/generated.pb.go index e127916a4027b..157ebb2269acf 100644 --- a/staging/src/k8s.io/api/core/v1/generated.pb.go +++ b/staging/src/k8s.io/api/core/v1/generated.pb.go @@ -30,11 +30,14 @@ limitations under the License. AttachedVolume AvoidPods AzureDiskVolumeSource + AzureFilePersistentVolumeSource AzureFileVolumeSource Binding Capabilities + CephFSPersistentVolumeSource CephFSVolumeSource CinderVolumeSource + ClientIPConfig ComponentCondition ComponentStatus ComponentStatusList @@ -174,6 +177,7 @@ limitations under the License. SecretKeySelector SecretList SecretProjection + SecretReference SecretVolumeSource SecurityContext SerializedReference @@ -185,6 +189,7 @@ limitations under the License. ServiceProxyOptions ServiceSpec ServiceStatus + SessionAffinityConfig StorageOSPersistentVolumeSource StorageOSVolumeSource Sysctl @@ -250,708 +255,734 @@ func (m *AzureDiskVolumeSource) Reset() { *m = AzureDiskVolum func (*AzureDiskVolumeSource) ProtoMessage() {} func (*AzureDiskVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{4} } +func (m *AzureFilePersistentVolumeSource) Reset() { *m = AzureFilePersistentVolumeSource{} } +func (*AzureFilePersistentVolumeSource) ProtoMessage() {} +func (*AzureFilePersistentVolumeSource) Descriptor() ([]byte, []int) { + return fileDescriptorGenerated, []int{5} +} + func (m *AzureFileVolumeSource) Reset() { *m = AzureFileVolumeSource{} } func (*AzureFileVolumeSource) ProtoMessage() {} -func (*AzureFileVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{5} } +func (*AzureFileVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{6} } func (m *Binding) Reset() { *m = Binding{} } func (*Binding) ProtoMessage() {} -func (*Binding) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{6} } +func (*Binding) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{7} } func (m *Capabilities) Reset() { *m = Capabilities{} } func (*Capabilities) ProtoMessage() {} -func (*Capabilities) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{7} } +func (*Capabilities) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{8} } + +func (m *CephFSPersistentVolumeSource) Reset() { *m = CephFSPersistentVolumeSource{} } +func (*CephFSPersistentVolumeSource) ProtoMessage() {} +func (*CephFSPersistentVolumeSource) Descriptor() ([]byte, []int) { + return fileDescriptorGenerated, []int{9} +} func (m *CephFSVolumeSource) Reset() { *m = CephFSVolumeSource{} } func (*CephFSVolumeSource) ProtoMessage() {} -func (*CephFSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{8} } +func (*CephFSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{10} } func (m *CinderVolumeSource) Reset() { *m = CinderVolumeSource{} } func (*CinderVolumeSource) ProtoMessage() {} -func (*CinderVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{9} } +func (*CinderVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{11} } + +func (m *ClientIPConfig) Reset() { *m = ClientIPConfig{} } +func (*ClientIPConfig) ProtoMessage() {} +func (*ClientIPConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{12} } func (m *ComponentCondition) Reset() { *m = ComponentCondition{} } func (*ComponentCondition) ProtoMessage() {} -func (*ComponentCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{10} } +func (*ComponentCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{13} } func (m *ComponentStatus) Reset() { *m = ComponentStatus{} } func (*ComponentStatus) ProtoMessage() {} -func (*ComponentStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{11} } +func (*ComponentStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{14} } func (m *ComponentStatusList) Reset() { *m = ComponentStatusList{} } func (*ComponentStatusList) ProtoMessage() {} -func (*ComponentStatusList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{12} } +func (*ComponentStatusList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{15} } func (m *ConfigMap) Reset() { *m = ConfigMap{} } func (*ConfigMap) ProtoMessage() {} -func (*ConfigMap) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{13} } +func (*ConfigMap) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{16} } func (m *ConfigMapEnvSource) Reset() { *m = ConfigMapEnvSource{} } func (*ConfigMapEnvSource) ProtoMessage() {} -func (*ConfigMapEnvSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{14} } +func (*ConfigMapEnvSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{17} } func (m *ConfigMapKeySelector) Reset() { *m = ConfigMapKeySelector{} } func (*ConfigMapKeySelector) ProtoMessage() {} -func (*ConfigMapKeySelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{15} } +func (*ConfigMapKeySelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{18} } func (m *ConfigMapList) Reset() { *m = ConfigMapList{} } func (*ConfigMapList) ProtoMessage() {} -func (*ConfigMapList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{16} } +func (*ConfigMapList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{19} } func (m *ConfigMapProjection) Reset() { *m = ConfigMapProjection{} } func (*ConfigMapProjection) ProtoMessage() {} -func (*ConfigMapProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{17} } +func (*ConfigMapProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{20} } func (m *ConfigMapVolumeSource) Reset() { *m = ConfigMapVolumeSource{} } func (*ConfigMapVolumeSource) ProtoMessage() {} -func (*ConfigMapVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{18} } +func (*ConfigMapVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{21} } func (m *Container) Reset() { *m = Container{} } func (*Container) ProtoMessage() {} -func (*Container) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{19} } +func (*Container) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} } func (m *ContainerImage) Reset() { *m = ContainerImage{} } func (*ContainerImage) ProtoMessage() {} -func (*ContainerImage) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{20} } +func (*ContainerImage) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} } func (m *ContainerPort) Reset() { *m = ContainerPort{} } func (*ContainerPort) ProtoMessage() {} -func (*ContainerPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{21} } +func (*ContainerPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{24} } func (m *ContainerState) Reset() { *m = ContainerState{} } func (*ContainerState) ProtoMessage() {} -func (*ContainerState) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} } +func (*ContainerState) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} } func (m *ContainerStateRunning) Reset() { *m = ContainerStateRunning{} } func (*ContainerStateRunning) ProtoMessage() {} -func (*ContainerStateRunning) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} } +func (*ContainerStateRunning) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} } func (m *ContainerStateTerminated) Reset() { *m = ContainerStateTerminated{} } func (*ContainerStateTerminated) ProtoMessage() {} func (*ContainerStateTerminated) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{24} + return fileDescriptorGenerated, []int{27} } func (m *ContainerStateWaiting) Reset() { *m = ContainerStateWaiting{} } func (*ContainerStateWaiting) ProtoMessage() {} -func (*ContainerStateWaiting) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} } +func (*ContainerStateWaiting) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{28} } func (m *ContainerStatus) Reset() { *m = ContainerStatus{} } func (*ContainerStatus) ProtoMessage() {} -func (*ContainerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} } +func (*ContainerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{29} } func (m *DaemonEndpoint) Reset() { *m = DaemonEndpoint{} } func (*DaemonEndpoint) ProtoMessage() {} -func (*DaemonEndpoint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{27} } +func (*DaemonEndpoint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} } func (m *DeleteOptions) Reset() { *m = DeleteOptions{} } func (*DeleteOptions) ProtoMessage() {} -func (*DeleteOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{28} } +func (*DeleteOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{31} } func (m *DownwardAPIProjection) Reset() { *m = DownwardAPIProjection{} } func (*DownwardAPIProjection) ProtoMessage() {} -func (*DownwardAPIProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{29} } +func (*DownwardAPIProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{32} } func (m *DownwardAPIVolumeFile) Reset() { *m = DownwardAPIVolumeFile{} } func (*DownwardAPIVolumeFile) ProtoMessage() {} -func (*DownwardAPIVolumeFile) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} } +func (*DownwardAPIVolumeFile) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{33} } func (m *DownwardAPIVolumeSource) Reset() { *m = DownwardAPIVolumeSource{} } func (*DownwardAPIVolumeSource) ProtoMessage() {} func (*DownwardAPIVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{31} + return fileDescriptorGenerated, []int{34} } func (m *EmptyDirVolumeSource) Reset() { *m = EmptyDirVolumeSource{} } func (*EmptyDirVolumeSource) ProtoMessage() {} -func (*EmptyDirVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{32} } +func (*EmptyDirVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{35} } func (m *EndpointAddress) Reset() { *m = EndpointAddress{} } func (*EndpointAddress) ProtoMessage() {} -func (*EndpointAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{33} } +func (*EndpointAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} } func (m *EndpointPort) Reset() { *m = EndpointPort{} } func (*EndpointPort) ProtoMessage() {} -func (*EndpointPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{34} } +func (*EndpointPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} } func (m *EndpointSubset) Reset() { *m = EndpointSubset{} } func (*EndpointSubset) ProtoMessage() {} -func (*EndpointSubset) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{35} } +func (*EndpointSubset) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} } func (m *Endpoints) Reset() { *m = Endpoints{} } func (*Endpoints) ProtoMessage() {} -func (*Endpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} } +func (*Endpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} } func (m *EndpointsList) Reset() { *m = EndpointsList{} } func (*EndpointsList) ProtoMessage() {} -func (*EndpointsList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} } +func (*EndpointsList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} } func (m *EnvFromSource) Reset() { *m = EnvFromSource{} } func (*EnvFromSource) ProtoMessage() {} -func (*EnvFromSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} } +func (*EnvFromSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} } func (m *EnvVar) Reset() { *m = EnvVar{} } func (*EnvVar) ProtoMessage() {} -func (*EnvVar) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} } +func (*EnvVar) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} } func (m *EnvVarSource) Reset() { *m = EnvVarSource{} } func (*EnvVarSource) ProtoMessage() {} -func (*EnvVarSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} } +func (*EnvVarSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} } func (m *Event) Reset() { *m = Event{} } func (*Event) ProtoMessage() {} -func (*Event) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} } +func (*Event) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{44} } func (m *EventList) Reset() { *m = EventList{} } func (*EventList) ProtoMessage() {} -func (*EventList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} } +func (*EventList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{45} } func (m *EventSource) Reset() { *m = EventSource{} } func (*EventSource) ProtoMessage() {} -func (*EventSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} } +func (*EventSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} } func (m *ExecAction) Reset() { *m = ExecAction{} } func (*ExecAction) ProtoMessage() {} -func (*ExecAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{44} } +func (*ExecAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{47} } func (m *FCVolumeSource) Reset() { *m = FCVolumeSource{} } func (*FCVolumeSource) ProtoMessage() {} -func (*FCVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{45} } +func (*FCVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{48} } func (m *FlexVolumeSource) Reset() { *m = FlexVolumeSource{} } func (*FlexVolumeSource) ProtoMessage() {} -func (*FlexVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} } +func (*FlexVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{49} } func (m *FlockerVolumeSource) Reset() { *m = FlockerVolumeSource{} } func (*FlockerVolumeSource) ProtoMessage() {} -func (*FlockerVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{47} } +func (*FlockerVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{50} } func (m *GCEPersistentDiskVolumeSource) Reset() { *m = GCEPersistentDiskVolumeSource{} } func (*GCEPersistentDiskVolumeSource) ProtoMessage() {} func (*GCEPersistentDiskVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{48} + return fileDescriptorGenerated, []int{51} } func (m *GitRepoVolumeSource) Reset() { *m = GitRepoVolumeSource{} } func (*GitRepoVolumeSource) ProtoMessage() {} -func (*GitRepoVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{49} } +func (*GitRepoVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{52} } func (m *GlusterfsVolumeSource) Reset() { *m = GlusterfsVolumeSource{} } func (*GlusterfsVolumeSource) ProtoMessage() {} -func (*GlusterfsVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{50} } +func (*GlusterfsVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{53} } func (m *HTTPGetAction) Reset() { *m = HTTPGetAction{} } func (*HTTPGetAction) ProtoMessage() {} -func (*HTTPGetAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{51} } +func (*HTTPGetAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{54} } func (m *HTTPHeader) Reset() { *m = HTTPHeader{} } func (*HTTPHeader) ProtoMessage() {} -func (*HTTPHeader) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{52} } +func (*HTTPHeader) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} } func (m *Handler) Reset() { *m = Handler{} } func (*Handler) ProtoMessage() {} -func (*Handler) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{53} } +func (*Handler) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{56} } func (m *HostAlias) Reset() { *m = HostAlias{} } func (*HostAlias) ProtoMessage() {} -func (*HostAlias) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{54} } +func (*HostAlias) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{57} } func (m *HostPathVolumeSource) Reset() { *m = HostPathVolumeSource{} } func (*HostPathVolumeSource) ProtoMessage() {} -func (*HostPathVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} } +func (*HostPathVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{58} } func (m *ISCSIVolumeSource) Reset() { *m = ISCSIVolumeSource{} } func (*ISCSIVolumeSource) ProtoMessage() {} -func (*ISCSIVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{56} } +func (*ISCSIVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{59} } func (m *KeyToPath) Reset() { *m = KeyToPath{} } func (*KeyToPath) ProtoMessage() {} -func (*KeyToPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{57} } +func (*KeyToPath) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{60} } func (m *Lifecycle) Reset() { *m = Lifecycle{} } func (*Lifecycle) ProtoMessage() {} -func (*Lifecycle) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{58} } +func (*Lifecycle) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{61} } func (m *LimitRange) Reset() { *m = LimitRange{} } func (*LimitRange) ProtoMessage() {} -func (*LimitRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{59} } +func (*LimitRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{62} } func (m *LimitRangeItem) Reset() { *m = LimitRangeItem{} } func (*LimitRangeItem) ProtoMessage() {} -func (*LimitRangeItem) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{60} } +func (*LimitRangeItem) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{63} } func (m *LimitRangeList) Reset() { *m = LimitRangeList{} } func (*LimitRangeList) ProtoMessage() {} -func (*LimitRangeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{61} } +func (*LimitRangeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{64} } func (m *LimitRangeSpec) Reset() { *m = LimitRangeSpec{} } func (*LimitRangeSpec) ProtoMessage() {} -func (*LimitRangeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{62} } +func (*LimitRangeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{65} } func (m *List) Reset() { *m = List{} } func (*List) ProtoMessage() {} -func (*List) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{63} } +func (*List) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{66} } func (m *ListOptions) Reset() { *m = ListOptions{} } func (*ListOptions) ProtoMessage() {} -func (*ListOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{64} } +func (*ListOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{67} } func (m *LoadBalancerIngress) Reset() { *m = LoadBalancerIngress{} } func (*LoadBalancerIngress) ProtoMessage() {} -func (*LoadBalancerIngress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{65} } +func (*LoadBalancerIngress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{68} } func (m *LoadBalancerStatus) Reset() { *m = LoadBalancerStatus{} } func (*LoadBalancerStatus) ProtoMessage() {} -func (*LoadBalancerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{66} } +func (*LoadBalancerStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{69} } func (m *LocalObjectReference) Reset() { *m = LocalObjectReference{} } func (*LocalObjectReference) ProtoMessage() {} -func (*LocalObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{67} } +func (*LocalObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{70} } func (m *LocalVolumeSource) Reset() { *m = LocalVolumeSource{} } func (*LocalVolumeSource) ProtoMessage() {} -func (*LocalVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{68} } +func (*LocalVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{71} } func (m *NFSVolumeSource) Reset() { *m = NFSVolumeSource{} } func (*NFSVolumeSource) ProtoMessage() {} -func (*NFSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{69} } +func (*NFSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{72} } func (m *Namespace) Reset() { *m = Namespace{} } func (*Namespace) ProtoMessage() {} -func (*Namespace) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{70} } +func (*Namespace) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{73} } func (m *NamespaceList) Reset() { *m = NamespaceList{} } func (*NamespaceList) ProtoMessage() {} -func (*NamespaceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{71} } +func (*NamespaceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{74} } func (m *NamespaceSpec) Reset() { *m = NamespaceSpec{} } func (*NamespaceSpec) ProtoMessage() {} -func (*NamespaceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{72} } +func (*NamespaceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{75} } func (m *NamespaceStatus) Reset() { *m = NamespaceStatus{} } func (*NamespaceStatus) ProtoMessage() {} -func (*NamespaceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{73} } +func (*NamespaceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{76} } func (m *Node) Reset() { *m = Node{} } func (*Node) ProtoMessage() {} -func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{74} } +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{77} } func (m *NodeAddress) Reset() { *m = NodeAddress{} } func (*NodeAddress) ProtoMessage() {} -func (*NodeAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{75} } +func (*NodeAddress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{78} } func (m *NodeAffinity) Reset() { *m = NodeAffinity{} } func (*NodeAffinity) ProtoMessage() {} -func (*NodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{76} } +func (*NodeAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{79} } func (m *NodeCondition) Reset() { *m = NodeCondition{} } func (*NodeCondition) ProtoMessage() {} -func (*NodeCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{77} } +func (*NodeCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{80} } func (m *NodeConfigSource) Reset() { *m = NodeConfigSource{} } func (*NodeConfigSource) ProtoMessage() {} -func (*NodeConfigSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{78} } +func (*NodeConfigSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{81} } func (m *NodeDaemonEndpoints) Reset() { *m = NodeDaemonEndpoints{} } func (*NodeDaemonEndpoints) ProtoMessage() {} -func (*NodeDaemonEndpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{79} } +func (*NodeDaemonEndpoints) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{82} } func (m *NodeList) Reset() { *m = NodeList{} } func (*NodeList) ProtoMessage() {} -func (*NodeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{80} } +func (*NodeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{83} } func (m *NodeProxyOptions) Reset() { *m = NodeProxyOptions{} } func (*NodeProxyOptions) ProtoMessage() {} -func (*NodeProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{81} } +func (*NodeProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{84} } func (m *NodeResources) Reset() { *m = NodeResources{} } func (*NodeResources) ProtoMessage() {} -func (*NodeResources) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{82} } +func (*NodeResources) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{85} } func (m *NodeSelector) Reset() { *m = NodeSelector{} } func (*NodeSelector) ProtoMessage() {} -func (*NodeSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{83} } +func (*NodeSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{86} } func (m *NodeSelectorRequirement) Reset() { *m = NodeSelectorRequirement{} } func (*NodeSelectorRequirement) ProtoMessage() {} func (*NodeSelectorRequirement) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{84} + return fileDescriptorGenerated, []int{87} } func (m *NodeSelectorTerm) Reset() { *m = NodeSelectorTerm{} } func (*NodeSelectorTerm) ProtoMessage() {} -func (*NodeSelectorTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{85} } +func (*NodeSelectorTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{88} } func (m *NodeSpec) Reset() { *m = NodeSpec{} } func (*NodeSpec) ProtoMessage() {} -func (*NodeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{86} } +func (*NodeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{89} } func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{87} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{90} } func (m *NodeSystemInfo) Reset() { *m = NodeSystemInfo{} } func (*NodeSystemInfo) ProtoMessage() {} -func (*NodeSystemInfo) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{88} } +func (*NodeSystemInfo) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{91} } func (m *ObjectFieldSelector) Reset() { *m = ObjectFieldSelector{} } func (*ObjectFieldSelector) ProtoMessage() {} -func (*ObjectFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{89} } +func (*ObjectFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{92} } func (m *ObjectMeta) Reset() { *m = ObjectMeta{} } func (*ObjectMeta) ProtoMessage() {} -func (*ObjectMeta) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{90} } +func (*ObjectMeta) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{93} } func (m *ObjectReference) Reset() { *m = ObjectReference{} } func (*ObjectReference) ProtoMessage() {} -func (*ObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{91} } +func (*ObjectReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{94} } func (m *PersistentVolume) Reset() { *m = PersistentVolume{} } func (*PersistentVolume) ProtoMessage() {} -func (*PersistentVolume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{92} } +func (*PersistentVolume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{95} } func (m *PersistentVolumeClaim) Reset() { *m = PersistentVolumeClaim{} } func (*PersistentVolumeClaim) ProtoMessage() {} -func (*PersistentVolumeClaim) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{93} } +func (*PersistentVolumeClaim) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{96} } func (m *PersistentVolumeClaimList) Reset() { *m = PersistentVolumeClaimList{} } func (*PersistentVolumeClaimList) ProtoMessage() {} func (*PersistentVolumeClaimList) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{94} + return fileDescriptorGenerated, []int{97} } func (m *PersistentVolumeClaimSpec) Reset() { *m = PersistentVolumeClaimSpec{} } func (*PersistentVolumeClaimSpec) ProtoMessage() {} func (*PersistentVolumeClaimSpec) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{95} + return fileDescriptorGenerated, []int{98} } func (m *PersistentVolumeClaimStatus) Reset() { *m = PersistentVolumeClaimStatus{} } func (*PersistentVolumeClaimStatus) ProtoMessage() {} func (*PersistentVolumeClaimStatus) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{96} + return fileDescriptorGenerated, []int{99} } func (m *PersistentVolumeClaimVolumeSource) Reset() { *m = PersistentVolumeClaimVolumeSource{} } func (*PersistentVolumeClaimVolumeSource) ProtoMessage() {} func (*PersistentVolumeClaimVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{97} + return fileDescriptorGenerated, []int{100} } func (m *PersistentVolumeList) Reset() { *m = PersistentVolumeList{} } func (*PersistentVolumeList) ProtoMessage() {} -func (*PersistentVolumeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{98} } +func (*PersistentVolumeList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{101} } -func (m *PersistentVolumeSource) Reset() { *m = PersistentVolumeSource{} } -func (*PersistentVolumeSource) ProtoMessage() {} -func (*PersistentVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{99} } +func (m *PersistentVolumeSource) Reset() { *m = PersistentVolumeSource{} } +func (*PersistentVolumeSource) ProtoMessage() {} +func (*PersistentVolumeSource) Descriptor() ([]byte, []int) { + return fileDescriptorGenerated, []int{102} +} func (m *PersistentVolumeSpec) Reset() { *m = PersistentVolumeSpec{} } func (*PersistentVolumeSpec) ProtoMessage() {} -func (*PersistentVolumeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{100} } +func (*PersistentVolumeSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{103} } func (m *PersistentVolumeStatus) Reset() { *m = PersistentVolumeStatus{} } func (*PersistentVolumeStatus) ProtoMessage() {} func (*PersistentVolumeStatus) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{101} + return fileDescriptorGenerated, []int{104} } func (m *PhotonPersistentDiskVolumeSource) Reset() { *m = PhotonPersistentDiskVolumeSource{} } func (*PhotonPersistentDiskVolumeSource) ProtoMessage() {} func (*PhotonPersistentDiskVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{102} + return fileDescriptorGenerated, []int{105} } func (m *Pod) Reset() { *m = Pod{} } func (*Pod) ProtoMessage() {} -func (*Pod) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{103} } +func (*Pod) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{106} } func (m *PodAffinity) Reset() { *m = PodAffinity{} } func (*PodAffinity) ProtoMessage() {} -func (*PodAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{104} } +func (*PodAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{107} } func (m *PodAffinityTerm) Reset() { *m = PodAffinityTerm{} } func (*PodAffinityTerm) ProtoMessage() {} -func (*PodAffinityTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{105} } +func (*PodAffinityTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{108} } func (m *PodAntiAffinity) Reset() { *m = PodAntiAffinity{} } func (*PodAntiAffinity) ProtoMessage() {} -func (*PodAntiAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{106} } +func (*PodAntiAffinity) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{109} } func (m *PodAttachOptions) Reset() { *m = PodAttachOptions{} } func (*PodAttachOptions) ProtoMessage() {} -func (*PodAttachOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{107} } +func (*PodAttachOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{110} } func (m *PodCondition) Reset() { *m = PodCondition{} } func (*PodCondition) ProtoMessage() {} -func (*PodCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{108} } +func (*PodCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{111} } func (m *PodExecOptions) Reset() { *m = PodExecOptions{} } func (*PodExecOptions) ProtoMessage() {} -func (*PodExecOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{109} } +func (*PodExecOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{112} } func (m *PodList) Reset() { *m = PodList{} } func (*PodList) ProtoMessage() {} -func (*PodList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{110} } +func (*PodList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{113} } func (m *PodLogOptions) Reset() { *m = PodLogOptions{} } func (*PodLogOptions) ProtoMessage() {} -func (*PodLogOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{111} } +func (*PodLogOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{114} } func (m *PodPortForwardOptions) Reset() { *m = PodPortForwardOptions{} } func (*PodPortForwardOptions) ProtoMessage() {} -func (*PodPortForwardOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{112} } +func (*PodPortForwardOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{115} } func (m *PodProxyOptions) Reset() { *m = PodProxyOptions{} } func (*PodProxyOptions) ProtoMessage() {} -func (*PodProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{113} } +func (*PodProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{116} } func (m *PodSecurityContext) Reset() { *m = PodSecurityContext{} } func (*PodSecurityContext) ProtoMessage() {} -func (*PodSecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{114} } +func (*PodSecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{117} } func (m *PodSignature) Reset() { *m = PodSignature{} } func (*PodSignature) ProtoMessage() {} -func (*PodSignature) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{115} } +func (*PodSignature) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{118} } func (m *PodSpec) Reset() { *m = PodSpec{} } func (*PodSpec) ProtoMessage() {} -func (*PodSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{116} } +func (*PodSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{119} } func (m *PodStatus) Reset() { *m = PodStatus{} } func (*PodStatus) ProtoMessage() {} -func (*PodStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{117} } +func (*PodStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{120} } func (m *PodStatusResult) Reset() { *m = PodStatusResult{} } func (*PodStatusResult) ProtoMessage() {} -func (*PodStatusResult) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{118} } +func (*PodStatusResult) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{121} } func (m *PodTemplate) Reset() { *m = PodTemplate{} } func (*PodTemplate) ProtoMessage() {} -func (*PodTemplate) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{119} } +func (*PodTemplate) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{122} } func (m *PodTemplateList) Reset() { *m = PodTemplateList{} } func (*PodTemplateList) ProtoMessage() {} -func (*PodTemplateList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{120} } +func (*PodTemplateList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{123} } func (m *PodTemplateSpec) Reset() { *m = PodTemplateSpec{} } func (*PodTemplateSpec) ProtoMessage() {} -func (*PodTemplateSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{121} } +func (*PodTemplateSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{124} } func (m *PortworxVolumeSource) Reset() { *m = PortworxVolumeSource{} } func (*PortworxVolumeSource) ProtoMessage() {} -func (*PortworxVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{122} } +func (*PortworxVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{125} } func (m *Preconditions) Reset() { *m = Preconditions{} } func (*Preconditions) ProtoMessage() {} -func (*Preconditions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{123} } +func (*Preconditions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{126} } func (m *PreferAvoidPodsEntry) Reset() { *m = PreferAvoidPodsEntry{} } func (*PreferAvoidPodsEntry) ProtoMessage() {} -func (*PreferAvoidPodsEntry) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{124} } +func (*PreferAvoidPodsEntry) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{127} } func (m *PreferredSchedulingTerm) Reset() { *m = PreferredSchedulingTerm{} } func (*PreferredSchedulingTerm) ProtoMessage() {} func (*PreferredSchedulingTerm) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{125} + return fileDescriptorGenerated, []int{128} } func (m *Probe) Reset() { *m = Probe{} } func (*Probe) ProtoMessage() {} -func (*Probe) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{126} } +func (*Probe) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{129} } func (m *ProjectedVolumeSource) Reset() { *m = ProjectedVolumeSource{} } func (*ProjectedVolumeSource) ProtoMessage() {} -func (*ProjectedVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{127} } +func (*ProjectedVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{130} } func (m *QuobyteVolumeSource) Reset() { *m = QuobyteVolumeSource{} } func (*QuobyteVolumeSource) ProtoMessage() {} -func (*QuobyteVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{128} } +func (*QuobyteVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{131} } func (m *RBDVolumeSource) Reset() { *m = RBDVolumeSource{} } func (*RBDVolumeSource) ProtoMessage() {} -func (*RBDVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{129} } +func (*RBDVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{132} } func (m *RangeAllocation) Reset() { *m = RangeAllocation{} } func (*RangeAllocation) ProtoMessage() {} -func (*RangeAllocation) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{130} } +func (*RangeAllocation) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{133} } func (m *ReplicationController) Reset() { *m = ReplicationController{} } func (*ReplicationController) ProtoMessage() {} -func (*ReplicationController) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{131} } +func (*ReplicationController) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{134} } func (m *ReplicationControllerCondition) Reset() { *m = ReplicationControllerCondition{} } func (*ReplicationControllerCondition) ProtoMessage() {} func (*ReplicationControllerCondition) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{132} + return fileDescriptorGenerated, []int{135} } func (m *ReplicationControllerList) Reset() { *m = ReplicationControllerList{} } func (*ReplicationControllerList) ProtoMessage() {} func (*ReplicationControllerList) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{133} + return fileDescriptorGenerated, []int{136} } func (m *ReplicationControllerSpec) Reset() { *m = ReplicationControllerSpec{} } func (*ReplicationControllerSpec) ProtoMessage() {} func (*ReplicationControllerSpec) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{134} + return fileDescriptorGenerated, []int{137} } func (m *ReplicationControllerStatus) Reset() { *m = ReplicationControllerStatus{} } func (*ReplicationControllerStatus) ProtoMessage() {} func (*ReplicationControllerStatus) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{135} + return fileDescriptorGenerated, []int{138} } func (m *ResourceFieldSelector) Reset() { *m = ResourceFieldSelector{} } func (*ResourceFieldSelector) ProtoMessage() {} -func (*ResourceFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{136} } +func (*ResourceFieldSelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{139} } func (m *ResourceQuota) Reset() { *m = ResourceQuota{} } func (*ResourceQuota) ProtoMessage() {} -func (*ResourceQuota) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{137} } +func (*ResourceQuota) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{140} } func (m *ResourceQuotaList) Reset() { *m = ResourceQuotaList{} } func (*ResourceQuotaList) ProtoMessage() {} -func (*ResourceQuotaList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{138} } +func (*ResourceQuotaList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{141} } func (m *ResourceQuotaSpec) Reset() { *m = ResourceQuotaSpec{} } func (*ResourceQuotaSpec) ProtoMessage() {} -func (*ResourceQuotaSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{139} } +func (*ResourceQuotaSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{142} } func (m *ResourceQuotaStatus) Reset() { *m = ResourceQuotaStatus{} } func (*ResourceQuotaStatus) ProtoMessage() {} -func (*ResourceQuotaStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{140} } +func (*ResourceQuotaStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{143} } func (m *ResourceRequirements) Reset() { *m = ResourceRequirements{} } func (*ResourceRequirements) ProtoMessage() {} -func (*ResourceRequirements) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{141} } +func (*ResourceRequirements) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{144} } func (m *SELinuxOptions) Reset() { *m = SELinuxOptions{} } func (*SELinuxOptions) ProtoMessage() {} -func (*SELinuxOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{142} } +func (*SELinuxOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{145} } func (m *ScaleIOVolumeSource) Reset() { *m = ScaleIOVolumeSource{} } func (*ScaleIOVolumeSource) ProtoMessage() {} -func (*ScaleIOVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{143} } +func (*ScaleIOVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{146} } func (m *Secret) Reset() { *m = Secret{} } func (*Secret) ProtoMessage() {} -func (*Secret) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{144} } +func (*Secret) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{147} } func (m *SecretEnvSource) Reset() { *m = SecretEnvSource{} } func (*SecretEnvSource) ProtoMessage() {} -func (*SecretEnvSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{145} } +func (*SecretEnvSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{148} } func (m *SecretKeySelector) Reset() { *m = SecretKeySelector{} } func (*SecretKeySelector) ProtoMessage() {} -func (*SecretKeySelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{146} } +func (*SecretKeySelector) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{149} } func (m *SecretList) Reset() { *m = SecretList{} } func (*SecretList) ProtoMessage() {} -func (*SecretList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{147} } +func (*SecretList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{150} } func (m *SecretProjection) Reset() { *m = SecretProjection{} } func (*SecretProjection) ProtoMessage() {} -func (*SecretProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{148} } +func (*SecretProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{151} } + +func (m *SecretReference) Reset() { *m = SecretReference{} } +func (*SecretReference) ProtoMessage() {} +func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{152} } func (m *SecretVolumeSource) Reset() { *m = SecretVolumeSource{} } func (*SecretVolumeSource) ProtoMessage() {} -func (*SecretVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{149} } +func (*SecretVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{153} } func (m *SecurityContext) Reset() { *m = SecurityContext{} } func (*SecurityContext) ProtoMessage() {} -func (*SecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{150} } +func (*SecurityContext) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{154} } func (m *SerializedReference) Reset() { *m = SerializedReference{} } func (*SerializedReference) ProtoMessage() {} -func (*SerializedReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{151} } +func (*SerializedReference) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{155} } func (m *Service) Reset() { *m = Service{} } func (*Service) ProtoMessage() {} -func (*Service) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{152} } +func (*Service) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{156} } func (m *ServiceAccount) Reset() { *m = ServiceAccount{} } func (*ServiceAccount) ProtoMessage() {} -func (*ServiceAccount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{153} } +func (*ServiceAccount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{157} } func (m *ServiceAccountList) Reset() { *m = ServiceAccountList{} } func (*ServiceAccountList) ProtoMessage() {} -func (*ServiceAccountList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{154} } +func (*ServiceAccountList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{158} } func (m *ServiceList) Reset() { *m = ServiceList{} } func (*ServiceList) ProtoMessage() {} -func (*ServiceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{155} } +func (*ServiceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{159} } func (m *ServicePort) Reset() { *m = ServicePort{} } func (*ServicePort) ProtoMessage() {} -func (*ServicePort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{156} } +func (*ServicePort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{160} } func (m *ServiceProxyOptions) Reset() { *m = ServiceProxyOptions{} } func (*ServiceProxyOptions) ProtoMessage() {} -func (*ServiceProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{157} } +func (*ServiceProxyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{161} } func (m *ServiceSpec) Reset() { *m = ServiceSpec{} } func (*ServiceSpec) ProtoMessage() {} -func (*ServiceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{158} } +func (*ServiceSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{162} } func (m *ServiceStatus) Reset() { *m = ServiceStatus{} } func (*ServiceStatus) ProtoMessage() {} -func (*ServiceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{159} } +func (*ServiceStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{163} } + +func (m *SessionAffinityConfig) Reset() { *m = SessionAffinityConfig{} } +func (*SessionAffinityConfig) ProtoMessage() {} +func (*SessionAffinityConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{164} } func (m *StorageOSPersistentVolumeSource) Reset() { *m = StorageOSPersistentVolumeSource{} } func (*StorageOSPersistentVolumeSource) ProtoMessage() {} func (*StorageOSPersistentVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{160} + return fileDescriptorGenerated, []int{165} } func (m *StorageOSVolumeSource) Reset() { *m = StorageOSVolumeSource{} } func (*StorageOSVolumeSource) ProtoMessage() {} -func (*StorageOSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{161} } +func (*StorageOSVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{166} } func (m *Sysctl) Reset() { *m = Sysctl{} } func (*Sysctl) ProtoMessage() {} -func (*Sysctl) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{162} } +func (*Sysctl) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{167} } func (m *TCPSocketAction) Reset() { *m = TCPSocketAction{} } func (*TCPSocketAction) ProtoMessage() {} -func (*TCPSocketAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{163} } +func (*TCPSocketAction) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{168} } func (m *Taint) Reset() { *m = Taint{} } func (*Taint) ProtoMessage() {} -func (*Taint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{164} } +func (*Taint) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{169} } func (m *Toleration) Reset() { *m = Toleration{} } func (*Toleration) ProtoMessage() {} -func (*Toleration) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{165} } +func (*Toleration) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{170} } func (m *Volume) Reset() { *m = Volume{} } func (*Volume) ProtoMessage() {} -func (*Volume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{166} } +func (*Volume) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{171} } func (m *VolumeMount) Reset() { *m = VolumeMount{} } func (*VolumeMount) ProtoMessage() {} -func (*VolumeMount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{167} } +func (*VolumeMount) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{172} } func (m *VolumeProjection) Reset() { *m = VolumeProjection{} } func (*VolumeProjection) ProtoMessage() {} -func (*VolumeProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{168} } +func (*VolumeProjection) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{173} } func (m *VolumeSource) Reset() { *m = VolumeSource{} } func (*VolumeSource) ProtoMessage() {} -func (*VolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{169} } +func (*VolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{174} } func (m *VsphereVirtualDiskVolumeSource) Reset() { *m = VsphereVirtualDiskVolumeSource{} } func (*VsphereVirtualDiskVolumeSource) ProtoMessage() {} func (*VsphereVirtualDiskVolumeSource) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{170} + return fileDescriptorGenerated, []int{175} } func (m *WeightedPodAffinityTerm) Reset() { *m = WeightedPodAffinityTerm{} } func (*WeightedPodAffinityTerm) ProtoMessage() {} func (*WeightedPodAffinityTerm) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{171} + return fileDescriptorGenerated, []int{176} } func init() { @@ -960,11 +991,14 @@ func init() { proto.RegisterType((*AttachedVolume)(nil), "k8s.io.api.core.v1.AttachedVolume") proto.RegisterType((*AvoidPods)(nil), "k8s.io.api.core.v1.AvoidPods") proto.RegisterType((*AzureDiskVolumeSource)(nil), "k8s.io.api.core.v1.AzureDiskVolumeSource") + proto.RegisterType((*AzureFilePersistentVolumeSource)(nil), "k8s.io.api.core.v1.AzureFilePersistentVolumeSource") proto.RegisterType((*AzureFileVolumeSource)(nil), "k8s.io.api.core.v1.AzureFileVolumeSource") proto.RegisterType((*Binding)(nil), "k8s.io.api.core.v1.Binding") proto.RegisterType((*Capabilities)(nil), "k8s.io.api.core.v1.Capabilities") + proto.RegisterType((*CephFSPersistentVolumeSource)(nil), "k8s.io.api.core.v1.CephFSPersistentVolumeSource") proto.RegisterType((*CephFSVolumeSource)(nil), "k8s.io.api.core.v1.CephFSVolumeSource") proto.RegisterType((*CinderVolumeSource)(nil), "k8s.io.api.core.v1.CinderVolumeSource") + proto.RegisterType((*ClientIPConfig)(nil), "k8s.io.api.core.v1.ClientIPConfig") proto.RegisterType((*ComponentCondition)(nil), "k8s.io.api.core.v1.ComponentCondition") proto.RegisterType((*ComponentStatus)(nil), "k8s.io.api.core.v1.ComponentStatus") proto.RegisterType((*ComponentStatusList)(nil), "k8s.io.api.core.v1.ComponentStatusList") @@ -1104,6 +1138,7 @@ func init() { proto.RegisterType((*SecretKeySelector)(nil), "k8s.io.api.core.v1.SecretKeySelector") proto.RegisterType((*SecretList)(nil), "k8s.io.api.core.v1.SecretList") proto.RegisterType((*SecretProjection)(nil), "k8s.io.api.core.v1.SecretProjection") + proto.RegisterType((*SecretReference)(nil), "k8s.io.api.core.v1.SecretReference") proto.RegisterType((*SecretVolumeSource)(nil), "k8s.io.api.core.v1.SecretVolumeSource") proto.RegisterType((*SecurityContext)(nil), "k8s.io.api.core.v1.SecurityContext") proto.RegisterType((*SerializedReference)(nil), "k8s.io.api.core.v1.SerializedReference") @@ -1115,6 +1150,7 @@ func init() { proto.RegisterType((*ServiceProxyOptions)(nil), "k8s.io.api.core.v1.ServiceProxyOptions") proto.RegisterType((*ServiceSpec)(nil), "k8s.io.api.core.v1.ServiceSpec") proto.RegisterType((*ServiceStatus)(nil), "k8s.io.api.core.v1.ServiceStatus") + proto.RegisterType((*SessionAffinityConfig)(nil), "k8s.io.api.core.v1.SessionAffinityConfig") proto.RegisterType((*StorageOSPersistentVolumeSource)(nil), "k8s.io.api.core.v1.StorageOSPersistentVolumeSource") proto.RegisterType((*StorageOSVolumeSource)(nil), "k8s.io.api.core.v1.StorageOSVolumeSource") proto.RegisterType((*Sysctl)(nil), "k8s.io.api.core.v1.Sysctl") @@ -1323,6 +1359,46 @@ func (m *AzureDiskVolumeSource) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *AzureFilePersistentVolumeSource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AzureFilePersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.SecretName))) + i += copy(dAtA[i:], m.SecretName) + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.ShareName))) + i += copy(dAtA[i:], m.ShareName) + dAtA[i] = 0x18 + i++ + if m.ReadOnly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + if m.SecretNamespace != nil { + dAtA[i] = 0x22 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.SecretNamespace))) + i += copy(dAtA[i:], *m.SecretNamespace) + } + return i, nil +} + func (m *AzureFileVolumeSource) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1439,7 +1515,7 @@ func (m *Capabilities) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *CephFSVolumeSource) Marshal() (dAtA []byte, err error) { +func (m *CephFSPersistentVolumeSource) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1449,7 +1525,7 @@ func (m *CephFSVolumeSource) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *CephFSVolumeSource) MarshalTo(dAtA []byte) (int, error) { +func (m *CephFSPersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1502,6 +1578,69 @@ func (m *CephFSVolumeSource) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *CephFSVolumeSource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CephFSVolumeSource) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Monitors) > 0 { + for _, s := range m.Monitors { + dAtA[i] = 0xa + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Path))) + i += copy(dAtA[i:], m.Path) + dAtA[i] = 0x1a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.User))) + i += copy(dAtA[i:], m.User) + dAtA[i] = 0x22 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.SecretFile))) + i += copy(dAtA[i:], m.SecretFile) + if m.SecretRef != nil { + dAtA[i] = 0x2a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.SecretRef.Size())) + n7, err := m.SecretRef.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n7 + } + dAtA[i] = 0x30 + i++ + if m.ReadOnly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + return i, nil +} + func (m *CinderVolumeSource) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1536,6 +1675,29 @@ func (m *CinderVolumeSource) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *ClientIPConfig) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ClientIPConfig) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.TimeoutSeconds != nil { + dAtA[i] = 0x8 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(*m.TimeoutSeconds)) + } + return i, nil +} + func (m *ComponentCondition) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1588,11 +1750,11 @@ func (m *ComponentStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n7, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n8, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n7 + i += n8 if len(m.Conditions) > 0 { for _, msg := range m.Conditions { dAtA[i] = 0x12 @@ -1626,11 +1788,11 @@ func (m *ComponentStatusList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n8, err := m.ListMeta.MarshalTo(dAtA[i:]) + n9, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n8 + i += n9 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -1664,11 +1826,11 @@ func (m *ConfigMap) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n9, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n10, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n10 if len(m.Data) > 0 { keysForData := make([]string, 0, len(m.Data)) for k := range m.Data { @@ -1712,11 +1874,11 @@ func (m *ConfigMapEnvSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LocalObjectReference.Size())) - n10, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) + n11, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n11 if m.Optional != nil { dAtA[i] = 0x10 i++ @@ -1748,11 +1910,11 @@ func (m *ConfigMapKeySelector) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LocalObjectReference.Size())) - n11, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) + n12, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n11 + i += n12 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Key))) @@ -1788,11 +1950,11 @@ func (m *ConfigMapList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n12, err := m.ListMeta.MarshalTo(dAtA[i:]) + n13, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n12 + i += n13 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -1826,11 +1988,11 @@ func (m *ConfigMapProjection) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LocalObjectReference.Size())) - n13, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) + n14, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n14 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -1874,11 +2036,11 @@ func (m *ConfigMapVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LocalObjectReference.Size())) - n14, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) + n15, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n15 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -1993,11 +2155,11 @@ func (m *Container) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x42 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Resources.Size())) - n15, err := m.Resources.MarshalTo(dAtA[i:]) + n16, err := m.Resources.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n15 + i += n16 if len(m.VolumeMounts) > 0 { for _, msg := range m.VolumeMounts { dAtA[i] = 0x4a @@ -2014,31 +2176,31 @@ func (m *Container) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x52 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LivenessProbe.Size())) - n16, err := m.LivenessProbe.MarshalTo(dAtA[i:]) + n17, err := m.LivenessProbe.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n17 } if m.ReadinessProbe != nil { dAtA[i] = 0x5a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ReadinessProbe.Size())) - n17, err := m.ReadinessProbe.MarshalTo(dAtA[i:]) + n18, err := m.ReadinessProbe.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n18 } if m.Lifecycle != nil { dAtA[i] = 0x62 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Lifecycle.Size())) - n18, err := m.Lifecycle.MarshalTo(dAtA[i:]) + n19, err := m.Lifecycle.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n19 } dAtA[i] = 0x6a i++ @@ -2052,11 +2214,11 @@ func (m *Container) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x7a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecurityContext.Size())) - n19, err := m.SecurityContext.MarshalTo(dAtA[i:]) + n20, err := m.SecurityContext.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n19 + i += n20 } dAtA[i] = 0x80 i++ @@ -2202,31 +2364,31 @@ func (m *ContainerState) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Waiting.Size())) - n20, err := m.Waiting.MarshalTo(dAtA[i:]) + n21, err := m.Waiting.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n20 + i += n21 } if m.Running != nil { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Running.Size())) - n21, err := m.Running.MarshalTo(dAtA[i:]) + n22, err := m.Running.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n21 + i += n22 } if m.Terminated != nil { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Terminated.Size())) - n22, err := m.Terminated.MarshalTo(dAtA[i:]) + n23, err := m.Terminated.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n22 + i += n23 } return i, nil } @@ -2249,11 +2411,11 @@ func (m *ContainerStateRunning) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.StartedAt.Size())) - n23, err := m.StartedAt.MarshalTo(dAtA[i:]) + n24, err := m.StartedAt.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n23 + i += n24 return i, nil } @@ -2289,19 +2451,19 @@ func (m *ContainerStateTerminated) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.StartedAt.Size())) - n24, err := m.StartedAt.MarshalTo(dAtA[i:]) + n25, err := m.StartedAt.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n24 + i += n25 dAtA[i] = 0x32 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.FinishedAt.Size())) - n25, err := m.FinishedAt.MarshalTo(dAtA[i:]) + n26, err := m.FinishedAt.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n25 + i += n26 dAtA[i] = 0x3a i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.ContainerID))) @@ -2357,19 +2519,19 @@ func (m *ContainerStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.State.Size())) - n26, err := m.State.MarshalTo(dAtA[i:]) + n27, err := m.State.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n26 + i += n27 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LastTerminationState.Size())) - n27, err := m.LastTerminationState.MarshalTo(dAtA[i:]) + n28, err := m.LastTerminationState.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n27 + i += n28 dAtA[i] = 0x20 i++ if m.Ready { @@ -2441,11 +2603,11 @@ func (m *DeleteOptions) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Preconditions.Size())) - n28, err := m.Preconditions.MarshalTo(dAtA[i:]) + n29, err := m.Preconditions.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n28 + i += n29 } if m.OrphanDependents != nil { dAtA[i] = 0x18 @@ -2519,21 +2681,21 @@ func (m *DownwardAPIVolumeFile) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.FieldRef.Size())) - n29, err := m.FieldRef.MarshalTo(dAtA[i:]) + n30, err := m.FieldRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n29 + i += n30 } if m.ResourceFieldRef != nil { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ResourceFieldRef.Size())) - n30, err := m.ResourceFieldRef.MarshalTo(dAtA[i:]) + n31, err := m.ResourceFieldRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n30 + i += n31 } if m.Mode != nil { dAtA[i] = 0x20 @@ -2600,11 +2762,11 @@ func (m *EmptyDirVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SizeLimit.Size())) - n31, err := m.SizeLimit.MarshalTo(dAtA[i:]) + n32, err := m.SizeLimit.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n31 + i += n32 return i, nil } @@ -2631,11 +2793,11 @@ func (m *EndpointAddress) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.TargetRef.Size())) - n32, err := m.TargetRef.MarshalTo(dAtA[i:]) + n33, err := m.TargetRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n32 + i += n33 } dAtA[i] = 0x1a i++ @@ -2751,11 +2913,11 @@ func (m *Endpoints) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n33, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n34, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n33 + i += n34 if len(m.Subsets) > 0 { for _, msg := range m.Subsets { dAtA[i] = 0x12 @@ -2789,11 +2951,11 @@ func (m *EndpointsList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n34, err := m.ListMeta.MarshalTo(dAtA[i:]) + n35, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n34 + i += n35 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -2832,21 +2994,21 @@ func (m *EnvFromSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ConfigMapRef.Size())) - n35, err := m.ConfigMapRef.MarshalTo(dAtA[i:]) + n36, err := m.ConfigMapRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n35 + i += n36 } if m.SecretRef != nil { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecretRef.Size())) - n36, err := m.SecretRef.MarshalTo(dAtA[i:]) + n37, err := m.SecretRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n36 + i += n37 } return i, nil } @@ -2878,11 +3040,11 @@ func (m *EnvVar) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ValueFrom.Size())) - n37, err := m.ValueFrom.MarshalTo(dAtA[i:]) + n38, err := m.ValueFrom.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n37 + i += n38 } return i, nil } @@ -2906,41 +3068,41 @@ func (m *EnvVarSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.FieldRef.Size())) - n38, err := m.FieldRef.MarshalTo(dAtA[i:]) + n39, err := m.FieldRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n38 + i += n39 } if m.ResourceFieldRef != nil { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ResourceFieldRef.Size())) - n39, err := m.ResourceFieldRef.MarshalTo(dAtA[i:]) + n40, err := m.ResourceFieldRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n39 + i += n40 } if m.ConfigMapKeyRef != nil { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ConfigMapKeyRef.Size())) - n40, err := m.ConfigMapKeyRef.MarshalTo(dAtA[i:]) + n41, err := m.ConfigMapKeyRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n40 + i += n41 } if m.SecretKeyRef != nil { dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecretKeyRef.Size())) - n41, err := m.SecretKeyRef.MarshalTo(dAtA[i:]) + n42, err := m.SecretKeyRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n41 + i += n42 } return i, nil } @@ -2963,19 +3125,19 @@ func (m *Event) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n42, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n43, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n42 + i += n43 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.InvolvedObject.Size())) - n43, err := m.InvolvedObject.MarshalTo(dAtA[i:]) + n44, err := m.InvolvedObject.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n43 + i += n44 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Reason))) @@ -2987,27 +3149,27 @@ func (m *Event) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Source.Size())) - n44, err := m.Source.MarshalTo(dAtA[i:]) + n45, err := m.Source.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n44 + i += n45 dAtA[i] = 0x32 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.FirstTimestamp.Size())) - n45, err := m.FirstTimestamp.MarshalTo(dAtA[i:]) + n46, err := m.FirstTimestamp.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n45 + i += n46 dAtA[i] = 0x3a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LastTimestamp.Size())) - n46, err := m.LastTimestamp.MarshalTo(dAtA[i:]) + n47, err := m.LastTimestamp.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n46 + i += n47 dAtA[i] = 0x40 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Count)) @@ -3036,11 +3198,11 @@ func (m *EventList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n47, err := m.ListMeta.MarshalTo(dAtA[i:]) + n48, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n47 + i += n48 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -3207,11 +3369,11 @@ func (m *FlexVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecretRef.Size())) - n48, err := m.SecretRef.MarshalTo(dAtA[i:]) + n49, err := m.SecretRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n48 + i += n49 } dAtA[i] = 0x20 i++ @@ -3395,11 +3557,11 @@ func (m *HTTPGetAction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Port.Size())) - n49, err := m.Port.MarshalTo(dAtA[i:]) + n50, err := m.Port.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n49 + i += n50 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Host))) @@ -3468,31 +3630,31 @@ func (m *Handler) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Exec.Size())) - n50, err := m.Exec.MarshalTo(dAtA[i:]) + n51, err := m.Exec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n50 + i += n51 } if m.HTTPGet != nil { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.HTTPGet.Size())) - n51, err := m.HTTPGet.MarshalTo(dAtA[i:]) + n52, err := m.HTTPGet.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n51 + i += n52 } if m.TCPSocket != nil { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.TCPSocket.Size())) - n52, err := m.TCPSocket.MarshalTo(dAtA[i:]) + n53, err := m.TCPSocket.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n52 + i += n53 } return i, nil } @@ -3553,6 +3715,12 @@ func (m *HostPathVolumeSource) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Path))) i += copy(dAtA[i:], m.Path) + if m.Type != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.Type))) + i += copy(dAtA[i:], *m.Type) + } return i, nil } @@ -3625,11 +3793,11 @@ func (m *ISCSIVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x52 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecretRef.Size())) - n53, err := m.SecretRef.MarshalTo(dAtA[i:]) + n54, err := m.SecretRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n53 + i += n54 } dAtA[i] = 0x58 i++ @@ -3639,6 +3807,12 @@ func (m *ISCSIVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0 } i++ + if m.InitiatorName != nil { + dAtA[i] = 0x62 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(*m.InitiatorName))) + i += copy(dAtA[i:], *m.InitiatorName) + } return i, nil } @@ -3692,21 +3866,21 @@ func (m *Lifecycle) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PostStart.Size())) - n54, err := m.PostStart.MarshalTo(dAtA[i:]) + n55, err := m.PostStart.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n54 + i += n55 } if m.PreStop != nil { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PreStop.Size())) - n55, err := m.PreStop.MarshalTo(dAtA[i:]) + n56, err := m.PreStop.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n55 + i += n56 } return i, nil } @@ -3729,19 +3903,19 @@ func (m *LimitRange) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n56, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n57, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n56 + i += n57 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n57, err := m.Spec.MarshalTo(dAtA[i:]) + n58, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n57 + i += n58 return i, nil } @@ -3788,11 +3962,11 @@ func (m *LimitRangeItem) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n58, err := (&v).MarshalTo(dAtA[i:]) + n59, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n58 + i += n59 } } if len(m.Min) > 0 { @@ -3819,11 +3993,11 @@ func (m *LimitRangeItem) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n59, err := (&v).MarshalTo(dAtA[i:]) + n60, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n59 + i += n60 } } if len(m.Default) > 0 { @@ -3850,11 +4024,11 @@ func (m *LimitRangeItem) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n60, err := (&v).MarshalTo(dAtA[i:]) + n61, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n60 + i += n61 } } if len(m.DefaultRequest) > 0 { @@ -3881,11 +4055,11 @@ func (m *LimitRangeItem) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n61, err := (&v).MarshalTo(dAtA[i:]) + n62, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n61 + i += n62 } } if len(m.MaxLimitRequestRatio) > 0 { @@ -3912,11 +4086,11 @@ func (m *LimitRangeItem) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n62, err := (&v).MarshalTo(dAtA[i:]) + n63, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n62 + i += n63 } } return i, nil @@ -3940,11 +4114,11 @@ func (m *LimitRangeList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n63, err := m.ListMeta.MarshalTo(dAtA[i:]) + n64, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n63 + i += n64 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -4008,11 +4182,11 @@ func (m *List) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n64, err := m.ListMeta.MarshalTo(dAtA[i:]) + n65, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n64 + i += n65 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -4231,27 +4405,27 @@ func (m *Namespace) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n65, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n66, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n65 + i += n66 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n66, err := m.Spec.MarshalTo(dAtA[i:]) + n67, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n66 + i += n67 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n67, err := m.Status.MarshalTo(dAtA[i:]) + n68, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n67 + i += n68 return i, nil } @@ -4273,11 +4447,11 @@ func (m *NamespaceList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n68, err := m.ListMeta.MarshalTo(dAtA[i:]) + n69, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n68 + i += n69 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -4366,27 +4540,27 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n69, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n70, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n69 + i += n70 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n70, err := m.Spec.MarshalTo(dAtA[i:]) + n71, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n70 + i += n71 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n71, err := m.Status.MarshalTo(dAtA[i:]) + n72, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n71 + i += n72 return i, nil } @@ -4435,11 +4609,11 @@ func (m *NodeAffinity) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.RequiredDuringSchedulingIgnoredDuringExecution.Size())) - n72, err := m.RequiredDuringSchedulingIgnoredDuringExecution.MarshalTo(dAtA[i:]) + n73, err := m.RequiredDuringSchedulingIgnoredDuringExecution.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n72 + i += n73 } if len(m.PreferredDuringSchedulingIgnoredDuringExecution) > 0 { for _, msg := range m.PreferredDuringSchedulingIgnoredDuringExecution { @@ -4482,19 +4656,19 @@ func (m *NodeCondition) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LastHeartbeatTime.Size())) - n73, err := m.LastHeartbeatTime.MarshalTo(dAtA[i:]) + n74, err := m.LastHeartbeatTime.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n73 + i += n74 dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LastTransitionTime.Size())) - n74, err := m.LastTransitionTime.MarshalTo(dAtA[i:]) + n75, err := m.LastTransitionTime.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n74 + i += n75 dAtA[i] = 0x2a i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Reason))) @@ -4525,11 +4699,11 @@ func (m *NodeConfigSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ConfigMapRef.Size())) - n75, err := m.ConfigMapRef.MarshalTo(dAtA[i:]) + n76, err := m.ConfigMapRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n75 + i += n76 } return i, nil } @@ -4552,11 +4726,11 @@ func (m *NodeDaemonEndpoints) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.KubeletEndpoint.Size())) - n76, err := m.KubeletEndpoint.MarshalTo(dAtA[i:]) + n77, err := m.KubeletEndpoint.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n76 + i += n77 return i, nil } @@ -4578,11 +4752,11 @@ func (m *NodeList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n77, err := m.ListMeta.MarshalTo(dAtA[i:]) + n78, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n77 + i += n78 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -4659,11 +4833,11 @@ func (m *NodeResources) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n78, err := (&v).MarshalTo(dAtA[i:]) + n79, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n78 + i += n79 } } return i, nil @@ -4821,11 +4995,11 @@ func (m *NodeSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x32 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ConfigSource.Size())) - n79, err := m.ConfigSource.MarshalTo(dAtA[i:]) + n80, err := m.ConfigSource.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n79 + i += n80 } return i, nil } @@ -4869,11 +5043,11 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n80, err := (&v).MarshalTo(dAtA[i:]) + n81, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n80 + i += n81 } } if len(m.Allocatable) > 0 { @@ -4900,11 +5074,11 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n81, err := (&v).MarshalTo(dAtA[i:]) + n82, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n81 + i += n82 } } dAtA[i] = 0x1a @@ -4938,19 +5112,19 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x32 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.DaemonEndpoints.Size())) - n82, err := m.DaemonEndpoints.MarshalTo(dAtA[i:]) + n83, err := m.DaemonEndpoints.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n82 + i += n83 dAtA[i] = 0x3a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.NodeInfo.Size())) - n83, err := m.NodeInfo.MarshalTo(dAtA[i:]) + n84, err := m.NodeInfo.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n83 + i += n84 if len(m.Images) > 0 { for _, msg := range m.Images { dAtA[i] = 0x42 @@ -5122,20 +5296,20 @@ func (m *ObjectMeta) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x42 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.CreationTimestamp.Size())) - n84, err := m.CreationTimestamp.MarshalTo(dAtA[i:]) + n85, err := m.CreationTimestamp.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n84 + i += n85 if m.DeletionTimestamp != nil { dAtA[i] = 0x4a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.DeletionTimestamp.Size())) - n85, err := m.DeletionTimestamp.MarshalTo(dAtA[i:]) + n86, err := m.DeletionTimestamp.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n85 + i += n86 } if m.DeletionGracePeriodSeconds != nil { dAtA[i] = 0x50 @@ -5223,11 +5397,11 @@ func (m *ObjectMeta) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Initializers.Size())) - n86, err := m.Initializers.MarshalTo(dAtA[i:]) + n87, err := m.Initializers.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n86 + i += n87 } return i, nil } @@ -5296,27 +5470,27 @@ func (m *PersistentVolume) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n87, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n88, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n87 + i += n88 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n88, err := m.Spec.MarshalTo(dAtA[i:]) + n89, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n88 + i += n89 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n89, err := m.Status.MarshalTo(dAtA[i:]) + n90, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n89 + i += n90 return i, nil } @@ -5338,27 +5512,27 @@ func (m *PersistentVolumeClaim) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n90, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n91, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n90 + i += n91 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n91, err := m.Spec.MarshalTo(dAtA[i:]) + n92, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n91 + i += n92 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n92, err := m.Status.MarshalTo(dAtA[i:]) + n93, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n92 + i += n93 return i, nil } @@ -5380,11 +5554,11 @@ func (m *PersistentVolumeClaimList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n93, err := m.ListMeta.MarshalTo(dAtA[i:]) + n94, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n93 + i += n94 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -5433,11 +5607,11 @@ func (m *PersistentVolumeClaimSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Resources.Size())) - n94, err := m.Resources.MarshalTo(dAtA[i:]) + n95, err := m.Resources.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n94 + i += n95 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.VolumeName))) @@ -5446,11 +5620,11 @@ func (m *PersistentVolumeClaimSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Selector.Size())) - n95, err := m.Selector.MarshalTo(dAtA[i:]) + n96, err := m.Selector.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n95 + i += n96 } if m.StorageClassName != nil { dAtA[i] = 0x2a @@ -5519,11 +5693,11 @@ func (m *PersistentVolumeClaimStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n96, err := (&v).MarshalTo(dAtA[i:]) + n97, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n96 + i += n97 } } return i, nil @@ -5577,11 +5751,11 @@ func (m *PersistentVolumeList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n97, err := m.ListMeta.MarshalTo(dAtA[i:]) + n98, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n97 + i += n98 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -5616,151 +5790,151 @@ func (m *PersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.GCEPersistentDisk.Size())) - n98, err := m.GCEPersistentDisk.MarshalTo(dAtA[i:]) + n99, err := m.GCEPersistentDisk.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n98 + i += n99 } if m.AWSElasticBlockStore != nil { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.AWSElasticBlockStore.Size())) - n99, err := m.AWSElasticBlockStore.MarshalTo(dAtA[i:]) + n100, err := m.AWSElasticBlockStore.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n99 + i += n100 } if m.HostPath != nil { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.HostPath.Size())) - n100, err := m.HostPath.MarshalTo(dAtA[i:]) + n101, err := m.HostPath.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n100 + i += n101 } if m.Glusterfs != nil { dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Glusterfs.Size())) - n101, err := m.Glusterfs.MarshalTo(dAtA[i:]) + n102, err := m.Glusterfs.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n101 + i += n102 } if m.NFS != nil { dAtA[i] = 0x2a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.NFS.Size())) - n102, err := m.NFS.MarshalTo(dAtA[i:]) + n103, err := m.NFS.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n102 + i += n103 } if m.RBD != nil { dAtA[i] = 0x32 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.RBD.Size())) - n103, err := m.RBD.MarshalTo(dAtA[i:]) + n104, err := m.RBD.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n103 + i += n104 } if m.ISCSI != nil { dAtA[i] = 0x3a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ISCSI.Size())) - n104, err := m.ISCSI.MarshalTo(dAtA[i:]) + n105, err := m.ISCSI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n104 + i += n105 } if m.Cinder != nil { dAtA[i] = 0x42 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Cinder.Size())) - n105, err := m.Cinder.MarshalTo(dAtA[i:]) + n106, err := m.Cinder.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n105 + i += n106 } if m.CephFS != nil { dAtA[i] = 0x4a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.CephFS.Size())) - n106, err := m.CephFS.MarshalTo(dAtA[i:]) + n107, err := m.CephFS.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n106 + i += n107 } if m.FC != nil { dAtA[i] = 0x52 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.FC.Size())) - n107, err := m.FC.MarshalTo(dAtA[i:]) + n108, err := m.FC.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n107 + i += n108 } if m.Flocker != nil { dAtA[i] = 0x5a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Flocker.Size())) - n108, err := m.Flocker.MarshalTo(dAtA[i:]) + n109, err := m.Flocker.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n108 + i += n109 } if m.FlexVolume != nil { dAtA[i] = 0x62 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.FlexVolume.Size())) - n109, err := m.FlexVolume.MarshalTo(dAtA[i:]) + n110, err := m.FlexVolume.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n109 + i += n110 } if m.AzureFile != nil { dAtA[i] = 0x6a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.AzureFile.Size())) - n110, err := m.AzureFile.MarshalTo(dAtA[i:]) + n111, err := m.AzureFile.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n110 + i += n111 } if m.VsphereVolume != nil { dAtA[i] = 0x72 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.VsphereVolume.Size())) - n111, err := m.VsphereVolume.MarshalTo(dAtA[i:]) + n112, err := m.VsphereVolume.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n111 + i += n112 } if m.Quobyte != nil { dAtA[i] = 0x7a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Quobyte.Size())) - n112, err := m.Quobyte.MarshalTo(dAtA[i:]) + n113, err := m.Quobyte.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n112 + i += n113 } if m.AzureDisk != nil { dAtA[i] = 0x82 @@ -5768,11 +5942,11 @@ func (m *PersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.AzureDisk.Size())) - n113, err := m.AzureDisk.MarshalTo(dAtA[i:]) + n114, err := m.AzureDisk.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n113 + i += n114 } if m.PhotonPersistentDisk != nil { dAtA[i] = 0x8a @@ -5780,11 +5954,11 @@ func (m *PersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PhotonPersistentDisk.Size())) - n114, err := m.PhotonPersistentDisk.MarshalTo(dAtA[i:]) + n115, err := m.PhotonPersistentDisk.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n114 + i += n115 } if m.PortworxVolume != nil { dAtA[i] = 0x92 @@ -5792,11 +5966,11 @@ func (m *PersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PortworxVolume.Size())) - n115, err := m.PortworxVolume.MarshalTo(dAtA[i:]) + n116, err := m.PortworxVolume.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n115 + i += n116 } if m.ScaleIO != nil { dAtA[i] = 0x9a @@ -5804,11 +5978,11 @@ func (m *PersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ScaleIO.Size())) - n116, err := m.ScaleIO.MarshalTo(dAtA[i:]) + n117, err := m.ScaleIO.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n116 + i += n117 } if m.Local != nil { dAtA[i] = 0xa2 @@ -5816,11 +5990,11 @@ func (m *PersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Local.Size())) - n117, err := m.Local.MarshalTo(dAtA[i:]) + n118, err := m.Local.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n117 + i += n118 } if m.StorageOS != nil { dAtA[i] = 0xaa @@ -5828,11 +6002,11 @@ func (m *PersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.StorageOS.Size())) - n118, err := m.StorageOS.MarshalTo(dAtA[i:]) + n119, err := m.StorageOS.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n118 + i += n119 } return i, nil } @@ -5876,21 +6050,21 @@ func (m *PersistentVolumeSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n119, err := (&v).MarshalTo(dAtA[i:]) + n120, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n119 + i += n120 } } dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PersistentVolumeSource.Size())) - n120, err := m.PersistentVolumeSource.MarshalTo(dAtA[i:]) + n121, err := m.PersistentVolumeSource.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n120 + i += n121 if len(m.AccessModes) > 0 { for _, s := range m.AccessModes { dAtA[i] = 0x1a @@ -5910,11 +6084,11 @@ func (m *PersistentVolumeSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ClaimRef.Size())) - n121, err := m.ClaimRef.MarshalTo(dAtA[i:]) + n122, err := m.ClaimRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n121 + i += n122 } dAtA[i] = 0x2a i++ @@ -5924,6 +6098,21 @@ func (m *PersistentVolumeSpec) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.StorageClassName))) i += copy(dAtA[i:], m.StorageClassName) + if len(m.MountOptions) > 0 { + for _, s := range m.MountOptions { + dAtA[i] = 0x3a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } return i, nil } @@ -6001,27 +6190,27 @@ func (m *Pod) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n122, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n123, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n122 + i += n123 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n123, err := m.Spec.MarshalTo(dAtA[i:]) + n124, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n123 + i += n124 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n124, err := m.Status.MarshalTo(dAtA[i:]) + n125, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n124 + i += n125 return i, nil } @@ -6086,11 +6275,11 @@ func (m *PodAffinityTerm) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LabelSelector.Size())) - n125, err := m.LabelSelector.MarshalTo(dAtA[i:]) + n126, err := m.LabelSelector.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n125 + i += n126 } if len(m.Namespaces) > 0 { for _, s := range m.Namespaces { @@ -6236,19 +6425,19 @@ func (m *PodCondition) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LastProbeTime.Size())) - n126, err := m.LastProbeTime.MarshalTo(dAtA[i:]) + n127, err := m.LastProbeTime.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n126 + i += n127 dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LastTransitionTime.Size())) - n127, err := m.LastTransitionTime.MarshalTo(dAtA[i:]) + n128, err := m.LastTransitionTime.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n127 + i += n128 dAtA[i] = 0x2a i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Reason))) @@ -6347,11 +6536,11 @@ func (m *PodList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n128, err := m.ListMeta.MarshalTo(dAtA[i:]) + n129, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n128 + i += n129 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -6411,11 +6600,11 @@ func (m *PodLogOptions) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SinceTime.Size())) - n129, err := m.SinceTime.MarshalTo(dAtA[i:]) + n130, err := m.SinceTime.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n129 + i += n130 } dAtA[i] = 0x30 i++ @@ -6504,11 +6693,11 @@ func (m *PodSecurityContext) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SELinuxOptions.Size())) - n130, err := m.SELinuxOptions.MarshalTo(dAtA[i:]) + n131, err := m.SELinuxOptions.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n130 + i += n131 } if m.RunAsUser != nil { dAtA[i] = 0x10 @@ -6559,11 +6748,11 @@ func (m *PodSignature) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PodController.Size())) - n131, err := m.PodController.MarshalTo(dAtA[i:]) + n132, err := m.PodController.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n131 + i += n132 } return i, nil } @@ -6687,11 +6876,11 @@ func (m *PodSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x72 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecurityContext.Size())) - n132, err := m.SecurityContext.MarshalTo(dAtA[i:]) + n133, err := m.SecurityContext.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n132 + i += n133 } if len(m.ImagePullSecrets) > 0 { for _, msg := range m.ImagePullSecrets { @@ -6723,11 +6912,11 @@ func (m *PodSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Affinity.Size())) - n133, err := m.Affinity.MarshalTo(dAtA[i:]) + n134, err := m.Affinity.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n133 + i += n134 } dAtA[i] = 0x9a i++ @@ -6856,11 +7045,11 @@ func (m *PodStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x3a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.StartTime.Size())) - n134, err := m.StartTime.MarshalTo(dAtA[i:]) + n135, err := m.StartTime.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n134 + i += n135 } if len(m.ContainerStatuses) > 0 { for _, msg := range m.ContainerStatuses { @@ -6911,19 +7100,19 @@ func (m *PodStatusResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n135, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n136, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n135 + i += n136 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n136, err := m.Status.MarshalTo(dAtA[i:]) + n137, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n136 + i += n137 return i, nil } @@ -6945,19 +7134,19 @@ func (m *PodTemplate) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n137, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n138, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n137 + i += n138 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Template.Size())) - n138, err := m.Template.MarshalTo(dAtA[i:]) + n139, err := m.Template.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n138 + i += n139 return i, nil } @@ -6979,11 +7168,11 @@ func (m *PodTemplateList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n139, err := m.ListMeta.MarshalTo(dAtA[i:]) + n140, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n139 + i += n140 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -7017,19 +7206,19 @@ func (m *PodTemplateSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n140, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n141, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n140 + i += n141 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n141, err := m.Spec.MarshalTo(dAtA[i:]) + n142, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n141 + i += n142 return i, nil } @@ -7109,19 +7298,19 @@ func (m *PreferAvoidPodsEntry) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PodSignature.Size())) - n142, err := m.PodSignature.MarshalTo(dAtA[i:]) + n143, err := m.PodSignature.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n142 + i += n143 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.EvictionTime.Size())) - n143, err := m.EvictionTime.MarshalTo(dAtA[i:]) + n144, err := m.EvictionTime.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n143 + i += n144 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Reason))) @@ -7154,11 +7343,11 @@ func (m *PreferredSchedulingTerm) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Preference.Size())) - n144, err := m.Preference.MarshalTo(dAtA[i:]) + n145, err := m.Preference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n144 + i += n145 return i, nil } @@ -7180,11 +7369,11 @@ func (m *Probe) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Handler.Size())) - n145, err := m.Handler.MarshalTo(dAtA[i:]) + n146, err := m.Handler.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n145 + i += n146 dAtA[i] = 0x10 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.InitialDelaySeconds)) @@ -7334,11 +7523,11 @@ func (m *RBDVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x3a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecretRef.Size())) - n146, err := m.SecretRef.MarshalTo(dAtA[i:]) + n147, err := m.SecretRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n146 + i += n147 } dAtA[i] = 0x40 i++ @@ -7369,11 +7558,11 @@ func (m *RangeAllocation) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n147, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n148, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n147 + i += n148 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Range))) @@ -7405,27 +7594,27 @@ func (m *ReplicationController) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n148, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n149, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n148 + i += n149 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n149, err := m.Spec.MarshalTo(dAtA[i:]) + n150, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n149 + i += n150 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n150, err := m.Status.MarshalTo(dAtA[i:]) + n151, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n150 + i += n151 return i, nil } @@ -7455,11 +7644,11 @@ func (m *ReplicationControllerCondition) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LastTransitionTime.Size())) - n151, err := m.LastTransitionTime.MarshalTo(dAtA[i:]) + n152, err := m.LastTransitionTime.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n151 + i += n152 dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Reason))) @@ -7489,11 +7678,11 @@ func (m *ReplicationControllerList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n152, err := m.ListMeta.MarshalTo(dAtA[i:]) + n153, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n152 + i += n153 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -7555,11 +7744,11 @@ func (m *ReplicationControllerSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Template.Size())) - n153, err := m.Template.MarshalTo(dAtA[i:]) + n154, err := m.Template.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n153 + i += n154 } dAtA[i] = 0x20 i++ @@ -7638,11 +7827,11 @@ func (m *ResourceFieldSelector) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Divisor.Size())) - n154, err := m.Divisor.MarshalTo(dAtA[i:]) + n155, err := m.Divisor.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n154 + i += n155 return i, nil } @@ -7664,27 +7853,27 @@ func (m *ResourceQuota) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n155, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n156, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n155 + i += n156 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n156, err := m.Spec.MarshalTo(dAtA[i:]) + n157, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n156 + i += n157 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n157, err := m.Status.MarshalTo(dAtA[i:]) + n158, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n157 + i += n158 return i, nil } @@ -7706,11 +7895,11 @@ func (m *ResourceQuotaList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n158, err := m.ListMeta.MarshalTo(dAtA[i:]) + n159, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n158 + i += n159 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -7765,11 +7954,11 @@ func (m *ResourceQuotaSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n159, err := (&v).MarshalTo(dAtA[i:]) + n160, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n159 + i += n160 } } if len(m.Scopes) > 0 { @@ -7829,11 +8018,11 @@ func (m *ResourceQuotaStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n160, err := (&v).MarshalTo(dAtA[i:]) + n161, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n160 + i += n161 } } if len(m.Used) > 0 { @@ -7860,11 +8049,11 @@ func (m *ResourceQuotaStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n161, err := (&v).MarshalTo(dAtA[i:]) + n162, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n161 + i += n162 } } return i, nil @@ -7909,11 +8098,11 @@ func (m *ResourceRequirements) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n162, err := (&v).MarshalTo(dAtA[i:]) + n163, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n162 + i += n163 } } if len(m.Requests) > 0 { @@ -7940,11 +8129,11 @@ func (m *ResourceRequirements) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64((&v).Size())) - n163, err := (&v).MarshalTo(dAtA[i:]) + n164, err := (&v).MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n163 + i += n164 } } return i, nil @@ -8011,11 +8200,11 @@ func (m *ScaleIOVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecretRef.Size())) - n164, err := m.SecretRef.MarshalTo(dAtA[i:]) + n165, err := m.SecretRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n164 + i += n165 } dAtA[i] = 0x20 i++ @@ -8074,11 +8263,11 @@ func (m *Secret) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n165, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n166, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n165 + i += n166 if len(m.Data) > 0 { keysForData := make([]string, 0, len(m.Data)) for k := range m.Data { @@ -8154,11 +8343,11 @@ func (m *SecretEnvSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LocalObjectReference.Size())) - n166, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) + n167, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n166 + i += n167 if m.Optional != nil { dAtA[i] = 0x10 i++ @@ -8190,11 +8379,11 @@ func (m *SecretKeySelector) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LocalObjectReference.Size())) - n167, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) + n168, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n167 + i += n168 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Key))) @@ -8230,11 +8419,11 @@ func (m *SecretList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n168, err := m.ListMeta.MarshalTo(dAtA[i:]) + n169, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n168 + i += n169 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -8268,11 +8457,11 @@ func (m *SecretProjection) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LocalObjectReference.Size())) - n169, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) + n170, err := m.LocalObjectReference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n169 + i += n170 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -8298,6 +8487,32 @@ func (m *SecretProjection) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *SecretReference) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SecretReference) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Namespace))) + i += copy(dAtA[i:], m.Namespace) + return i, nil +} + func (m *SecretVolumeSource) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -8366,11 +8581,11 @@ func (m *SecurityContext) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Capabilities.Size())) - n170, err := m.Capabilities.MarshalTo(dAtA[i:]) + n171, err := m.Capabilities.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n170 + i += n171 } if m.Privileged != nil { dAtA[i] = 0x10 @@ -8386,11 +8601,11 @@ func (m *SecurityContext) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SELinuxOptions.Size())) - n171, err := m.SELinuxOptions.MarshalTo(dAtA[i:]) + n172, err := m.SELinuxOptions.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n171 + i += n172 } if m.RunAsUser != nil { dAtA[i] = 0x20 @@ -8448,11 +8663,11 @@ func (m *SerializedReference) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Reference.Size())) - n172, err := m.Reference.MarshalTo(dAtA[i:]) + n173, err := m.Reference.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n172 + i += n173 return i, nil } @@ -8474,27 +8689,27 @@ func (m *Service) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n173, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n174, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n173 + i += n174 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n174, err := m.Spec.MarshalTo(dAtA[i:]) + n175, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n174 + i += n175 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n175, err := m.Status.MarshalTo(dAtA[i:]) + n176, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n175 + i += n176 return i, nil } @@ -8516,11 +8731,11 @@ func (m *ServiceAccount) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n176, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n177, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n176 + i += n177 if len(m.Secrets) > 0 { for _, msg := range m.Secrets { dAtA[i] = 0x12 @@ -8576,11 +8791,11 @@ func (m *ServiceAccountList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n177, err := m.ListMeta.MarshalTo(dAtA[i:]) + n178, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n177 + i += n178 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -8614,11 +8829,11 @@ func (m *ServiceList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n178, err := m.ListMeta.MarshalTo(dAtA[i:]) + n179, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n178 + i += n179 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -8663,11 +8878,11 @@ func (m *ServicePort) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.TargetPort.Size())) - n179, err := m.TargetPort.MarshalTo(dAtA[i:]) + n180, err := m.TargetPort.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n179 + i += n180 dAtA[i] = 0x28 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.NodePort)) @@ -8810,6 +9025,16 @@ func (m *ServiceSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0 } i++ + if m.SessionAffinityConfig != nil { + dAtA[i] = 0x72 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.SessionAffinityConfig.Size())) + n181, err := m.SessionAffinityConfig.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n181 + } return i, nil } @@ -8831,11 +9056,39 @@ func (m *ServiceStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LoadBalancer.Size())) - n180, err := m.LoadBalancer.MarshalTo(dAtA[i:]) + n182, err := m.LoadBalancer.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n180 + i += n182 + return i, nil +} + +func (m *SessionAffinityConfig) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SessionAffinityConfig) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.ClientIP != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.ClientIP.Size())) + n183, err := m.ClientIP.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n183 + } return i, nil } @@ -8878,11 +9131,11 @@ func (m *StorageOSPersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecretRef.Size())) - n181, err := m.SecretRef.MarshalTo(dAtA[i:]) + n184, err := m.SecretRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n181 + i += n184 } return i, nil } @@ -8926,11 +9179,11 @@ func (m *StorageOSVolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SecretRef.Size())) - n182, err := m.SecretRef.MarshalTo(dAtA[i:]) + n185, err := m.SecretRef.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n182 + i += n185 } return i, nil } @@ -8979,11 +9232,11 @@ func (m *TCPSocketAction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Port.Size())) - n183, err := m.Port.MarshalTo(dAtA[i:]) + n186, err := m.Port.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n183 + i += n186 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Host))) @@ -9021,11 +9274,11 @@ func (m *Taint) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.TimeAdded.Size())) - n184, err := m.TimeAdded.MarshalTo(dAtA[i:]) + n187, err := m.TimeAdded.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n184 + i += n187 return i, nil } @@ -9090,11 +9343,11 @@ func (m *Volume) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.VolumeSource.Size())) - n185, err := m.VolumeSource.MarshalTo(dAtA[i:]) + n188, err := m.VolumeSource.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n185 + i += n188 return i, nil } @@ -9155,31 +9408,31 @@ func (m *VolumeProjection) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Secret.Size())) - n186, err := m.Secret.MarshalTo(dAtA[i:]) + n189, err := m.Secret.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n186 + i += n189 } if m.DownwardAPI != nil { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.DownwardAPI.Size())) - n187, err := m.DownwardAPI.MarshalTo(dAtA[i:]) + n190, err := m.DownwardAPI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n187 + i += n190 } if m.ConfigMap != nil { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ConfigMap.Size())) - n188, err := m.ConfigMap.MarshalTo(dAtA[i:]) + n191, err := m.ConfigMap.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n188 + i += n191 } return i, nil } @@ -9203,151 +9456,151 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.HostPath.Size())) - n189, err := m.HostPath.MarshalTo(dAtA[i:]) + n192, err := m.HostPath.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n189 + i += n192 } if m.EmptyDir != nil { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.EmptyDir.Size())) - n190, err := m.EmptyDir.MarshalTo(dAtA[i:]) + n193, err := m.EmptyDir.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n190 + i += n193 } if m.GCEPersistentDisk != nil { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.GCEPersistentDisk.Size())) - n191, err := m.GCEPersistentDisk.MarshalTo(dAtA[i:]) + n194, err := m.GCEPersistentDisk.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n191 + i += n194 } if m.AWSElasticBlockStore != nil { dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.AWSElasticBlockStore.Size())) - n192, err := m.AWSElasticBlockStore.MarshalTo(dAtA[i:]) + n195, err := m.AWSElasticBlockStore.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n192 + i += n195 } if m.GitRepo != nil { dAtA[i] = 0x2a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.GitRepo.Size())) - n193, err := m.GitRepo.MarshalTo(dAtA[i:]) + n196, err := m.GitRepo.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n193 + i += n196 } if m.Secret != nil { dAtA[i] = 0x32 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Secret.Size())) - n194, err := m.Secret.MarshalTo(dAtA[i:]) + n197, err := m.Secret.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n194 + i += n197 } if m.NFS != nil { dAtA[i] = 0x3a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.NFS.Size())) - n195, err := m.NFS.MarshalTo(dAtA[i:]) + n198, err := m.NFS.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n195 + i += n198 } if m.ISCSI != nil { dAtA[i] = 0x42 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ISCSI.Size())) - n196, err := m.ISCSI.MarshalTo(dAtA[i:]) + n199, err := m.ISCSI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n196 + i += n199 } if m.Glusterfs != nil { dAtA[i] = 0x4a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Glusterfs.Size())) - n197, err := m.Glusterfs.MarshalTo(dAtA[i:]) + n200, err := m.Glusterfs.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n197 + i += n200 } if m.PersistentVolumeClaim != nil { dAtA[i] = 0x52 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PersistentVolumeClaim.Size())) - n198, err := m.PersistentVolumeClaim.MarshalTo(dAtA[i:]) + n201, err := m.PersistentVolumeClaim.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n198 + i += n201 } if m.RBD != nil { dAtA[i] = 0x5a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.RBD.Size())) - n199, err := m.RBD.MarshalTo(dAtA[i:]) + n202, err := m.RBD.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n199 + i += n202 } if m.FlexVolume != nil { dAtA[i] = 0x62 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.FlexVolume.Size())) - n200, err := m.FlexVolume.MarshalTo(dAtA[i:]) + n203, err := m.FlexVolume.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n200 + i += n203 } if m.Cinder != nil { dAtA[i] = 0x6a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Cinder.Size())) - n201, err := m.Cinder.MarshalTo(dAtA[i:]) + n204, err := m.Cinder.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n201 + i += n204 } if m.CephFS != nil { dAtA[i] = 0x72 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.CephFS.Size())) - n202, err := m.CephFS.MarshalTo(dAtA[i:]) + n205, err := m.CephFS.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n202 + i += n205 } if m.Flocker != nil { dAtA[i] = 0x7a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Flocker.Size())) - n203, err := m.Flocker.MarshalTo(dAtA[i:]) + n206, err := m.Flocker.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n203 + i += n206 } if m.DownwardAPI != nil { dAtA[i] = 0x82 @@ -9355,11 +9608,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.DownwardAPI.Size())) - n204, err := m.DownwardAPI.MarshalTo(dAtA[i:]) + n207, err := m.DownwardAPI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n204 + i += n207 } if m.FC != nil { dAtA[i] = 0x8a @@ -9367,11 +9620,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.FC.Size())) - n205, err := m.FC.MarshalTo(dAtA[i:]) + n208, err := m.FC.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n205 + i += n208 } if m.AzureFile != nil { dAtA[i] = 0x92 @@ -9379,11 +9632,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.AzureFile.Size())) - n206, err := m.AzureFile.MarshalTo(dAtA[i:]) + n209, err := m.AzureFile.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n206 + i += n209 } if m.ConfigMap != nil { dAtA[i] = 0x9a @@ -9391,11 +9644,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ConfigMap.Size())) - n207, err := m.ConfigMap.MarshalTo(dAtA[i:]) + n210, err := m.ConfigMap.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n207 + i += n210 } if m.VsphereVolume != nil { dAtA[i] = 0xa2 @@ -9403,11 +9656,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.VsphereVolume.Size())) - n208, err := m.VsphereVolume.MarshalTo(dAtA[i:]) + n211, err := m.VsphereVolume.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n208 + i += n211 } if m.Quobyte != nil { dAtA[i] = 0xaa @@ -9415,11 +9668,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Quobyte.Size())) - n209, err := m.Quobyte.MarshalTo(dAtA[i:]) + n212, err := m.Quobyte.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n209 + i += n212 } if m.AzureDisk != nil { dAtA[i] = 0xb2 @@ -9427,11 +9680,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.AzureDisk.Size())) - n210, err := m.AzureDisk.MarshalTo(dAtA[i:]) + n213, err := m.AzureDisk.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n210 + i += n213 } if m.PhotonPersistentDisk != nil { dAtA[i] = 0xba @@ -9439,11 +9692,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PhotonPersistentDisk.Size())) - n211, err := m.PhotonPersistentDisk.MarshalTo(dAtA[i:]) + n214, err := m.PhotonPersistentDisk.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n211 + i += n214 } if m.PortworxVolume != nil { dAtA[i] = 0xc2 @@ -9451,11 +9704,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PortworxVolume.Size())) - n212, err := m.PortworxVolume.MarshalTo(dAtA[i:]) + n215, err := m.PortworxVolume.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n212 + i += n215 } if m.ScaleIO != nil { dAtA[i] = 0xca @@ -9463,11 +9716,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ScaleIO.Size())) - n213, err := m.ScaleIO.MarshalTo(dAtA[i:]) + n216, err := m.ScaleIO.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n213 + i += n216 } if m.Projected != nil { dAtA[i] = 0xd2 @@ -9475,11 +9728,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Projected.Size())) - n214, err := m.Projected.MarshalTo(dAtA[i:]) + n217, err := m.Projected.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n214 + i += n217 } if m.StorageOS != nil { dAtA[i] = 0xda @@ -9487,11 +9740,11 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.StorageOS.Size())) - n215, err := m.StorageOS.MarshalTo(dAtA[i:]) + n218, err := m.StorageOS.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n215 + i += n218 } return i, nil } @@ -9551,11 +9804,11 @@ func (m *WeightedPodAffinityTerm) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PodAffinityTerm.Size())) - n216, err := m.PodAffinityTerm.MarshalTo(dAtA[i:]) + n219, err := m.PodAffinityTerm.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n216 + i += n219 return i, nil } @@ -9663,6 +9916,21 @@ func (m *AzureDiskVolumeSource) Size() (n int) { return n } +func (m *AzureFilePersistentVolumeSource) Size() (n int) { + var l int + _ = l + l = len(m.SecretName) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.ShareName) + n += 1 + l + sovGenerated(uint64(l)) + n += 2 + if m.SecretNamespace != nil { + l = len(*m.SecretNamespace) + n += 1 + l + sovGenerated(uint64(l)) + } + return n +} + func (m *AzureFileVolumeSource) Size() (n int) { var l int _ = l @@ -9702,6 +9970,29 @@ func (m *Capabilities) Size() (n int) { return n } +func (m *CephFSPersistentVolumeSource) Size() (n int) { + var l int + _ = l + if len(m.Monitors) > 0 { + for _, s := range m.Monitors { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } + l = len(m.Path) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.User) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.SecretFile) + n += 1 + l + sovGenerated(uint64(l)) + if m.SecretRef != nil { + l = m.SecretRef.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + n += 2 + return n +} + func (m *CephFSVolumeSource) Size() (n int) { var l int _ = l @@ -9736,6 +10027,15 @@ func (m *CinderVolumeSource) Size() (n int) { return n } +func (m *ClientIPConfig) Size() (n int) { + var l int + _ = l + if m.TimeoutSeconds != nil { + n += 1 + sovGenerated(uint64(*m.TimeoutSeconds)) + } + return n +} + func (m *ComponentCondition) Size() (n int) { var l int _ = l @@ -10479,6 +10779,10 @@ func (m *HostPathVolumeSource) Size() (n int) { _ = l l = len(m.Path) n += 1 + l + sovGenerated(uint64(l)) + if m.Type != nil { + l = len(*m.Type) + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -10507,6 +10811,10 @@ func (m *ISCSIVolumeSource) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) } n += 2 + if m.InitiatorName != nil { + l = len(*m.InitiatorName) + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -11330,6 +11638,12 @@ func (m *PersistentVolumeSpec) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) l = len(m.StorageClassName) n += 1 + l + sovGenerated(uint64(l)) + if len(m.MountOptions) > 0 { + for _, s := range m.MountOptions { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -12176,6 +12490,16 @@ func (m *SecretProjection) Size() (n int) { return n } +func (m *SecretReference) Size() (n int) { + var l int + _ = l + l = len(m.Name) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.Namespace) + n += 1 + l + sovGenerated(uint64(l)) + return n +} + func (m *SecretVolumeSource) Size() (n int) { var l int _ = l @@ -12361,6 +12685,10 @@ func (m *ServiceSpec) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) n += 1 + sovGenerated(uint64(m.HealthCheckNodePort)) n += 2 + if m.SessionAffinityConfig != nil { + l = m.SessionAffinityConfig.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -12372,6 +12700,16 @@ func (m *ServiceStatus) Size() (n int) { return n } +func (m *SessionAffinityConfig) Size() (n int) { + var l int + _ = l + if m.ClientIP != nil { + l = m.ClientIP.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + return n +} + func (m *StorageOSPersistentVolumeSource) Size() (n int) { var l int _ = l @@ -12709,6 +13047,19 @@ func (this *AzureDiskVolumeSource) String() string { }, "") return s } +func (this *AzureFilePersistentVolumeSource) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&AzureFilePersistentVolumeSource{`, + `SecretName:` + fmt.Sprintf("%v", this.SecretName) + `,`, + `ShareName:` + fmt.Sprintf("%v", this.ShareName) + `,`, + `ReadOnly:` + fmt.Sprintf("%v", this.ReadOnly) + `,`, + `SecretNamespace:` + valueToStringGenerated(this.SecretNamespace) + `,`, + `}`, + }, "") + return s +} func (this *AzureFileVolumeSource) String() string { if this == nil { return "nil" @@ -12743,6 +13094,21 @@ func (this *Capabilities) String() string { }, "") return s } +func (this *CephFSPersistentVolumeSource) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&CephFSPersistentVolumeSource{`, + `Monitors:` + fmt.Sprintf("%v", this.Monitors) + `,`, + `Path:` + fmt.Sprintf("%v", this.Path) + `,`, + `User:` + fmt.Sprintf("%v", this.User) + `,`, + `SecretFile:` + fmt.Sprintf("%v", this.SecretFile) + `,`, + `SecretRef:` + strings.Replace(fmt.Sprintf("%v", this.SecretRef), "SecretReference", "SecretReference", 1) + `,`, + `ReadOnly:` + fmt.Sprintf("%v", this.ReadOnly) + `,`, + `}`, + }, "") + return s +} func (this *CephFSVolumeSource) String() string { if this == nil { return "nil" @@ -12770,6 +13136,16 @@ func (this *CinderVolumeSource) String() string { }, "") return s } +func (this *ClientIPConfig) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ClientIPConfig{`, + `TimeoutSeconds:` + valueToStringGenerated(this.TimeoutSeconds) + `,`, + `}`, + }, "") + return s +} func (this *ComponentCondition) String() string { if this == nil { return "nil" @@ -13359,6 +13735,7 @@ func (this *HostPathVolumeSource) String() string { } s := strings.Join([]string{`&HostPathVolumeSource{`, `Path:` + fmt.Sprintf("%v", this.Path) + `,`, + `Type:` + valueToStringGenerated(this.Type) + `,`, `}`, }, "") return s @@ -13378,6 +13755,7 @@ func (this *ISCSIVolumeSource) String() string { `DiscoveryCHAPAuth:` + fmt.Sprintf("%v", this.DiscoveryCHAPAuth) + `,`, `SecretRef:` + strings.Replace(fmt.Sprintf("%v", this.SecretRef), "LocalObjectReference", "LocalObjectReference", 1) + `,`, `SessionCHAPAuth:` + fmt.Sprintf("%v", this.SessionCHAPAuth) + `,`, + `InitiatorName:` + valueToStringGenerated(this.InitiatorName) + `,`, `}`, }, "") return s @@ -14017,11 +14395,11 @@ func (this *PersistentVolumeSource) String() string { `RBD:` + strings.Replace(fmt.Sprintf("%v", this.RBD), "RBDVolumeSource", "RBDVolumeSource", 1) + `,`, `ISCSI:` + strings.Replace(fmt.Sprintf("%v", this.ISCSI), "ISCSIVolumeSource", "ISCSIVolumeSource", 1) + `,`, `Cinder:` + strings.Replace(fmt.Sprintf("%v", this.Cinder), "CinderVolumeSource", "CinderVolumeSource", 1) + `,`, - `CephFS:` + strings.Replace(fmt.Sprintf("%v", this.CephFS), "CephFSVolumeSource", "CephFSVolumeSource", 1) + `,`, + `CephFS:` + strings.Replace(fmt.Sprintf("%v", this.CephFS), "CephFSPersistentVolumeSource", "CephFSPersistentVolumeSource", 1) + `,`, `FC:` + strings.Replace(fmt.Sprintf("%v", this.FC), "FCVolumeSource", "FCVolumeSource", 1) + `,`, `Flocker:` + strings.Replace(fmt.Sprintf("%v", this.Flocker), "FlockerVolumeSource", "FlockerVolumeSource", 1) + `,`, `FlexVolume:` + strings.Replace(fmt.Sprintf("%v", this.FlexVolume), "FlexVolumeSource", "FlexVolumeSource", 1) + `,`, - `AzureFile:` + strings.Replace(fmt.Sprintf("%v", this.AzureFile), "AzureFileVolumeSource", "AzureFileVolumeSource", 1) + `,`, + `AzureFile:` + strings.Replace(fmt.Sprintf("%v", this.AzureFile), "AzureFilePersistentVolumeSource", "AzureFilePersistentVolumeSource", 1) + `,`, `VsphereVolume:` + strings.Replace(fmt.Sprintf("%v", this.VsphereVolume), "VsphereVirtualDiskVolumeSource", "VsphereVirtualDiskVolumeSource", 1) + `,`, `Quobyte:` + strings.Replace(fmt.Sprintf("%v", this.Quobyte), "QuobyteVolumeSource", "QuobyteVolumeSource", 1) + `,`, `AzureDisk:` + strings.Replace(fmt.Sprintf("%v", this.AzureDisk), "AzureDiskVolumeSource", "AzureDiskVolumeSource", 1) + `,`, @@ -14055,6 +14433,7 @@ func (this *PersistentVolumeSpec) String() string { `ClaimRef:` + strings.Replace(fmt.Sprintf("%v", this.ClaimRef), "ObjectReference", "ObjectReference", 1) + `,`, `PersistentVolumeReclaimPolicy:` + fmt.Sprintf("%v", this.PersistentVolumeReclaimPolicy) + `,`, `StorageClassName:` + fmt.Sprintf("%v", this.StorageClassName) + `,`, + `MountOptions:` + fmt.Sprintf("%v", this.MountOptions) + `,`, `}`, }, "") return s @@ -14770,6 +15149,17 @@ func (this *SecretProjection) String() string { }, "") return s } +func (this *SecretReference) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&SecretReference{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, + `}`, + }, "") + return s +} func (this *SecretVolumeSource) String() string { if this == nil { return "nil" @@ -14907,6 +15297,7 @@ func (this *ServiceSpec) String() string { `ExternalTrafficPolicy:` + fmt.Sprintf("%v", this.ExternalTrafficPolicy) + `,`, `HealthCheckNodePort:` + fmt.Sprintf("%v", this.HealthCheckNodePort) + `,`, `PublishNotReadyAddresses:` + fmt.Sprintf("%v", this.PublishNotReadyAddresses) + `,`, + `SessionAffinityConfig:` + strings.Replace(fmt.Sprintf("%v", this.SessionAffinityConfig), "SessionAffinityConfig", "SessionAffinityConfig", 1) + `,`, `}`, }, "") return s @@ -14921,6 +15312,16 @@ func (this *ServiceStatus) String() string { }, "") return s } +func (this *SessionAffinityConfig) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&SessionAffinityConfig{`, + `ClientIP:` + strings.Replace(fmt.Sprintf("%v", this.ClientIP), "ClientIPConfig", "ClientIPConfig", 1) + `,`, + `}`, + }, "") + return s +} func (this *StorageOSPersistentVolumeSource) String() string { if this == nil { return "nil" @@ -15806,6 +16207,164 @@ func (m *AzureDiskVolumeSource) Unmarshal(dAtA []byte) error { } return nil } +func (m *AzureFilePersistentVolumeSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: AzureFilePersistentVolumeSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: AzureFilePersistentVolumeSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SecretName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SecretName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ShareName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ShareName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ReadOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ReadOnly = bool(v != 0) + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SecretNamespace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.SecretNamespace = &s + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *AzureFileVolumeSource) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -16152,6 +16711,225 @@ func (m *Capabilities) Unmarshal(dAtA []byte) error { } return nil } +func (m *CephFSPersistentVolumeSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CephFSPersistentVolumeSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CephFSPersistentVolumeSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Monitors", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Monitors = append(m.Monitors, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Path", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Path = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.User = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SecretFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SecretFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SecretRef", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SecretRef == nil { + m.SecretRef = &SecretReference{} + } + if err := m.SecretRef.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ReadOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ReadOnly = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *CephFSVolumeSource) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -16237,100 +17015,228 @@ func (m *CephFSVolumeSource) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Path = string(dAtA[iNdEx:postIndex]) + m.Path = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.User = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SecretFile", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SecretFile = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SecretRef", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SecretRef == nil { + m.SecretRef = &LocalObjectReference{} + } + if err := m.SecretRef.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ReadOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ReadOnly = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CinderVolumeSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CinderVolumeSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CinderVolumeSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VolumeID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FSType", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FSType = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.User = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SecretFile", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.SecretFile = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SecretRef", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.SecretRef == nil { - m.SecretRef = &LocalObjectReference{} - } - if err := m.SecretRef.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ReadOnly", wireType) } @@ -16371,7 +17277,7 @@ func (m *CephFSVolumeSource) Unmarshal(dAtA []byte) error { } return nil } -func (m *CinderVolumeSource) Unmarshal(dAtA []byte) error { +func (m *ClientIPConfig) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -16394,75 +17300,17 @@ func (m *CinderVolumeSource) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: CinderVolumeSource: wiretype end group for non-group") + return fmt.Errorf("proto: ClientIPConfig: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: CinderVolumeSource: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ClientIPConfig: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field VolumeID", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.VolumeID = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field FSType", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.FSType = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ReadOnly", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field TimeoutSeconds", wireType) } - var v int + var v int32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -16472,12 +17320,12 @@ func (m *CinderVolumeSource) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= (int32(b) & 0x7F) << shift if b < 0x80 { break } } - m.ReadOnly = bool(v != 0) + m.TimeoutSeconds = &v default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -23456,6 +24304,36 @@ func (m *HostPathVolumeSource) Unmarshal(dAtA []byte) error { } m.Path = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := HostPathType(dAtA[iNdEx:postIndex]) + m.Type = &s + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -23763,6 +24641,36 @@ func (m *ISCSIVolumeSource) Unmarshal(dAtA []byte) error { } } m.SessionCHAPAuth = bool(v != 0) + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InitiatorName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.InitiatorName = &s + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -31104,7 +32012,7 @@ func (m *PersistentVolumeSource) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.CephFS == nil { - m.CephFS = &CephFSVolumeSource{} + m.CephFS = &CephFSPersistentVolumeSource{} } if err := m.CephFS.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -31236,7 +32144,7 @@ func (m *PersistentVolumeSource) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.AzureFile == nil { - m.AzureFile = &AzureFileVolumeSource{} + m.AzureFile = &AzureFilePersistentVolumeSource{} } if err := m.AzureFile.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -31703,13 +32611,75 @@ func (m *PersistentVolumeSpec) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if err := m.PersistentVolumeSource.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.PersistentVolumeSource.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AccessModes", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AccessModes = append(m.AccessModes, PersistentVolumeAccessMode(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClaimRef", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ClaimRef == nil { + m.ClaimRef = &ObjectReference{} + } + if err := m.ClaimRef.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 3: + case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AccessModes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field PersistentVolumeReclaimPolicy", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -31734,44 +32704,11 @@ func (m *PersistentVolumeSpec) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.AccessModes = append(m.AccessModes, PersistentVolumeAccessMode(dAtA[iNdEx:postIndex])) - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ClaimRef", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.ClaimRef == nil { - m.ClaimRef = &ObjectReference{} - } - if err := m.ClaimRef.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.PersistentVolumeReclaimPolicy = PersistentVolumeReclaimPolicy(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 5: + case 6: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PersistentVolumeReclaimPolicy", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field StorageClassName", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -31796,11 +32733,11 @@ func (m *PersistentVolumeSpec) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.PersistentVolumeReclaimPolicy = PersistentVolumeReclaimPolicy(dAtA[iNdEx:postIndex]) + m.StorageClassName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 6: + case 7: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field StorageClassName", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field MountOptions", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -31825,7 +32762,7 @@ func (m *PersistentVolumeSpec) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.StorageClassName = string(dAtA[iNdEx:postIndex]) + m.MountOptions = append(m.MountOptions, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex default: iNdEx = preIndex @@ -40301,6 +41238,114 @@ func (m *SecretProjection) Unmarshal(dAtA []byte) error { } return nil } +func (m *SecretReference) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SecretReference: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SecretReference: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Namespace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *SecretVolumeSource) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -41979,6 +43024,39 @@ func (m *ServiceSpec) Unmarshal(dAtA []byte) error { } } m.PublishNotReadyAddresses = bool(v != 0) + case 14: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SessionAffinityConfig", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SessionAffinityConfig == nil { + m.SessionAffinityConfig = &SessionAffinityConfig{} + } + if err := m.SessionAffinityConfig.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -42080,6 +43158,89 @@ func (m *ServiceStatus) Unmarshal(dAtA []byte) error { } return nil } +func (m *SessionAffinityConfig) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SessionAffinityConfig: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SessionAffinityConfig: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClientIP", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ClientIP == nil { + m.ClientIP = &ClientIPConfig{} + } + if err := m.ClientIP.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *StorageOSPersistentVolumeSource) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -44761,726 +45922,739 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 11523 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x59, 0x90, 0x24, 0xc7, - 0x75, 0x18, 0xab, 0x7b, 0xae, 0x7e, 0x73, 0xe7, 0x1e, 0xe8, 0x1d, 0x00, 0xdb, 0x8b, 0x02, 0x09, - 0x2c, 0xae, 0x19, 0x61, 0x01, 0x90, 0x10, 0x01, 0x82, 0x9c, 0x99, 0x9e, 0xd9, 0x1d, 0xec, 0xd5, - 0xc8, 0x9e, 0x5d, 0x08, 0x20, 0x04, 0xb1, 0xb6, 0x2b, 0x67, 0xa6, 0x30, 0x35, 0x55, 0x8d, 0xaa, - 0xea, 0xd9, 0x1d, 0x84, 0x18, 0x61, 0xd3, 0x14, 0x7d, 0x50, 0x1f, 0x0a, 0x87, 0xc2, 0x96, 0x45, - 0x86, 0x1c, 0xe1, 0x23, 0x24, 0x9a, 0xb6, 0x43, 0x32, 0x65, 0x1d, 0xa4, 0x1c, 0xb6, 0xe5, 0x23, - 0xc8, 0x1f, 0x59, 0x52, 0x84, 0x83, 0x8c, 0x70, 0x78, 0x24, 0x0e, 0x23, 0xec, 0xf0, 0x87, 0x1d, - 0x3e, 0xbe, 0xb4, 0x96, 0x4d, 0x47, 0x9e, 0x95, 0x59, 0x5d, 0xd5, 0xdd, 0xb3, 0x98, 0x1d, 0x80, - 0x0c, 0xff, 0x75, 0xbf, 0xf7, 0xf2, 0x65, 0x56, 0x1e, 0x2f, 0xdf, 0x7b, 0xf9, 0xf2, 0x25, 0xbc, - 0xb4, 0xfd, 0x62, 0x3c, 0xef, 0x85, 0x0b, 0xdb, 0x9d, 0x5b, 0x24, 0x0a, 0x48, 0x42, 0xe2, 0x85, - 0x5d, 0x12, 0xb8, 0x61, 0xb4, 0x20, 0x10, 0x4e, 0xdb, 0x5b, 0x68, 0x85, 0x11, 0x59, 0xd8, 0x7d, - 0x76, 0x61, 0x93, 0x04, 0x24, 0x72, 0x12, 0xe2, 0xce, 0xb7, 0xa3, 0x30, 0x09, 0x11, 0xe2, 0x34, - 0xf3, 0x4e, 0xdb, 0x9b, 0xa7, 0x34, 0xf3, 0xbb, 0xcf, 0xce, 0x3d, 0xb3, 0xe9, 0x25, 0x5b, 0x9d, - 0x5b, 0xf3, 0xad, 0x70, 0x67, 0x61, 0x33, 0xdc, 0x0c, 0x17, 0x18, 0xe9, 0xad, 0xce, 0x06, 0xfb, - 0xc7, 0xfe, 0xb0, 0x5f, 0x9c, 0xc5, 0xdc, 0xf3, 0x69, 0x35, 0x3b, 0x4e, 0x6b, 0xcb, 0x0b, 0x48, - 0xb4, 0xb7, 0xd0, 0xde, 0xde, 0x64, 0xf5, 0x46, 0x24, 0x0e, 0x3b, 0x51, 0x8b, 0x64, 0x2b, 0xee, - 0x59, 0x2a, 0x5e, 0xd8, 0x21, 0x89, 0x93, 0xd3, 0xdc, 0xb9, 0x85, 0xa2, 0x52, 0x51, 0x27, 0x48, - 0xbc, 0x9d, 0xee, 0x6a, 0x3e, 0xde, 0xaf, 0x40, 0xdc, 0xda, 0x22, 0x3b, 0x4e, 0x57, 0xb9, 0xe7, - 0x8a, 0xca, 0x75, 0x12, 0xcf, 0x5f, 0xf0, 0x82, 0x24, 0x4e, 0xa2, 0x6c, 0x21, 0xfb, 0xbb, 0x16, - 0x9c, 0x5b, 0x7c, 0xbd, 0xb9, 0xe2, 0x3b, 0x71, 0xe2, 0xb5, 0x96, 0xfc, 0xb0, 0xb5, 0xdd, 0x4c, - 0xc2, 0x88, 0xdc, 0x0c, 0xfd, 0xce, 0x0e, 0x69, 0xb2, 0x8e, 0x40, 0x4f, 0xc3, 0xd8, 0x2e, 0xfb, - 0xbf, 0x56, 0xaf, 0x5a, 0xe7, 0xac, 0xf3, 0x95, 0xa5, 0x99, 0x6f, 0xef, 0xd7, 0x3e, 0x72, 0xb0, - 0x5f, 0x1b, 0xbb, 0x29, 0xe0, 0x58, 0x51, 0xa0, 0xc7, 0x60, 0x64, 0x23, 0x5e, 0xdf, 0x6b, 0x93, - 0x6a, 0x89, 0xd1, 0x4e, 0x09, 0xda, 0x91, 0xd5, 0x26, 0x85, 0x62, 0x81, 0x45, 0x0b, 0x50, 0x69, - 0x3b, 0x51, 0xe2, 0x25, 0x5e, 0x18, 0x54, 0xcb, 0xe7, 0xac, 0xf3, 0xc3, 0x4b, 0xb3, 0x82, 0xb4, - 0xd2, 0x90, 0x08, 0x9c, 0xd2, 0xd0, 0x66, 0x44, 0xc4, 0x71, 0xaf, 0x07, 0xfe, 0x5e, 0x75, 0xe8, - 0x9c, 0x75, 0x7e, 0x2c, 0x6d, 0x06, 0x16, 0x70, 0xac, 0x28, 0xec, 0x5f, 0x2e, 0xc1, 0xd8, 0xe2, - 0xc6, 0x86, 0x17, 0x78, 0xc9, 0x1e, 0xba, 0x09, 0x13, 0x41, 0xe8, 0x12, 0xf9, 0x9f, 0x7d, 0xc5, - 0xf8, 0x85, 0x73, 0xf3, 0xdd, 0x53, 0x69, 0xfe, 0x9a, 0x46, 0xb7, 0x34, 0x73, 0xb0, 0x5f, 0x9b, - 0xd0, 0x21, 0xd8, 0xe0, 0x83, 0x30, 0x8c, 0xb7, 0x43, 0x57, 0xb1, 0x2d, 0x31, 0xb6, 0xb5, 0x3c, - 0xb6, 0x8d, 0x94, 0x6c, 0x69, 0xfa, 0x60, 0xbf, 0x36, 0xae, 0x01, 0xb0, 0xce, 0x04, 0xdd, 0x82, - 0x69, 0xfa, 0x37, 0x48, 0x3c, 0xc5, 0xb7, 0xcc, 0xf8, 0x3e, 0x5a, 0xc4, 0x57, 0x23, 0x5d, 0x3a, - 0x71, 0xb0, 0x5f, 0x9b, 0xce, 0x00, 0x71, 0x96, 0xa1, 0xfd, 0x1e, 0x4c, 0x2d, 0x26, 0x89, 0xd3, - 0xda, 0x22, 0x2e, 0x1f, 0x41, 0xf4, 0x3c, 0x0c, 0x05, 0xce, 0x0e, 0x11, 0xe3, 0x7b, 0x4e, 0x74, - 0xec, 0xd0, 0x35, 0x67, 0x87, 0xdc, 0xdd, 0xaf, 0xcd, 0xdc, 0x08, 0xbc, 0x77, 0x3b, 0x62, 0x56, - 0x50, 0x18, 0x66, 0xd4, 0xe8, 0x02, 0x80, 0x4b, 0x76, 0xbd, 0x16, 0x69, 0x38, 0xc9, 0x96, 0x18, - 0x6f, 0x24, 0xca, 0x42, 0x5d, 0x61, 0xb0, 0x46, 0x65, 0xdf, 0x81, 0xca, 0xe2, 0x6e, 0xe8, 0xb9, - 0x8d, 0xd0, 0x8d, 0xd1, 0x36, 0x4c, 0xb7, 0x23, 0xb2, 0x41, 0x22, 0x05, 0xaa, 0x5a, 0xe7, 0xca, - 0xe7, 0xc7, 0x2f, 0x9c, 0xcf, 0xfd, 0x58, 0x93, 0x74, 0x25, 0x48, 0xa2, 0xbd, 0xa5, 0x07, 0x44, - 0x7d, 0xd3, 0x19, 0x2c, 0xce, 0x72, 0xb6, 0xff, 0x75, 0x09, 0x4e, 0x2d, 0xbe, 0xd7, 0x89, 0x48, - 0xdd, 0x8b, 0xb7, 0xb3, 0x33, 0xdc, 0xf5, 0xe2, 0xed, 0x6b, 0x69, 0x0f, 0xa8, 0xa9, 0x55, 0x17, - 0x70, 0xac, 0x28, 0xd0, 0x33, 0x30, 0x4a, 0x7f, 0xdf, 0xc0, 0x6b, 0xe2, 0x93, 0x4f, 0x08, 0xe2, - 0xf1, 0xba, 0x93, 0x38, 0x75, 0x8e, 0xc2, 0x92, 0x06, 0x5d, 0x85, 0xf1, 0x16, 0x5b, 0x90, 0x9b, - 0x57, 0x43, 0x97, 0xb0, 0xc1, 0xac, 0x2c, 0x3d, 0x45, 0xc9, 0x97, 0x53, 0xf0, 0xdd, 0xfd, 0x5a, - 0x95, 0xb7, 0x4d, 0xb0, 0xd0, 0x70, 0x58, 0x2f, 0x8f, 0x6c, 0xb5, 0xbe, 0x86, 0x18, 0x27, 0xc8, - 0x59, 0x5b, 0xe7, 0xb5, 0xa5, 0x32, 0xcc, 0x96, 0xca, 0x44, 0xfe, 0x32, 0x41, 0xcf, 0xc2, 0xd0, - 0xb6, 0x17, 0xb8, 0xd5, 0x11, 0xc6, 0xeb, 0x61, 0x3a, 0xe6, 0x97, 0xbd, 0xc0, 0xbd, 0xbb, 0x5f, - 0x9b, 0x35, 0x9a, 0x43, 0x81, 0x98, 0x91, 0xda, 0xff, 0xc0, 0x12, 0xdd, 0xb8, 0xea, 0xf9, 0xa6, - 0xa0, 0xb8, 0x00, 0x10, 0x93, 0x56, 0x44, 0x12, 0xad, 0x23, 0xd5, 0x74, 0x68, 0x2a, 0x0c, 0xd6, - 0xa8, 0xa8, 0x18, 0x88, 0xb7, 0x9c, 0x88, 0xcd, 0x2a, 0xd1, 0x9d, 0x4a, 0x0c, 0x34, 0x25, 0x02, - 0xa7, 0x34, 0x86, 0x18, 0x28, 0xf7, 0x15, 0x03, 0xbf, 0x63, 0xc1, 0xe8, 0x92, 0x17, 0xb8, 0x5e, - 0xb0, 0x89, 0x3e, 0x07, 0x63, 0x54, 0x4a, 0xbb, 0x4e, 0xe2, 0x08, 0x09, 0xf0, 0x13, 0xda, 0x2c, - 0x53, 0x42, 0x73, 0xbe, 0xbd, 0xbd, 0x49, 0x01, 0xf1, 0x3c, 0xa5, 0xa6, 0xf3, 0xee, 0xfa, 0xad, - 0x77, 0x48, 0x2b, 0xb9, 0x4a, 0x12, 0x27, 0xfd, 0x9c, 0x14, 0x86, 0x15, 0x57, 0x74, 0x19, 0x46, - 0x12, 0x27, 0xda, 0x24, 0x89, 0x10, 0x05, 0xb9, 0x4b, 0x96, 0x97, 0xc4, 0x74, 0x6e, 0x92, 0xa0, - 0x45, 0x52, 0x01, 0xb9, 0xce, 0x8a, 0x62, 0xc1, 0xc2, 0x6e, 0xc1, 0xc4, 0xb2, 0xd3, 0x76, 0x6e, - 0x79, 0xbe, 0x97, 0x78, 0x24, 0x46, 0x8f, 0x43, 0xd9, 0x71, 0x5d, 0xb6, 0x3e, 0x2a, 0x4b, 0xa7, - 0x0e, 0xf6, 0x6b, 0xe5, 0x45, 0x97, 0x0e, 0x14, 0x28, 0xaa, 0x3d, 0x4c, 0x29, 0xd0, 0x93, 0x30, - 0xe4, 0x46, 0x61, 0xbb, 0x5a, 0x62, 0x94, 0xa7, 0xe9, 0x98, 0xd6, 0xa3, 0xb0, 0x9d, 0x21, 0x65, - 0x34, 0xf6, 0xb7, 0x4a, 0x80, 0x96, 0x49, 0x7b, 0x6b, 0xb5, 0x69, 0x8c, 0xe4, 0x79, 0x18, 0xdb, - 0x09, 0x03, 0x2f, 0x09, 0xa3, 0x58, 0x54, 0xc8, 0x26, 0xd0, 0x55, 0x01, 0xc3, 0x0a, 0x8b, 0xce, - 0xc1, 0x50, 0x3b, 0x5d, 0xfc, 0x13, 0x52, 0x70, 0xb0, 0x65, 0xcf, 0x30, 0x94, 0xa2, 0x13, 0x93, - 0x48, 0x4c, 0x7c, 0x45, 0x71, 0x23, 0x26, 0x11, 0x66, 0x98, 0x74, 0xde, 0xd0, 0x19, 0x25, 0xa6, - 0x75, 0x66, 0xde, 0x50, 0x0c, 0xd6, 0xa8, 0xd0, 0x0d, 0xa8, 0xf0, 0x7f, 0x98, 0x6c, 0xb0, 0x39, - 0x5e, 0x20, 0x33, 0xae, 0x84, 0x2d, 0xc7, 0xcf, 0x76, 0xf9, 0x24, 0x9b, 0x5d, 0xb2, 0x38, 0x4e, - 0x39, 0x19, 0xb3, 0x6b, 0xa4, 0xef, 0xec, 0xfa, 0x25, 0x0b, 0xd0, 0xb2, 0x17, 0xb8, 0x24, 0x3a, - 0x86, 0x0d, 0xf3, 0x70, 0x13, 0xff, 0x3f, 0xd0, 0xa6, 0x85, 0x3b, 0xed, 0x30, 0x20, 0x41, 0xb2, - 0x1c, 0x06, 0x2e, 0xdf, 0x44, 0x3f, 0x09, 0x43, 0x09, 0xad, 0x8a, 0x37, 0xeb, 0x31, 0x39, 0x18, - 0xb4, 0x82, 0xbb, 0xfb, 0xb5, 0xd3, 0xdd, 0x25, 0x58, 0x13, 0x58, 0x19, 0xf4, 0x93, 0x30, 0x12, - 0x27, 0x4e, 0xd2, 0x89, 0x45, 0x43, 0x1f, 0x91, 0x0d, 0x6d, 0x32, 0xe8, 0xdd, 0xfd, 0xda, 0xb4, - 0x2a, 0xc6, 0x41, 0x58, 0x14, 0x40, 0x4f, 0xc0, 0xe8, 0x0e, 0x89, 0x63, 0x67, 0x53, 0xca, 0xbf, - 0x69, 0x51, 0x76, 0xf4, 0x2a, 0x07, 0x63, 0x89, 0x47, 0x8f, 0xc2, 0x30, 0x89, 0xa2, 0x30, 0x12, - 0xf3, 0x60, 0x52, 0x10, 0x0e, 0xaf, 0x50, 0x20, 0xe6, 0x38, 0xfb, 0xdf, 0x59, 0x30, 0xad, 0xda, - 0xca, 0xeb, 0x3a, 0x86, 0xe5, 0xfd, 0x26, 0x40, 0x4b, 0x7e, 0x60, 0xcc, 0x96, 0xd7, 0xf8, 0x85, - 0xc7, 0xf2, 0x26, 0x5d, 0x77, 0x37, 0xa6, 0x9c, 0x15, 0x28, 0xc6, 0x1a, 0x37, 0xfb, 0x9f, 0x59, - 0x70, 0x22, 0xf3, 0x45, 0x57, 0xbc, 0x38, 0x41, 0x6f, 0x75, 0x7d, 0xd5, 0xfc, 0x60, 0x5f, 0x45, - 0x4b, 0xb3, 0x6f, 0x52, 0xb3, 0x44, 0x42, 0xb4, 0x2f, 0xba, 0x04, 0xc3, 0x5e, 0x42, 0x76, 0xe4, - 0xc7, 0x3c, 0xda, 0xf3, 0x63, 0x78, 0xab, 0xd2, 0x11, 0x59, 0xa3, 0x25, 0x31, 0x67, 0x60, 0xff, - 0x0f, 0x0b, 0x2a, 0xcb, 0x61, 0xb0, 0xe1, 0x6d, 0x5e, 0x75, 0xda, 0xc7, 0x30, 0x16, 0x6b, 0x30, - 0xc4, 0xb8, 0xf3, 0x86, 0x3f, 0x9e, 0xdf, 0x70, 0xd1, 0x9c, 0x79, 0xba, 0x8b, 0x71, 0x6d, 0x41, - 0x89, 0x1f, 0x0a, 0xc2, 0x8c, 0xc5, 0xdc, 0x27, 0xa0, 0xa2, 0x08, 0xd0, 0x0c, 0x94, 0xb7, 0x09, - 0xd7, 0x10, 0x2b, 0x98, 0xfe, 0x44, 0x27, 0x61, 0x78, 0xd7, 0xf1, 0x3b, 0x62, 0x79, 0x62, 0xfe, - 0xe7, 0x93, 0xa5, 0x17, 0x2d, 0xfb, 0x9b, 0x6c, 0x8d, 0x89, 0x4a, 0x56, 0x82, 0x5d, 0xb1, 0xfc, - 0xdf, 0x83, 0x93, 0x7e, 0x8e, 0xd4, 0x11, 0x1d, 0x31, 0xb8, 0x94, 0x7a, 0x48, 0xb4, 0xf5, 0x64, - 0x1e, 0x16, 0xe7, 0xd6, 0x41, 0x05, 0x77, 0xd8, 0xa6, 0x33, 0xca, 0xf1, 0x59, 0x7b, 0xc5, 0xce, - 0x7f, 0x5d, 0xc0, 0xb0, 0xc2, 0x52, 0x01, 0x71, 0x52, 0x35, 0xfe, 0x32, 0xd9, 0x6b, 0x12, 0x9f, - 0xb4, 0x92, 0x30, 0xfa, 0x40, 0x9b, 0xff, 0x30, 0xef, 0x7d, 0x2e, 0x5f, 0xc6, 0x05, 0x83, 0xf2, - 0x65, 0xb2, 0xc7, 0x87, 0x42, 0xff, 0xba, 0x72, 0xcf, 0xaf, 0xfb, 0x0d, 0x0b, 0x26, 0xd5, 0xd7, - 0x1d, 0xc3, 0x42, 0x5a, 0x32, 0x17, 0xd2, 0xc3, 0x3d, 0xe7, 0x63, 0xc1, 0x12, 0xfa, 0x21, 0x13, - 0x01, 0x82, 0xa6, 0x11, 0x85, 0xb4, 0x6b, 0xa8, 0xcc, 0xfe, 0x20, 0x07, 0x64, 0x90, 0xef, 0xba, - 0x4c, 0xf6, 0xd6, 0x43, 0xba, 0xe1, 0xe7, 0x7f, 0x97, 0x31, 0x6a, 0x43, 0x3d, 0x47, 0xed, 0x37, - 0x4b, 0x70, 0x4a, 0xf5, 0x80, 0xb1, 0xa5, 0xfe, 0xa8, 0xf7, 0xc1, 0xb3, 0x30, 0xee, 0x92, 0x0d, - 0xa7, 0xe3, 0x27, 0xca, 0x08, 0x18, 0xe6, 0x86, 0x60, 0x3d, 0x05, 0x63, 0x9d, 0xe6, 0x10, 0xdd, - 0xf6, 0x6f, 0x80, 0xc9, 0xde, 0xc4, 0xa1, 0x33, 0x98, 0xea, 0x5b, 0x9a, 0x29, 0x37, 0xa1, 0x9b, - 0x72, 0xc2, 0x6c, 0x7b, 0x14, 0x86, 0xbd, 0x1d, 0xba, 0x17, 0x97, 0xcc, 0x2d, 0x76, 0x8d, 0x02, - 0x31, 0xc7, 0xa1, 0x8f, 0xc1, 0x68, 0x2b, 0xdc, 0xd9, 0x71, 0x02, 0xb7, 0x5a, 0x66, 0x1a, 0xe0, - 0x38, 0xdd, 0xae, 0x97, 0x39, 0x08, 0x4b, 0x1c, 0x7a, 0x08, 0x86, 0x9c, 0x68, 0x33, 0xae, 0x0e, - 0x31, 0x9a, 0x31, 0x5a, 0xd3, 0x62, 0xb4, 0x19, 0x63, 0x06, 0xa5, 0x9a, 0xdd, 0xed, 0x30, 0xda, - 0xf6, 0x82, 0xcd, 0xba, 0x17, 0x31, 0x35, 0x4d, 0xd3, 0xec, 0x5e, 0x57, 0x18, 0xac, 0x51, 0xa1, - 0x55, 0x18, 0x6e, 0x87, 0x51, 0x12, 0x57, 0x47, 0x58, 0x77, 0x3f, 0x52, 0xb0, 0x94, 0xf8, 0xd7, - 0x36, 0xc2, 0x28, 0x49, 0x3f, 0x80, 0xfe, 0x8b, 0x31, 0x2f, 0x8e, 0x7e, 0x12, 0xca, 0x24, 0xd8, - 0xad, 0x8e, 0x32, 0x2e, 0x73, 0x79, 0x5c, 0x56, 0x82, 0xdd, 0x9b, 0x4e, 0x94, 0xca, 0x99, 0x95, - 0x60, 0x17, 0xd3, 0x32, 0xe8, 0x0d, 0xa8, 0x48, 0x37, 0x50, 0x5c, 0x1d, 0x2b, 0x9e, 0x62, 0x58, - 0x10, 0x61, 0xf2, 0x6e, 0xc7, 0x8b, 0xc8, 0x0e, 0x09, 0x92, 0x38, 0x35, 0x5f, 0x24, 0x36, 0xc6, - 0x29, 0x37, 0xf4, 0x06, 0x4c, 0x70, 0xcd, 0xef, 0x6a, 0xd8, 0x09, 0x92, 0xb8, 0x5a, 0x61, 0xcd, - 0xcb, 0xf5, 0x19, 0xdc, 0x4c, 0xe9, 0x96, 0x4e, 0x0a, 0xa6, 0x13, 0x1a, 0x30, 0xc6, 0x06, 0x2b, - 0x84, 0x61, 0xd2, 0xf7, 0x76, 0x49, 0x40, 0xe2, 0xb8, 0x11, 0x85, 0xb7, 0x48, 0x15, 0x58, 0xcb, - 0xcf, 0xe4, 0x9b, 0xd2, 0xe1, 0x2d, 0xb2, 0x34, 0x7b, 0xb0, 0x5f, 0x9b, 0xbc, 0xa2, 0x97, 0xc1, - 0x26, 0x0b, 0x74, 0x03, 0xa6, 0xa8, 0x4a, 0xe9, 0xa5, 0x4c, 0xc7, 0xfb, 0x31, 0x45, 0x07, 0xfb, - 0xb5, 0x29, 0x6c, 0x14, 0xc2, 0x19, 0x26, 0xe8, 0x55, 0xa8, 0xf8, 0xde, 0x06, 0x69, 0xed, 0xb5, - 0x7c, 0x52, 0x9d, 0x60, 0x1c, 0x73, 0x97, 0xd5, 0x15, 0x49, 0xc4, 0x55, 0x76, 0xf5, 0x17, 0xa7, - 0xc5, 0xd1, 0x4d, 0x38, 0x9d, 0x90, 0x68, 0xc7, 0x0b, 0x1c, 0xba, 0x1c, 0x84, 0x3e, 0xc9, 0x1c, - 0x12, 0x93, 0x6c, 0xbe, 0x9d, 0x15, 0x5d, 0x77, 0x7a, 0x3d, 0x97, 0x0a, 0x17, 0x94, 0x46, 0xd7, - 0x61, 0x9a, 0xad, 0x84, 0x46, 0xc7, 0xf7, 0x1b, 0xa1, 0xef, 0xb5, 0xf6, 0xaa, 0x53, 0x8c, 0xe1, - 0xc7, 0xa4, 0xc7, 0x61, 0xcd, 0x44, 0x53, 0x03, 0x2b, 0xfd, 0x87, 0xb3, 0xa5, 0xd1, 0x2d, 0x98, - 0x8e, 0x49, 0xab, 0x13, 0x79, 0xc9, 0x1e, 0x9d, 0xbf, 0xe4, 0x4e, 0x52, 0x9d, 0x2e, 0x36, 0x13, - 0x9b, 0x26, 0x29, 0xf7, 0xec, 0x64, 0x80, 0x38, 0xcb, 0x90, 0x2e, 0xed, 0x38, 0x71, 0xbd, 0xa0, - 0x3a, 0xc3, 0x24, 0x86, 0x5a, 0x19, 0x4d, 0x0a, 0xc4, 0x1c, 0xc7, 0x6c, 0x6e, 0xfa, 0xe3, 0x3a, - 0x95, 0xa0, 0xb3, 0x8c, 0x30, 0xb5, 0xb9, 0x25, 0x02, 0xa7, 0x34, 0x74, 0x5b, 0x4e, 0x92, 0xbd, - 0x2a, 0x62, 0xa4, 0x6a, 0xb9, 0xac, 0xaf, 0xbf, 0x81, 0x29, 0x1c, 0x5d, 0x81, 0x51, 0x12, 0xec, - 0xae, 0x46, 0xe1, 0x4e, 0xf5, 0x44, 0xf1, 0x9a, 0x5d, 0xe1, 0x24, 0x5c, 0xa0, 0xa7, 0x06, 0x80, - 0x00, 0x63, 0xc9, 0x02, 0xdd, 0x81, 0x6a, 0xce, 0x88, 0xf0, 0x01, 0x38, 0xc9, 0x06, 0xe0, 0x65, - 0x51, 0xb6, 0xba, 0x5e, 0x40, 0x77, 0xb7, 0x07, 0x0e, 0x17, 0x72, 0xb7, 0x6f, 0xc1, 0x94, 0x12, - 0x2c, 0x6c, 0x6c, 0x51, 0x0d, 0x86, 0xa9, 0xc4, 0x94, 0x46, 0x70, 0x85, 0x76, 0x25, 0x15, 0xa4, - 0x31, 0xe6, 0x70, 0xd6, 0x95, 0xde, 0x7b, 0x64, 0x69, 0x2f, 0x21, 0xdc, 0x2c, 0x2a, 0x6b, 0x5d, - 0x29, 0x11, 0x38, 0xa5, 0xb1, 0xff, 0x2f, 0x57, 0x4c, 0x52, 0xe9, 0x35, 0x80, 0xbc, 0x7e, 0x1a, - 0xc6, 0xb6, 0xc2, 0x38, 0xa1, 0xd4, 0xac, 0x8e, 0xe1, 0x54, 0x15, 0xb9, 0x24, 0xe0, 0x58, 0x51, - 0xa0, 0x97, 0x60, 0xb2, 0xa5, 0x57, 0x20, 0x36, 0x9b, 0x53, 0xa2, 0x88, 0x59, 0x3b, 0x36, 0x69, - 0xd1, 0x8b, 0x30, 0xc6, 0x3c, 0xc3, 0xad, 0xd0, 0x17, 0x06, 0x98, 0xdc, 0x31, 0xc7, 0x1a, 0x02, - 0x7e, 0x57, 0xfb, 0x8d, 0x15, 0x35, 0x35, 0x63, 0x69, 0x13, 0xd6, 0x1a, 0x42, 0xcc, 0x2b, 0x33, - 0xf6, 0x12, 0x83, 0x62, 0x81, 0xb5, 0xff, 0x7a, 0x49, 0xeb, 0x65, 0x6a, 0x52, 0x10, 0xd4, 0x80, - 0xd1, 0xdb, 0x8e, 0x97, 0x78, 0xc1, 0xa6, 0xd8, 0xcf, 0x9f, 0xe8, 0x29, 0xf3, 0x59, 0xa1, 0xd7, - 0x79, 0x01, 0xbe, 0x2b, 0x89, 0x3f, 0x58, 0xb2, 0xa1, 0x1c, 0xa3, 0x4e, 0x10, 0x50, 0x8e, 0xa5, - 0x41, 0x39, 0x62, 0x5e, 0x80, 0x73, 0x14, 0x7f, 0xb0, 0x64, 0x83, 0xde, 0x02, 0x90, 0xf3, 0x86, - 0xb8, 0xc2, 0x23, 0xfb, 0x74, 0x7f, 0xa6, 0xeb, 0xaa, 0xcc, 0xd2, 0x14, 0xdd, 0xf3, 0xd2, 0xff, - 0x58, 0xe3, 0x67, 0x27, 0x4c, 0xef, 0xe9, 0x6e, 0x0c, 0xfa, 0x2c, 0x5d, 0xaa, 0x4e, 0x94, 0x10, - 0x77, 0x31, 0x11, 0x9d, 0xf3, 0xe4, 0x60, 0x6a, 0xeb, 0xba, 0xb7, 0x43, 0xf4, 0x65, 0x2d, 0x98, - 0xe0, 0x94, 0x9f, 0xfd, 0xdb, 0x65, 0xa8, 0x16, 0x35, 0x97, 0x4e, 0x3a, 0x72, 0xc7, 0x4b, 0x96, - 0xa9, 0xba, 0x62, 0x99, 0x93, 0x6e, 0x45, 0xc0, 0xb1, 0xa2, 0xa0, 0xa3, 0x1f, 0x7b, 0x9b, 0xd2, - 0xea, 0x18, 0x4e, 0x47, 0xbf, 0xc9, 0xa0, 0x58, 0x60, 0x29, 0x5d, 0x44, 0x9c, 0x58, 0xb8, 0xfc, - 0xb5, 0x59, 0x82, 0x19, 0x14, 0x0b, 0xac, 0xee, 0x30, 0x18, 0xea, 0xe3, 0x30, 0x30, 0xba, 0x68, - 0xf8, 0x68, 0xbb, 0x08, 0xbd, 0x0d, 0xb0, 0xe1, 0x05, 0x5e, 0xbc, 0xc5, 0xb8, 0x8f, 0x1c, 0x9a, - 0xbb, 0x52, 0x76, 0x56, 0x15, 0x17, 0xac, 0x71, 0x44, 0x2f, 0xc0, 0xb8, 0x5a, 0x80, 0x6b, 0xf5, - 0xea, 0xa8, 0xe9, 0x4f, 0x4e, 0xa5, 0x51, 0x1d, 0xeb, 0x74, 0xf6, 0x3b, 0xd9, 0xf9, 0x22, 0x56, - 0x80, 0xd6, 0xbf, 0xd6, 0xa0, 0xfd, 0x5b, 0xea, 0xdd, 0xbf, 0xf6, 0xef, 0x97, 0x61, 0xda, 0xa8, - 0xac, 0x13, 0x0f, 0x20, 0xb3, 0x2e, 0xd2, 0x8d, 0xc8, 0x49, 0x88, 0x58, 0x7f, 0x76, 0xff, 0xa5, - 0xa2, 0x6f, 0x56, 0x74, 0x05, 0xf0, 0xf2, 0xe8, 0x6d, 0xa8, 0xf8, 0x4e, 0xcc, 0x9c, 0x0f, 0x44, - 0xac, 0xbb, 0x41, 0x98, 0xa5, 0x8a, 0xbe, 0x13, 0x27, 0xda, 0x5e, 0xc0, 0x79, 0xa7, 0x2c, 0xe9, - 0x8e, 0x49, 0x95, 0x13, 0x79, 0xa6, 0xa4, 0x1a, 0x41, 0x35, 0x98, 0x3d, 0xcc, 0x71, 0xe8, 0x45, - 0x98, 0x88, 0x08, 0x9b, 0x15, 0xcb, 0x54, 0xd7, 0x62, 0xd3, 0x6c, 0x38, 0x55, 0xca, 0xb0, 0x86, - 0xc3, 0x06, 0x65, 0xaa, 0x6b, 0x8f, 0xf4, 0xd0, 0xb5, 0x9f, 0x80, 0x51, 0xf6, 0x43, 0xcd, 0x00, - 0x35, 0x1a, 0x6b, 0x1c, 0x8c, 0x25, 0x3e, 0x3b, 0x61, 0xc6, 0x06, 0x9c, 0x30, 0x4f, 0xc2, 0x54, - 0xdd, 0x21, 0x3b, 0x61, 0xb0, 0x12, 0xb8, 0xed, 0xd0, 0x0b, 0x12, 0x54, 0x85, 0x21, 0xb6, 0x3b, - 0xf0, 0xb5, 0x3d, 0x44, 0x39, 0xe0, 0x21, 0xaa, 0x39, 0xdb, 0x7f, 0x54, 0x82, 0xc9, 0x3a, 0xf1, - 0x49, 0x42, 0xb8, 0xad, 0x11, 0xa3, 0x55, 0x40, 0x9b, 0x91, 0xd3, 0x22, 0x0d, 0x12, 0x79, 0xa1, - 0xdb, 0x24, 0xad, 0x30, 0x60, 0x27, 0x35, 0x74, 0xbb, 0x3b, 0x7d, 0xb0, 0x5f, 0x43, 0x17, 0xbb, - 0xb0, 0x38, 0xa7, 0x04, 0x7a, 0x13, 0x26, 0xdb, 0x11, 0x31, 0x7c, 0x68, 0x56, 0x91, 0xba, 0xd0, - 0xd0, 0x09, 0xb9, 0xa6, 0x6a, 0x80, 0xb0, 0xc9, 0x0a, 0x7d, 0x06, 0x66, 0xc2, 0xa8, 0xbd, 0xe5, - 0x04, 0x75, 0xd2, 0x26, 0x81, 0x4b, 0x55, 0x71, 0xe1, 0x23, 0x38, 0x79, 0xb0, 0x5f, 0x9b, 0xb9, - 0x9e, 0xc1, 0xe1, 0x2e, 0x6a, 0xf4, 0x26, 0xcc, 0xb6, 0xa3, 0xb0, 0xed, 0x6c, 0xb2, 0x89, 0x22, - 0x34, 0x0e, 0x2e, 0x7d, 0x9e, 0x3e, 0xd8, 0xaf, 0xcd, 0x36, 0xb2, 0xc8, 0xbb, 0xfb, 0xb5, 0x13, - 0xac, 0xa3, 0x28, 0x24, 0x45, 0xe2, 0x6e, 0x36, 0xf6, 0x26, 0x9c, 0xaa, 0x87, 0xb7, 0x83, 0xdb, - 0x4e, 0xe4, 0x2e, 0x36, 0xd6, 0x34, 0xe3, 0xfe, 0x9a, 0x34, 0x2e, 0xf9, 0xb9, 0x57, 0xee, 0x3e, - 0xa5, 0x95, 0xe4, 0xea, 0xff, 0xaa, 0xe7, 0x93, 0x02, 0x27, 0xc2, 0xdf, 0x2c, 0x19, 0x35, 0xa5, - 0xf4, 0xca, 0x53, 0x6f, 0x15, 0x7a, 0xea, 0x5f, 0x83, 0xb1, 0x0d, 0x8f, 0xf8, 0x2e, 0x26, 0x1b, - 0x62, 0x64, 0x1e, 0x2f, 0x3e, 0xc0, 0x58, 0xa5, 0x94, 0xd2, 0x69, 0xc4, 0x4d, 0xd3, 0x55, 0x51, - 0x18, 0x2b, 0x36, 0x68, 0x1b, 0x66, 0xa4, 0xed, 0x23, 0xb1, 0x62, 0x11, 0x3f, 0xd1, 0xcb, 0xa0, - 0x32, 0x99, 0xb3, 0x01, 0xc4, 0x19, 0x36, 0xb8, 0x8b, 0x31, 0xb5, 0x45, 0x77, 0xe8, 0x76, 0x35, - 0xc4, 0xa6, 0x34, 0xb3, 0x45, 0x99, 0x59, 0xcd, 0xa0, 0xf6, 0x57, 0x2d, 0x78, 0xa0, 0xab, 0x67, - 0x84, 0x7b, 0xe1, 0x88, 0x47, 0x21, 0x6b, 0xee, 0x97, 0xfa, 0x9b, 0xfb, 0xf6, 0xaf, 0x5b, 0x70, - 0x72, 0x65, 0xa7, 0x9d, 0xec, 0xd5, 0x3d, 0xf3, 0x34, 0xe1, 0x13, 0x30, 0xb2, 0x43, 0x5c, 0xaf, - 0xb3, 0x23, 0x46, 0xae, 0x26, 0x45, 0xfa, 0x55, 0x06, 0xbd, 0xbb, 0x5f, 0x9b, 0x6c, 0x26, 0x61, - 0xe4, 0x6c, 0x12, 0x0e, 0xc0, 0x82, 0x1c, 0xfd, 0x0c, 0xd7, 0x4d, 0xaf, 0x78, 0x3b, 0x9e, 0x3c, - 0x90, 0xea, 0xe9, 0xf2, 0x9a, 0x97, 0x1d, 0x3a, 0xff, 0x5a, 0xc7, 0x09, 0x12, 0x2f, 0xd9, 0x33, - 0x75, 0x59, 0xc6, 0x08, 0xa7, 0x3c, 0xed, 0xef, 0x5a, 0x30, 0x2d, 0xe5, 0xc9, 0xa2, 0xeb, 0x46, - 0x24, 0x8e, 0xd1, 0x1c, 0x94, 0xbc, 0xb6, 0x68, 0x29, 0x88, 0xd2, 0xa5, 0xb5, 0x06, 0x2e, 0x79, - 0x6d, 0xd4, 0x80, 0x0a, 0x3f, 0xdb, 0x4a, 0x27, 0xd8, 0x40, 0x27, 0x64, 0xcc, 0xf6, 0x5b, 0x97, - 0x25, 0x71, 0xca, 0x44, 0x6a, 0xc6, 0x6c, 0x2f, 0x2a, 0x9b, 0x27, 0x2d, 0x97, 0x04, 0x1c, 0x2b, - 0x0a, 0x74, 0x1e, 0xc6, 0x82, 0xd0, 0xe5, 0x47, 0x8d, 0x7c, 0x5d, 0xb3, 0x69, 0x7b, 0x4d, 0xc0, - 0xb0, 0xc2, 0xda, 0x3f, 0x6f, 0xc1, 0x84, 0xfc, 0xb2, 0x01, 0x95, 0x74, 0xba, 0xbc, 0x52, 0x05, - 0x3d, 0x5d, 0x5e, 0x54, 0xc9, 0x66, 0x18, 0x43, 0xb7, 0x2e, 0x1f, 0x46, 0xb7, 0xb6, 0xbf, 0x52, - 0x82, 0x29, 0xd9, 0x9c, 0x66, 0xe7, 0x56, 0x4c, 0x12, 0xb4, 0x0e, 0x15, 0x87, 0x77, 0x39, 0x91, - 0xb3, 0xf6, 0xd1, 0x7c, 0xab, 0xcb, 0x18, 0x9f, 0x74, 0x44, 0x17, 0x65, 0x69, 0x9c, 0x32, 0x42, - 0x3e, 0xcc, 0x06, 0x61, 0xc2, 0xb6, 0x3e, 0x85, 0xef, 0x75, 0x36, 0x90, 0xe5, 0x7e, 0x46, 0x70, - 0x9f, 0xbd, 0x96, 0xe5, 0x82, 0xbb, 0x19, 0xa3, 0x15, 0xe9, 0xe9, 0x29, 0xb3, 0x1a, 0xce, 0xf5, - 0xaa, 0xa1, 0xd8, 0xd1, 0x63, 0xff, 0x9e, 0x05, 0x15, 0x49, 0x76, 0x1c, 0xc7, 0x40, 0x57, 0x61, - 0x34, 0x66, 0x83, 0x20, 0xbb, 0xc6, 0xee, 0xd5, 0x70, 0x3e, 0x5e, 0xe9, 0x8e, 0xce, 0xff, 0xc7, - 0x58, 0xf2, 0x60, 0xae, 0x6a, 0xd5, 0xfc, 0x0f, 0x89, 0xab, 0x5a, 0xb5, 0xa7, 0x60, 0x97, 0xf9, - 0xcf, 0xac, 0xcd, 0x9a, 0x3d, 0x4f, 0x15, 0xcf, 0x76, 0x44, 0x36, 0xbc, 0x3b, 0x59, 0xc5, 0xb3, - 0xc1, 0xa0, 0x58, 0x60, 0xd1, 0x5b, 0x30, 0xd1, 0x92, 0x1e, 0xde, 0x54, 0x0c, 0x3c, 0xd6, 0xd3, - 0x5f, 0xae, 0x8e, 0x56, 0x78, 0x40, 0xce, 0xb2, 0x56, 0x1e, 0x1b, 0xdc, 0xa8, 0x84, 0x49, 0x4f, - 0x85, 0xcb, 0x3d, 0x9d, 0x2b, 0x11, 0x49, 0x52, 0xbe, 0x85, 0x07, 0xc2, 0xf6, 0xaf, 0x58, 0x30, - 0xc2, 0xfd, 0x84, 0x83, 0x39, 0x56, 0xb5, 0xa3, 0xa2, 0xb4, 0xef, 0x6e, 0x52, 0xa0, 0x38, 0x39, - 0x42, 0x57, 0xa1, 0xc2, 0x7e, 0x30, 0x7f, 0x49, 0xb9, 0x38, 0x12, 0x89, 0xd7, 0xaa, 0x37, 0xf0, - 0xa6, 0x2c, 0x86, 0x53, 0x0e, 0xf6, 0x2f, 0x96, 0xa9, 0xa8, 0x4a, 0x49, 0x8d, 0x5d, 0xdc, 0xba, - 0x7f, 0xbb, 0x78, 0xe9, 0x7e, 0xed, 0xe2, 0x9b, 0x30, 0xdd, 0xd2, 0xce, 0xa5, 0xd2, 0x91, 0x3c, - 0xdf, 0x73, 0x92, 0x68, 0x47, 0x58, 0xdc, 0x57, 0xb6, 0x6c, 0x32, 0xc1, 0x59, 0xae, 0xe8, 0xb3, - 0x30, 0xc1, 0xc7, 0x59, 0xd4, 0x32, 0xc4, 0x6a, 0xf9, 0x58, 0xf1, 0x7c, 0xd1, 0xab, 0x60, 0x33, - 0xb1, 0xa9, 0x15, 0xc7, 0x06, 0x33, 0xfb, 0x4b, 0xc3, 0x30, 0xbc, 0xb2, 0x4b, 0x82, 0xe4, 0x18, - 0x04, 0x52, 0x0b, 0xa6, 0xbc, 0x60, 0x37, 0xf4, 0x77, 0x89, 0xcb, 0xf1, 0x87, 0xd9, 0x5c, 0x4f, - 0x0b, 0xd6, 0x53, 0x6b, 0x06, 0x0b, 0x9c, 0x61, 0x79, 0x3f, 0x2c, 0xf7, 0x8b, 0x30, 0xc2, 0xc7, - 0x5e, 0x98, 0xed, 0xb9, 0x5e, 0x70, 0xd6, 0x89, 0x62, 0x15, 0xa4, 0x5e, 0x05, 0xee, 0x76, 0x17, - 0xc5, 0xd1, 0x3b, 0x30, 0xb5, 0xe1, 0x45, 0x71, 0x42, 0x4d, 0xee, 0x38, 0x71, 0x76, 0xda, 0xf7, - 0x60, 0xa9, 0xab, 0x7e, 0x58, 0x35, 0x38, 0xe1, 0x0c, 0x67, 0xb4, 0x09, 0x93, 0xd4, 0x78, 0x4c, - 0xab, 0x1a, 0x3d, 0x74, 0x55, 0xca, 0x15, 0x77, 0x45, 0x67, 0x84, 0x4d, 0xbe, 0x54, 0x98, 0xb4, - 0x98, 0xb1, 0x39, 0xc6, 0x34, 0x0a, 0x25, 0x4c, 0xb8, 0x95, 0xc9, 0x71, 0x54, 0x26, 0xb1, 0x78, - 0x8e, 0x8a, 0x29, 0x93, 0xd2, 0xa8, 0x0d, 0xfb, 0x6b, 0x74, 0x77, 0xa4, 0x7d, 0x78, 0x0c, 0x5b, - 0xcb, 0x2b, 0xe6, 0xd6, 0x72, 0xa6, 0x70, 0x3c, 0x0b, 0xb6, 0x95, 0xcf, 0xc1, 0xb8, 0x36, 0xdc, - 0x68, 0x01, 0x2a, 0x2d, 0x19, 0x7c, 0x20, 0xa4, 0xae, 0x52, 0x5f, 0x54, 0x54, 0x02, 0x4e, 0x69, - 0x68, 0x6f, 0x50, 0x65, 0x2f, 0x1b, 0x8c, 0x44, 0x55, 0x41, 0xcc, 0x30, 0xf6, 0x73, 0x00, 0x2b, - 0x77, 0x48, 0x6b, 0x91, 0x1b, 0x5f, 0xda, 0x19, 0x97, 0x55, 0x7c, 0xc6, 0x65, 0xff, 0xb1, 0x05, - 0x53, 0xab, 0xcb, 0x86, 0x52, 0x3e, 0x0f, 0xc0, 0xb5, 0xd0, 0xd7, 0x5f, 0xbf, 0x26, 0xbd, 0xc3, - 0xdc, 0xc1, 0xa7, 0xa0, 0x58, 0xa3, 0x40, 0x67, 0xa0, 0xec, 0x77, 0x02, 0xa1, 0x1c, 0x8e, 0x1e, - 0xec, 0xd7, 0xca, 0x57, 0x3a, 0x01, 0xa6, 0x30, 0x2d, 0xfe, 0xa7, 0x3c, 0x70, 0xfc, 0x4f, 0xdf, - 0xf8, 0x57, 0x54, 0x83, 0xe1, 0xdb, 0xb7, 0x3d, 0x37, 0xae, 0x0e, 0xa7, 0x9e, 0xeb, 0xd7, 0x5f, - 0x5f, 0xab, 0xc7, 0x98, 0xc3, 0xed, 0xbf, 0x58, 0x86, 0x99, 0x55, 0x9f, 0xdc, 0x31, 0x3e, 0xeb, - 0x31, 0x18, 0x71, 0x23, 0x6f, 0x97, 0x44, 0xd9, 0x5d, 0xbc, 0xce, 0xa0, 0x58, 0x60, 0x07, 0x8e, - 0x59, 0xba, 0xd1, 0xbd, 0x1f, 0x1f, 0x75, 0x94, 0x56, 0xff, 0xae, 0x78, 0x0b, 0x46, 0xf9, 0x51, - 0x29, 0xef, 0x8c, 0xf1, 0x0b, 0xcf, 0xe6, 0x35, 0x21, 0xdb, 0x17, 0xf3, 0xc2, 0xf9, 0xc1, 0xe3, - 0x46, 0x94, 0x10, 0x13, 0x50, 0x2c, 0x59, 0xce, 0x7d, 0x12, 0x26, 0x74, 0xca, 0x43, 0x05, 0x90, - 0xfc, 0x25, 0x0b, 0x4e, 0xac, 0xfa, 0x61, 0x6b, 0x3b, 0x13, 0x40, 0xf6, 0x02, 0x8c, 0xd3, 0xf5, - 0x14, 0x1b, 0x91, 0x94, 0x46, 0x94, 0xa9, 0x40, 0x61, 0x9d, 0x4e, 0x2b, 0x76, 0xe3, 0xc6, 0x5a, - 0x3d, 0x2f, 0x38, 0x55, 0xa0, 0xb0, 0x4e, 0x67, 0xff, 0x81, 0x05, 0x0f, 0x5f, 0x5c, 0x5e, 0x69, - 0x90, 0x28, 0xf6, 0xe2, 0x84, 0x04, 0x49, 0x57, 0x7c, 0x2c, 0x55, 0xee, 0x5c, 0xad, 0x29, 0xa9, - 0x72, 0x57, 0x67, 0xad, 0x10, 0xd8, 0x0f, 0x4b, 0xec, 0xf7, 0xaf, 0x59, 0x70, 0xe2, 0xa2, 0x97, - 0x60, 0xd2, 0x0e, 0xb3, 0xf1, 0xa9, 0x11, 0x69, 0x87, 0xb1, 0x97, 0x84, 0xd1, 0x5e, 0x36, 0x3e, - 0x15, 0x2b, 0x0c, 0xd6, 0xa8, 0x78, 0xcd, 0xbb, 0x5e, 0x4c, 0x5b, 0x5a, 0x32, 0x2d, 0x4c, 0x2c, - 0xe0, 0x58, 0x51, 0xd0, 0x0f, 0x73, 0xbd, 0x88, 0x69, 0x08, 0x7b, 0x62, 0x39, 0xab, 0x0f, 0xab, - 0x4b, 0x04, 0x4e, 0x69, 0xec, 0xaf, 0x5a, 0x70, 0xea, 0xa2, 0xdf, 0x89, 0x13, 0x12, 0x6d, 0xc4, - 0x46, 0x63, 0x9f, 0x83, 0x0a, 0x91, 0x5a, 0xb8, 0x68, 0xab, 0xda, 0x37, 0x94, 0x7a, 0xce, 0x83, - 0x63, 0x15, 0xdd, 0x00, 0xd1, 0x98, 0x87, 0x8b, 0x22, 0xfc, 0x46, 0x09, 0x26, 0x2f, 0xad, 0xaf, - 0x37, 0x2e, 0x92, 0x44, 0x88, 0xcc, 0xfe, 0x5e, 0x24, 0xac, 0x19, 0xc2, 0xbd, 0x74, 0x9d, 0x4e, - 0xe2, 0xf9, 0xf3, 0xfc, 0x5e, 0xc2, 0xfc, 0x5a, 0x90, 0x5c, 0x8f, 0x9a, 0x49, 0xe4, 0x05, 0x9b, - 0xb9, 0xa6, 0xb3, 0x14, 0xec, 0xe5, 0x22, 0xc1, 0x8e, 0x9e, 0x83, 0x11, 0x76, 0x31, 0x42, 0x6a, - 0x1d, 0x0f, 0x2a, 0x55, 0x81, 0x41, 0xef, 0xee, 0xd7, 0x2a, 0x37, 0xf0, 0x1a, 0xff, 0x83, 0x05, - 0x29, 0xba, 0x01, 0xe3, 0x5b, 0x49, 0xd2, 0xbe, 0x44, 0x1c, 0x97, 0x44, 0x52, 0x3a, 0x9c, 0xcd, - 0x93, 0x0e, 0xb4, 0x13, 0x38, 0x59, 0xba, 0xa0, 0x52, 0x58, 0x8c, 0x75, 0x3e, 0x76, 0x13, 0x20, - 0xc5, 0x1d, 0x91, 0xd9, 0x60, 0xff, 0xc0, 0x82, 0xd1, 0x4b, 0x4e, 0xe0, 0xfa, 0x24, 0x42, 0x2f, - 0xc3, 0x10, 0xb9, 0x43, 0x5a, 0x62, 0x07, 0xcf, 0x6d, 0x70, 0xba, 0xcb, 0x71, 0x47, 0x18, 0xfd, - 0x8f, 0x59, 0x29, 0x74, 0x09, 0x46, 0x69, 0x6b, 0x2f, 0xaa, 0x30, 0xe5, 0x47, 0x8a, 0xbe, 0x58, - 0x0d, 0x3b, 0xdf, 0x18, 0x05, 0x08, 0xcb, 0xe2, 0xcc, 0xa1, 0xd3, 0x6a, 0x37, 0xa9, 0x00, 0x4b, - 0x7a, 0x99, 0x5b, 0xeb, 0xcb, 0x0d, 0x4e, 0x24, 0xb8, 0x71, 0x87, 0x8e, 0x04, 0xe2, 0x94, 0x89, - 0xbd, 0x0e, 0x15, 0x3a, 0xa8, 0x8b, 0xbe, 0xe7, 0xf4, 0xf6, 0x25, 0x3d, 0x05, 0x15, 0xe9, 0xd7, - 0x89, 0x45, 0xa4, 0x33, 0xe3, 0x2a, 0xdd, 0x3e, 0x31, 0x4e, 0xf1, 0xf6, 0x8b, 0x70, 0x92, 0x1d, - 0x94, 0x3a, 0xc9, 0x96, 0xb1, 0xc6, 0xfa, 0x4e, 0x66, 0xfb, 0xeb, 0x43, 0x30, 0xbb, 0xd6, 0x5c, - 0x6e, 0x9a, 0xee, 0xc2, 0x17, 0x61, 0x82, 0xef, 0xed, 0x74, 0x8a, 0x3a, 0xbe, 0x28, 0xaf, 0x8e, - 0x03, 0xd6, 0x35, 0x1c, 0x36, 0x28, 0xd1, 0xc3, 0x50, 0xf6, 0xde, 0x0d, 0xb2, 0x01, 0x6e, 0x6b, - 0xaf, 0x5d, 0xc3, 0x14, 0x4e, 0xd1, 0x54, 0x4d, 0xe0, 0x22, 0x51, 0xa1, 0x95, 0xaa, 0xf0, 0x0a, - 0x4c, 0x79, 0x71, 0x2b, 0xf6, 0xd6, 0x02, 0x2a, 0x2f, 0x9c, 0x96, 0x9c, 0xec, 0xa9, 0x0e, 0x4f, - 0x9b, 0xaa, 0xb0, 0x38, 0x43, 0xad, 0xc9, 0xe7, 0xe1, 0x81, 0x55, 0x8d, 0xbe, 0x51, 0xd0, 0x54, - 0x8b, 0x6a, 0xb3, 0xaf, 0x8b, 0x59, 0xb0, 0x8d, 0xd0, 0xa2, 0xf8, 0x07, 0xc7, 0x58, 0xe2, 0xd0, - 0x45, 0x98, 0x6d, 0x6d, 0x39, 0xed, 0xc5, 0x4e, 0xb2, 0x55, 0xf7, 0xe2, 0x56, 0xb8, 0x4b, 0xa2, - 0x3d, 0xa6, 0xdb, 0x8e, 0xa5, 0x6e, 0x23, 0x85, 0x58, 0xbe, 0xb4, 0xd8, 0xa0, 0x94, 0xb8, 0xbb, - 0x8c, 0xa9, 0x54, 0xc0, 0x91, 0x29, 0x15, 0x8b, 0x30, 0x2d, 0xeb, 0x6a, 0x92, 0x98, 0x09, 0xfc, - 0x71, 0xd6, 0x3a, 0x75, 0xc3, 0x44, 0x80, 0x55, 0xdb, 0xb2, 0xf4, 0xf6, 0x3b, 0x50, 0x51, 0x81, - 0x60, 0x32, 0x96, 0xd1, 0x2a, 0x88, 0x65, 0xec, 0x2f, 0xaa, 0xa5, 0x3b, 0xbb, 0x9c, 0xeb, 0xce, - 0xfe, 0x5b, 0x16, 0xa4, 0xf1, 0x30, 0xe8, 0x12, 0x54, 0xda, 0x21, 0x3b, 0xd2, 0x8a, 0xe4, 0x39, - 0xf1, 0x83, 0xb9, 0xab, 0x9a, 0x4b, 0x10, 0xde, 0x0d, 0x0d, 0x59, 0x02, 0xa7, 0x85, 0xd1, 0x12, - 0x8c, 0xb6, 0x23, 0xd2, 0x4c, 0xd8, 0x05, 0x82, 0xbe, 0x7c, 0xf8, 0x50, 0x73, 0x7a, 0x2c, 0x0b, - 0xda, 0xbf, 0x69, 0x01, 0x70, 0x6f, 0xb1, 0x13, 0x6c, 0x92, 0x63, 0xb0, 0x80, 0xeb, 0x30, 0x14, - 0xb7, 0x49, 0xab, 0xd7, 0x61, 0x63, 0xda, 0x9e, 0x66, 0x9b, 0xb4, 0xd2, 0x0e, 0xa7, 0xff, 0x30, - 0x2b, 0x6d, 0xff, 0x1c, 0xc0, 0x54, 0x4a, 0x46, 0x2d, 0x13, 0xf4, 0x8c, 0x11, 0x2f, 0x7f, 0x26, - 0x13, 0x2f, 0x5f, 0x61, 0xd4, 0x5a, 0x88, 0xfc, 0x3b, 0x50, 0xde, 0x71, 0xee, 0x08, 0xf3, 0xe7, - 0xa9, 0xde, 0xcd, 0xa0, 0xfc, 0xe7, 0xaf, 0x3a, 0x77, 0xb8, 0x82, 0xf9, 0x94, 0x9c, 0x20, 0x57, - 0x9d, 0x3b, 0x77, 0xf9, 0x91, 0x22, 0x93, 0x35, 0xd4, 0xca, 0xfa, 0xc2, 0x9f, 0xa4, 0xff, 0xd9, - 0xb6, 0x41, 0x2b, 0x61, 0x75, 0x79, 0x81, 0xf0, 0x9d, 0x0e, 0x54, 0x97, 0x17, 0x64, 0xeb, 0xf2, - 0x82, 0x01, 0xea, 0xf2, 0x02, 0xf4, 0x1e, 0x8c, 0x8a, 0xb3, 0x0a, 0x16, 0xe8, 0x37, 0x7e, 0x61, - 0x61, 0x80, 0xfa, 0xc4, 0x51, 0x07, 0xaf, 0x73, 0x41, 0x2a, 0xd0, 0x02, 0xda, 0xb7, 0x5e, 0x59, - 0x21, 0xfa, 0x1b, 0x16, 0x4c, 0x89, 0xdf, 0x98, 0xbc, 0xdb, 0x21, 0x71, 0x22, 0x36, 0xea, 0x8f, - 0x0f, 0xde, 0x06, 0x51, 0x90, 0x37, 0xe5, 0xe3, 0x52, 0x5a, 0x9a, 0xc8, 0xbe, 0x2d, 0xca, 0xb4, - 0x02, 0xfd, 0x63, 0x0b, 0x4e, 0xee, 0x38, 0x77, 0x78, 0x8d, 0x1c, 0x86, 0x9d, 0xc4, 0x0b, 0x45, - 0xe0, 0xe2, 0xcb, 0x83, 0x0d, 0x7f, 0x57, 0x71, 0xde, 0x48, 0x19, 0xe3, 0x74, 0x32, 0x8f, 0xa4, - 0x6f, 0x53, 0x73, 0xdb, 0x35, 0xb7, 0x01, 0x63, 0x72, 0xbe, 0xe5, 0x98, 0x29, 0x75, 0x5d, 0x0b, - 0x39, 0xf4, 0x51, 0x91, 0x66, 0xd6, 0xb0, 0x7a, 0xc4, 0x5c, 0xbb, 0xaf, 0xf5, 0xbc, 0x03, 0x13, - 0xfa, 0x1c, 0xbb, 0xaf, 0x75, 0xbd, 0x0b, 0x27, 0x72, 0xe6, 0xd2, 0x7d, 0xad, 0xf2, 0x36, 0x9c, - 0x29, 0x9c, 0x1f, 0xf7, 0xb3, 0x62, 0xfb, 0x1b, 0x96, 0x2e, 0x07, 0x8f, 0xc1, 0x6f, 0xb4, 0x6c, - 0xfa, 0x8d, 0xce, 0xf6, 0x5e, 0x39, 0x05, 0xce, 0xa3, 0xb7, 0xf4, 0x46, 0x53, 0xa9, 0x8e, 0x5e, - 0x85, 0x11, 0x9f, 0x42, 0xe4, 0x01, 0x99, 0xdd, 0x7f, 0x45, 0xa6, 0x2a, 0x11, 0x83, 0xc7, 0x58, - 0x70, 0xb0, 0x7f, 0xc7, 0x82, 0xa1, 0x63, 0xe8, 0x09, 0x6c, 0xf6, 0xc4, 0x33, 0x85, 0xac, 0xc5, - 0x6d, 0xf0, 0x79, 0xec, 0xdc, 0x5e, 0xb9, 0x93, 0x90, 0x20, 0x66, 0x7a, 0x75, 0x6e, 0xc7, 0xfc, - 0x9f, 0x12, 0x8c, 0xd3, 0xaa, 0x64, 0x34, 0xc7, 0x4b, 0x30, 0xe9, 0x3b, 0xb7, 0x88, 0x2f, 0x7d, - 0xd9, 0x59, 0xeb, 0xf2, 0x8a, 0x8e, 0xc4, 0x26, 0x2d, 0x2d, 0xbc, 0xa1, 0xbb, 0xf5, 0x85, 0xfe, - 0xa2, 0x0a, 0x1b, 0x3e, 0x7f, 0x6c, 0xd2, 0x52, 0x43, 0xe7, 0xb6, 0x93, 0xb4, 0xb6, 0x84, 0xe5, - 0xa9, 0x9a, 0xfb, 0x3a, 0x05, 0x62, 0x8e, 0xa3, 0x7a, 0x98, 0x9c, 0x9d, 0x37, 0x49, 0xc4, 0xf4, - 0x30, 0xae, 0xe5, 0x2a, 0x3d, 0x0c, 0x9b, 0x68, 0x9c, 0xa5, 0x47, 0x9f, 0x84, 0x29, 0xda, 0x39, - 0x61, 0x27, 0x91, 0xb1, 0x2a, 0xc3, 0x2c, 0x56, 0x85, 0x85, 0x26, 0xaf, 0x1b, 0x18, 0x9c, 0xa1, - 0x44, 0x0d, 0x38, 0xe9, 0x05, 0x2d, 0xbf, 0xe3, 0x92, 0x1b, 0x81, 0x17, 0x78, 0x89, 0xe7, 0xf8, - 0xde, 0x7b, 0xc4, 0x15, 0x7a, 0xb0, 0x0a, 0x2b, 0x5a, 0xcb, 0xa1, 0xc1, 0xb9, 0x25, 0xed, 0x9f, - 0x81, 0x13, 0x57, 0x42, 0xc7, 0x5d, 0x72, 0x7c, 0x27, 0x68, 0x91, 0x68, 0x2d, 0xd8, 0xec, 0x7b, - 0x52, 0xae, 0x9f, 0x6b, 0x97, 0xfa, 0x9d, 0x6b, 0xdb, 0x5b, 0x80, 0xf4, 0x0a, 0x44, 0x8c, 0x16, - 0x86, 0x51, 0x8f, 0x57, 0x25, 0xa6, 0xff, 0xe3, 0xf9, 0x4a, 0x72, 0x57, 0xcb, 0xb4, 0xe8, 0x23, - 0x0e, 0xc0, 0x92, 0x11, 0x35, 0xa4, 0xf2, 0xb4, 0xea, 0xfe, 0x36, 0xae, 0xfd, 0x02, 0xcc, 0xb2, - 0x92, 0x87, 0xb4, 0xbf, 0xfe, 0xaa, 0x05, 0xd3, 0xd7, 0x32, 0x97, 0x53, 0x1f, 0x83, 0x91, 0x98, - 0x44, 0x39, 0x4e, 0xca, 0x26, 0x83, 0x62, 0x81, 0x3d, 0x72, 0x67, 0xc8, 0x0f, 0x2d, 0xa8, 0xb0, - 0xe0, 0xdf, 0x36, 0xb5, 0xa5, 0xee, 0xbf, 0x52, 0xbb, 0x6c, 0x28, 0xb5, 0xb9, 0x46, 0xba, 0x6a, - 0x4e, 0x91, 0x4e, 0x8b, 0x2e, 0xab, 0x4b, 0x9b, 0x3d, 0xec, 0xf3, 0x94, 0x0d, 0xbf, 0xe2, 0x37, - 0x65, 0xde, 0xec, 0x94, 0xd7, 0x38, 0xd9, 0x51, 0xb5, 0xa2, 0xfd, 0x90, 0x1c, 0x55, 0xab, 0xf6, - 0x14, 0x48, 0xbf, 0x86, 0xd6, 0x64, 0xb6, 0x2b, 0x7c, 0x9a, 0x85, 0x74, 0xb2, 0xb5, 0xa9, 0x6e, - 0x37, 0xd7, 0x44, 0x88, 0xa6, 0x80, 0xde, 0x65, 0x82, 0x4c, 0xfc, 0xe3, 0x57, 0xd6, 0xd3, 0x22, - 0xf6, 0x25, 0x98, 0xce, 0x74, 0x18, 0x7a, 0x01, 0x86, 0xdb, 0x5b, 0x4e, 0x4c, 0x32, 0x21, 0x3a, - 0xc3, 0x0d, 0x0a, 0xbc, 0xbb, 0x5f, 0x9b, 0x52, 0x05, 0x18, 0x04, 0x73, 0x6a, 0xfb, 0xbf, 0x5b, - 0x30, 0x74, 0x2d, 0x74, 0x8f, 0x63, 0x32, 0xbd, 0x62, 0x4c, 0xa6, 0x87, 0x8a, 0x52, 0x5f, 0x14, - 0xce, 0xa3, 0xd5, 0xcc, 0x3c, 0x3a, 0x5b, 0xc8, 0xa1, 0xf7, 0x14, 0xda, 0x81, 0x71, 0x96, 0x50, - 0x43, 0x84, 0x0b, 0x3d, 0x67, 0xd8, 0x57, 0xb5, 0x8c, 0x7d, 0x35, 0xad, 0x91, 0x6a, 0x56, 0xd6, - 0x13, 0x30, 0x2a, 0x42, 0x56, 0xb2, 0xc1, 0xab, 0x82, 0x16, 0x4b, 0xbc, 0xfd, 0x2b, 0x65, 0x30, - 0x12, 0x78, 0xa0, 0xdf, 0xb3, 0x60, 0x3e, 0xe2, 0xd7, 0x75, 0xdc, 0x7a, 0x27, 0xf2, 0x82, 0xcd, - 0x66, 0x6b, 0x8b, 0xb8, 0x1d, 0xdf, 0x0b, 0x36, 0xd7, 0x36, 0x83, 0x50, 0x81, 0x57, 0xee, 0x90, - 0x56, 0x87, 0x39, 0xa8, 0xfb, 0x64, 0x0b, 0x51, 0x47, 0xc2, 0x17, 0x0e, 0xf6, 0x6b, 0xf3, 0xf8, - 0x50, 0xbc, 0xf1, 0x21, 0xdb, 0x82, 0xfe, 0xc0, 0x82, 0x05, 0x9e, 0xd7, 0x62, 0xf0, 0xf6, 0xf7, - 0xb0, 0x46, 0x1b, 0x92, 0x55, 0xca, 0x64, 0x9d, 0x44, 0x3b, 0x4b, 0x9f, 0x10, 0x1d, 0xba, 0xd0, - 0x38, 0x5c, 0x5d, 0xf8, 0xb0, 0x8d, 0xb3, 0xff, 0x65, 0x19, 0x26, 0x69, 0x2f, 0xa6, 0x57, 0xd4, - 0x5f, 0x30, 0xa6, 0xc4, 0x23, 0x99, 0x29, 0x31, 0x6b, 0x10, 0x1f, 0xcd, 0xed, 0xf4, 0x18, 0x66, - 0x7d, 0x27, 0x4e, 0x2e, 0x11, 0x27, 0x4a, 0x6e, 0x11, 0x87, 0x9d, 0xc1, 0x8a, 0x69, 0x7e, 0x98, - 0x63, 0x5d, 0xe5, 0xc5, 0xba, 0x92, 0x65, 0x86, 0xbb, 0xf9, 0xa3, 0x5d, 0x40, 0xec, 0xbc, 0x37, - 0x72, 0x82, 0x98, 0x7f, 0x8b, 0x27, 0x9c, 0xd7, 0x87, 0xab, 0x75, 0x4e, 0xd4, 0x8a, 0xae, 0x74, - 0x71, 0xc3, 0x39, 0x35, 0x68, 0xe7, 0xf8, 0xc3, 0x83, 0x9e, 0xe3, 0x8f, 0xf4, 0x89, 0x10, 0xdf, - 0x81, 0x19, 0x31, 0x2a, 0x1b, 0xde, 0xa6, 0xd8, 0xa4, 0xdf, 0xc8, 0xc4, 0xf9, 0x58, 0x83, 0x47, - 0x24, 0xf4, 0x09, 0xf2, 0xb1, 0x7f, 0x16, 0x4e, 0xd0, 0xea, 0xcc, 0x78, 0xe6, 0x18, 0x11, 0x98, - 0xde, 0xee, 0xdc, 0x22, 0x3e, 0x49, 0x24, 0x4c, 0x54, 0x9a, 0xab, 0xf6, 0x9b, 0xa5, 0x53, 0xdd, - 0xf2, 0xb2, 0xc9, 0x02, 0x67, 0x79, 0xda, 0xbf, 0x6a, 0x01, 0x8b, 0x18, 0x3c, 0x86, 0xed, 0xef, - 0x53, 0xe6, 0xf6, 0x57, 0x2d, 0x92, 0x40, 0x05, 0x3b, 0xdf, 0xf3, 0x7c, 0x58, 0x1a, 0x51, 0x78, - 0x67, 0x4f, 0xea, 0xfe, 0xfd, 0x35, 0xae, 0xff, 0x6d, 0xf1, 0x05, 0xa9, 0x6e, 0x2f, 0xa2, 0xcf, - 0xc3, 0x58, 0xcb, 0x69, 0x3b, 0x2d, 0x9e, 0x39, 0xa9, 0xd0, 0xfb, 0x63, 0x14, 0x9a, 0x5f, 0x16, - 0x25, 0xb8, 0x37, 0xe3, 0x27, 0xe4, 0x57, 0x4a, 0x70, 0x5f, 0x0f, 0x86, 0xaa, 0x72, 0x6e, 0x1b, - 0x26, 0x0d, 0x66, 0xf7, 0xd5, 0xf4, 0xfd, 0x3c, 0xdf, 0x2e, 0x94, 0xc5, 0xb2, 0x03, 0xb3, 0x81, - 0xf6, 0x9f, 0x0a, 0x47, 0xa9, 0x4e, 0x7f, 0xb4, 0xdf, 0x86, 0xc0, 0x24, 0xa9, 0x16, 0x11, 0x99, - 0x61, 0x83, 0xbb, 0x39, 0xdb, 0x7f, 0xc7, 0x82, 0x07, 0x74, 0x42, 0xed, 0x62, 0x69, 0x3f, 0x7f, - 0x72, 0x1d, 0xc6, 0xc2, 0x36, 0x89, 0x9c, 0xd4, 0x26, 0x3b, 0x2f, 0x3b, 0xfd, 0xba, 0x80, 0xdf, - 0xdd, 0xaf, 0x9d, 0xd4, 0xb9, 0x4b, 0x38, 0x56, 0x25, 0x91, 0x0d, 0x23, 0xac, 0x33, 0x62, 0x71, - 0xe9, 0x97, 0x65, 0x17, 0x62, 0xe7, 0x50, 0x31, 0x16, 0x18, 0xfb, 0xe7, 0x2c, 0x3e, 0xb1, 0xf4, - 0xa6, 0xa3, 0x77, 0x61, 0x66, 0x87, 0x9a, 0x6f, 0x2b, 0x77, 0xda, 0x11, 0xf7, 0x86, 0xcb, 0x7e, - 0x7a, 0xaa, 0x5f, 0x3f, 0x69, 0x1f, 0xb9, 0x54, 0x15, 0x6d, 0x9e, 0xb9, 0x9a, 0x61, 0x86, 0xbb, - 0xd8, 0xdb, 0x7f, 0x56, 0xe2, 0x2b, 0x91, 0x69, 0x75, 0x4f, 0xc0, 0x68, 0x3b, 0x74, 0x97, 0xd7, - 0xea, 0x58, 0xf4, 0x90, 0x12, 0x57, 0x0d, 0x0e, 0xc6, 0x12, 0x8f, 0x2e, 0x00, 0x90, 0x3b, 0x09, - 0x89, 0x02, 0xc7, 0x57, 0xa7, 0xe4, 0x4a, 0x79, 0x5a, 0x51, 0x18, 0xac, 0x51, 0xd1, 0x32, 0xed, - 0x28, 0xdc, 0xf5, 0x5c, 0x76, 0xeb, 0xa2, 0x6c, 0x96, 0x69, 0x28, 0x0c, 0xd6, 0xa8, 0xa8, 0xa9, - 0xdc, 0x09, 0x62, 0xbe, 0x01, 0x3a, 0xb7, 0x44, 0x66, 0x9b, 0xb1, 0xd4, 0x54, 0xbe, 0xa1, 0x23, - 0xb1, 0x49, 0x8b, 0x16, 0x61, 0x24, 0x71, 0xd8, 0xd9, 0xef, 0x70, 0x71, 0x2c, 0xcd, 0x3a, 0xa5, - 0xd0, 0x13, 0x08, 0xd1, 0x02, 0x58, 0x14, 0x44, 0x6f, 0x4a, 0x11, 0xcc, 0x45, 0xb2, 0x88, 0x89, - 0x2a, 0x9c, 0xb6, 0xba, 0xf8, 0xd6, 0x65, 0xb0, 0x88, 0xb5, 0x32, 0x78, 0xd9, 0x5f, 0xac, 0x00, - 0xa4, 0xda, 0x1e, 0x7a, 0xaf, 0x4b, 0x44, 0x3c, 0xdd, 0x5b, 0x3f, 0x3c, 0x3a, 0xf9, 0x80, 0xbe, - 0x64, 0xc1, 0xb8, 0xe3, 0xfb, 0x61, 0xcb, 0x49, 0x58, 0x2f, 0x97, 0x7a, 0x8b, 0x28, 0x51, 0xff, - 0x62, 0x5a, 0x82, 0x37, 0xe1, 0x39, 0x79, 0xac, 0xab, 0x61, 0xfa, 0xb6, 0x42, 0xaf, 0x18, 0xfd, - 0x84, 0x34, 0x02, 0xf8, 0xf4, 0x98, 0xcb, 0x1a, 0x01, 0x15, 0x26, 0x8d, 0x35, 0xfd, 0x1f, 0xdd, - 0x30, 0x12, 0xca, 0x0c, 0x15, 0xdf, 0x9d, 0x35, 0x94, 0x9e, 0x7e, 0xb9, 0x64, 0x50, 0x43, 0x8f, - 0x0d, 0x1f, 0x2e, 0xbe, 0x60, 0xae, 0x69, 0xd7, 0x7d, 0xe2, 0xc2, 0xdf, 0x81, 0x69, 0xd7, 0xdc, - 0x6e, 0xc5, 0x6c, 0x7a, 0xbc, 0x88, 0x6f, 0x66, 0x77, 0x4e, 0x37, 0xd8, 0x0c, 0x02, 0x67, 0x19, - 0xa3, 0x06, 0x8f, 0xd2, 0x5f, 0x0b, 0x36, 0x42, 0x11, 0x5b, 0x67, 0x17, 0x8e, 0xe5, 0x5e, 0x9c, - 0x90, 0x1d, 0x4a, 0x99, 0xee, 0xa3, 0xd7, 0x44, 0x59, 0xac, 0xb8, 0xa0, 0x57, 0x61, 0x84, 0x5d, - 0x9f, 0x8a, 0xab, 0x63, 0xc5, 0x7e, 0x40, 0xf3, 0xe6, 0x6f, 0xba, 0xa8, 0xd8, 0xdf, 0x18, 0x0b, - 0x0e, 0xe8, 0x92, 0xbc, 0xbf, 0x1f, 0xaf, 0x05, 0x37, 0x62, 0xc2, 0xee, 0xef, 0x57, 0x96, 0x3e, - 0x9a, 0x5e, 0xcd, 0xe7, 0xf0, 0xdc, 0xa4, 0x79, 0x46, 0x49, 0xaa, 0xaf, 0x88, 0xff, 0x32, 0x17, - 0x5f, 0x15, 0x8a, 0x9b, 0x67, 0xe6, 0xeb, 0x4b, 0xbb, 0xf3, 0xa6, 0xc9, 0x02, 0x67, 0x79, 0x1e, - 0xeb, 0xf6, 0x39, 0x17, 0xc0, 0x4c, 0x76, 0x61, 0xdd, 0xd7, 0xed, 0xfa, 0x07, 0x43, 0x30, 0x65, - 0x4e, 0x04, 0xb4, 0x00, 0x15, 0xc1, 0x44, 0x65, 0xdf, 0x52, 0x73, 0xfb, 0xaa, 0x44, 0xe0, 0x94, - 0x86, 0x65, 0x1f, 0x63, 0xc5, 0xb5, 0xa0, 0xa9, 0x34, 0xfb, 0x98, 0xc2, 0x60, 0x8d, 0x8a, 0x2a, - 0xd1, 0xb7, 0xc2, 0x30, 0x51, 0x5b, 0x81, 0x9a, 0x2d, 0x4b, 0x0c, 0x8a, 0x05, 0x96, 0x6e, 0x01, - 0xdb, 0x24, 0x0a, 0x88, 0x6f, 0x7a, 0x32, 0xd5, 0x16, 0x70, 0x59, 0x47, 0x62, 0x93, 0x96, 0x6e, - 0x69, 0x61, 0xcc, 0xa6, 0x9f, 0x50, 0xd5, 0xd3, 0x20, 0xb4, 0x26, 0xbf, 0x3e, 0x28, 0xf1, 0xe8, - 0x0d, 0x78, 0x40, 0xdd, 0xf6, 0xc3, 0xdc, 0x33, 0x2c, 0x6b, 0x1c, 0x31, 0x2c, 0xeb, 0x07, 0x96, - 0xf3, 0xc9, 0x70, 0x51, 0x79, 0xf4, 0x0a, 0x4c, 0x09, 0x15, 0x58, 0x72, 0x1c, 0x35, 0x63, 0x0e, - 0x2e, 0x1b, 0x58, 0x9c, 0xa1, 0x46, 0x75, 0x98, 0xa1, 0x10, 0xa6, 0x85, 0x4a, 0x0e, 0xfc, 0xd6, - 0xa2, 0xda, 0xeb, 0x2f, 0x67, 0xf0, 0xb8, 0xab, 0x04, 0x5a, 0x84, 0x69, 0xae, 0xa3, 0x50, 0x9b, - 0x92, 0x8d, 0x83, 0x08, 0x79, 0x55, 0x0b, 0xe1, 0xba, 0x89, 0xc6, 0x59, 0x7a, 0xf4, 0x22, 0x4c, - 0x38, 0x51, 0x6b, 0xcb, 0x4b, 0x48, 0x2b, 0xe9, 0x44, 0x3c, 0x3b, 0x86, 0x16, 0xb4, 0xb1, 0xa8, - 0xe1, 0xb0, 0x41, 0x69, 0xbf, 0x07, 0x27, 0x72, 0xa2, 0xe5, 0xe9, 0xc4, 0x71, 0xda, 0x9e, 0xfc, - 0xa6, 0x4c, 0x38, 0xd9, 0x62, 0x63, 0x4d, 0x7e, 0x8d, 0x46, 0x45, 0x67, 0x27, 0x73, 0x89, 0x6b, - 0x09, 0x33, 0xd5, 0xec, 0x5c, 0x95, 0x08, 0x9c, 0xd2, 0xd8, 0xdf, 0x01, 0xd0, 0x1c, 0x3a, 0x03, - 0x04, 0x13, 0xbd, 0x08, 0x13, 0x32, 0xcb, 0xab, 0x96, 0x53, 0x51, 0x7d, 0xe6, 0x45, 0x0d, 0x87, - 0x0d, 0x4a, 0xda, 0xb6, 0x40, 0xba, 0xa9, 0xb2, 0xc1, 0x6b, 0xca, 0x7f, 0x85, 0x53, 0x1a, 0xf4, - 0x34, 0x8c, 0xc5, 0xc4, 0xdf, 0xb8, 0xe2, 0x05, 0xdb, 0x62, 0x62, 0x2b, 0x29, 0xdc, 0x14, 0x70, - 0xac, 0x28, 0xd0, 0x12, 0x94, 0x3b, 0x9e, 0x2b, 0xa6, 0xb2, 0xdc, 0xf0, 0xcb, 0x37, 0xd6, 0xea, - 0x77, 0xf7, 0x6b, 0x8f, 0x14, 0x25, 0xaf, 0xa5, 0xa6, 0x7d, 0x3c, 0x4f, 0x97, 0x1f, 0x2d, 0x9c, - 0x77, 0x36, 0x30, 0x72, 0xc8, 0xb3, 0x81, 0x0b, 0x00, 0xe2, 0xab, 0xe5, 0x5c, 0x2e, 0xa7, 0xa3, - 0x76, 0x51, 0x61, 0xb0, 0x46, 0x85, 0x62, 0x98, 0x6d, 0x45, 0xc4, 0x91, 0x36, 0x34, 0x8f, 0xfb, - 0x1e, 0xbb, 0x77, 0x07, 0xc1, 0x72, 0x96, 0x19, 0xee, 0xe6, 0x8f, 0x42, 0x98, 0x75, 0xc5, 0xe5, - 0xd2, 0xb4, 0xd2, 0xca, 0xe1, 0x83, 0xcd, 0x59, 0x5c, 0x4d, 0x96, 0x11, 0xee, 0xe6, 0x8d, 0xde, - 0x86, 0x39, 0x09, 0xec, 0xbe, 0xcf, 0xcb, 0x96, 0x4b, 0x79, 0xe9, 0xec, 0xc1, 0x7e, 0x6d, 0xae, - 0x5e, 0x48, 0x85, 0x7b, 0x70, 0x40, 0x18, 0x46, 0xd8, 0x59, 0x52, 0x5c, 0x1d, 0x67, 0xfb, 0xdc, - 0x93, 0xc5, 0xce, 0x00, 0x3a, 0xd7, 0xe7, 0xd9, 0x39, 0x94, 0x88, 0xbf, 0x4d, 0x8f, 0xe5, 0x18, - 0x10, 0x0b, 0x4e, 0x68, 0x03, 0xc6, 0x9d, 0x20, 0x08, 0x13, 0x87, 0xab, 0x50, 0x13, 0xc5, 0xba, - 0x9f, 0xc6, 0x78, 0x31, 0x2d, 0xc1, 0xb9, 0xab, 0x90, 0x3e, 0x0d, 0x83, 0x75, 0xc6, 0xe8, 0x36, - 0x4c, 0x87, 0xb7, 0xa9, 0x70, 0x94, 0x5e, 0x8a, 0xb8, 0x3a, 0xc9, 0xea, 0x7a, 0x7e, 0x40, 0x3f, - 0xad, 0x51, 0x58, 0x93, 0x5a, 0x26, 0x53, 0x9c, 0xad, 0x05, 0xcd, 0x1b, 0xde, 0xea, 0xa9, 0x34, - 0xd0, 0x3c, 0xf5, 0x56, 0xeb, 0xce, 0x69, 0x76, 0x3f, 0x9c, 0xc7, 0x93, 0xb2, 0xd5, 0x3f, 0x9d, - 0xb9, 0x1f, 0x9e, 0xa2, 0xb0, 0x4e, 0x87, 0xb6, 0x60, 0x22, 0x3d, 0xb2, 0x8a, 0x62, 0x96, 0x3e, - 0x66, 0xfc, 0xc2, 0x85, 0xc1, 0x3e, 0x6e, 0x4d, 0x2b, 0xc9, 0x2d, 0x07, 0x1d, 0x82, 0x0d, 0xce, - 0x73, 0x3f, 0x09, 0xe3, 0xda, 0xc0, 0x1e, 0x26, 0x5c, 0x7a, 0xee, 0x15, 0x98, 0xc9, 0x0e, 0xdd, - 0xa1, 0xc2, 0xad, 0xff, 0x67, 0x09, 0xa6, 0x73, 0x4e, 0xae, 0x58, 0x02, 0xdc, 0x8c, 0x40, 0x4d, - 0xf3, 0xdd, 0x9a, 0x62, 0xb1, 0x34, 0x80, 0x58, 0x94, 0x32, 0xba, 0x5c, 0x28, 0xa3, 0x85, 0x28, - 0x1c, 0x7a, 0x3f, 0xa2, 0xd0, 0xdc, 0x7d, 0x86, 0x07, 0xda, 0x7d, 0x8e, 0x40, 0x7c, 0x1a, 0x1b, - 0xd8, 0xe8, 0x00, 0x1b, 0xd8, 0x2f, 0x96, 0x60, 0x26, 0x0d, 0x2d, 0x17, 0xe9, 0xa6, 0xef, 0xff, - 0x79, 0xc7, 0xab, 0xc6, 0x79, 0x47, 0x7e, 0x3a, 0xe9, 0x4c, 0xab, 0x0a, 0xcf, 0x3e, 0x70, 0xe6, - 0xec, 0xe3, 0xc9, 0x81, 0xb8, 0xf5, 0x3e, 0x07, 0xf9, 0xbb, 0x25, 0x38, 0x95, 0x2d, 0xb2, 0xec, - 0x3b, 0xde, 0xce, 0x31, 0xf4, 0xcd, 0x75, 0xa3, 0x6f, 0x9e, 0x19, 0xe4, 0x6b, 0x58, 0xd3, 0x0a, - 0x3b, 0xe8, 0xf5, 0x4c, 0x07, 0x2d, 0x0c, 0xce, 0xb2, 0x77, 0x2f, 0x7d, 0xc7, 0x82, 0x33, 0xb9, - 0xe5, 0x8e, 0xc1, 0xfb, 0x7a, 0xcd, 0xf4, 0xbe, 0x3e, 0x31, 0xf0, 0x37, 0x15, 0xb8, 0x63, 0xbf, - 0x5a, 0x2e, 0xf8, 0x16, 0xe6, 0xbf, 0xba, 0x0e, 0xe3, 0x4e, 0xab, 0x45, 0xe2, 0xf8, 0x6a, 0xe8, - 0xaa, 0x7c, 0x53, 0xcf, 0xb0, 0x3d, 0x29, 0x05, 0xdf, 0xdd, 0xaf, 0xcd, 0x65, 0x59, 0xa4, 0x68, - 0xac, 0x73, 0x30, 0x73, 0xd8, 0x95, 0x8e, 0x34, 0x87, 0xdd, 0x05, 0x80, 0x5d, 0x65, 0xd5, 0x66, - 0x9d, 0x61, 0x9a, 0xbd, 0xab, 0x51, 0xa1, 0x9f, 0x66, 0xba, 0x22, 0x0f, 0x19, 0xe1, 0x87, 0x1c, - 0xcf, 0x0d, 0x38, 0x56, 0x7a, 0xf8, 0x09, 0xbf, 0xa1, 0xaa, 0x1c, 0x87, 0x8a, 0x25, 0xfa, 0x0c, - 0xcc, 0xc4, 0x3c, 0x09, 0xc2, 0xb2, 0xef, 0xc4, 0xec, 0x5e, 0x84, 0x90, 0x89, 0xec, 0xda, 0x69, - 0x33, 0x83, 0xc3, 0x5d, 0xd4, 0xf6, 0x3f, 0x2c, 0xc3, 0x83, 0x3d, 0xa6, 0x28, 0x5a, 0x34, 0x8f, - 0x78, 0x9f, 0xca, 0x7a, 0x77, 0xe6, 0x72, 0x0b, 0x1b, 0xee, 0x9e, 0xcc, 0x18, 0x97, 0xde, 0xf7, - 0x18, 0x7f, 0xd9, 0xd2, 0xfc, 0x6e, 0x3c, 0x10, 0xf4, 0x53, 0x87, 0x5c, 0x7a, 0x3f, 0xaa, 0x8e, - 0xfa, 0x2f, 0x58, 0xf0, 0x48, 0xee, 0x67, 0x19, 0xa1, 0x22, 0x0b, 0x50, 0x69, 0x51, 0xa0, 0x76, - 0x77, 0x29, 0xbd, 0x41, 0x28, 0x11, 0x38, 0xa5, 0x31, 0x22, 0x42, 0x4a, 0x7d, 0x23, 0x42, 0xfe, - 0xb9, 0x05, 0x27, 0xb3, 0x8d, 0x38, 0x06, 0xc9, 0xb4, 0x66, 0x4a, 0xa6, 0x8f, 0x0e, 0x32, 0xe4, - 0x05, 0x42, 0xe9, 0xdf, 0x4f, 0xc1, 0xe9, 0xae, 0x9d, 0x8b, 0xf7, 0xdd, 0x2e, 0xcc, 0x6e, 0x32, - 0x15, 0x5e, 0xbb, 0x15, 0x26, 0x3e, 0x26, 0xf7, 0x02, 0x5d, 0xcf, 0x2b, 0x64, 0xdc, 0x0c, 0xe9, - 0x22, 0xc1, 0xdd, 0x55, 0xa0, 0x2f, 0x58, 0x70, 0xd2, 0xb9, 0x1d, 0x77, 0xbd, 0x49, 0x22, 0xe6, - 0xcc, 0xf3, 0xb9, 0xde, 0xb1, 0x3e, 0x6f, 0x98, 0x2c, 0x55, 0x0f, 0xf6, 0x6b, 0x27, 0xf3, 0xa8, - 0x70, 0x6e, 0x5d, 0x08, 0x8b, 0x94, 0x7b, 0x54, 0xcb, 0xe9, 0x71, 0x6f, 0x31, 0xef, 0x56, 0x09, - 0x97, 0x51, 0x12, 0x83, 0x15, 0x1f, 0x74, 0x13, 0x2a, 0x9b, 0xf2, 0xaa, 0x97, 0x90, 0x81, 0xb9, - 0x9b, 0x4a, 0xee, 0x7d, 0x30, 0x1e, 0xb1, 0xaf, 0x50, 0x38, 0x65, 0x85, 0x5e, 0x81, 0x72, 0xb0, - 0x11, 0x8b, 0x3b, 0xd4, 0xf9, 0xf1, 0x3d, 0x66, 0x04, 0x15, 0xbf, 0x80, 0x7a, 0x6d, 0xb5, 0x89, - 0x69, 0x41, 0x5a, 0x3e, 0xba, 0xe5, 0x0a, 0x87, 0x6e, 0x6e, 0x79, 0xbc, 0x54, 0xef, 0x2e, 0x8f, - 0x97, 0xea, 0x98, 0x16, 0x44, 0xab, 0x30, 0xcc, 0xee, 0x99, 0x08, 0x6f, 0x6d, 0xee, 0x05, 0xfa, - 0xae, 0x3b, 0x34, 0xfc, 0x46, 0x2a, 0x03, 0x63, 0x5e, 0x1c, 0xbd, 0x0a, 0x23, 0x2d, 0x96, 0x4c, - 0x5f, 0x98, 0xd6, 0xf9, 0x49, 0x21, 0xba, 0xd2, 0xed, 0xf3, 0x33, 0x2a, 0x0e, 0xc7, 0x82, 0x03, - 0xe3, 0x45, 0xda, 0x5b, 0x1b, 0xb1, 0xb0, 0x98, 0xf3, 0x79, 0x75, 0x3d, 0x7c, 0x20, 0x78, 0x31, - 0x38, 0x16, 0x1c, 0xd0, 0x27, 0xa1, 0xb4, 0xd1, 0x12, 0x17, 0x4d, 0x72, 0x7d, 0xb3, 0xe6, 0xdd, - 0xe0, 0xa5, 0x91, 0x83, 0xfd, 0x5a, 0x69, 0x75, 0x19, 0x97, 0x36, 0x5a, 0xe8, 0x1a, 0x8c, 0x6e, - 0xf0, 0x0b, 0x9e, 0x22, 0x71, 0xea, 0xe3, 0xf9, 0x77, 0x4f, 0xbb, 0xee, 0x80, 0xf2, 0x9b, 0x15, - 0x02, 0x81, 0x25, 0x13, 0xb4, 0x0e, 0xb0, 0xa1, 0x2e, 0xaa, 0x8a, 0xcc, 0xa9, 0x1f, 0x1d, 0xe4, - 0x3a, 0xab, 0x30, 0x1a, 0x15, 0x14, 0x6b, 0x7c, 0xe8, 0xcc, 0x74, 0xe4, 0x8b, 0x1e, 0x2c, 0x6b, - 0x6a, 0xc1, 0xcc, 0xcc, 0x7d, 0xf6, 0x83, 0xcf, 0x4c, 0x85, 0xc2, 0x29, 0x2b, 0xb4, 0x0d, 0x93, - 0xbb, 0x71, 0x7b, 0x8b, 0xc8, 0xc5, 0xc8, 0x12, 0xa8, 0x9a, 0x66, 0x65, 0x9a, 0xed, 0x56, 0x10, - 0x7a, 0x51, 0xd2, 0x71, 0xfc, 0x2e, 0xf9, 0xc1, 0x12, 0x80, 0xdd, 0xd4, 0x99, 0x61, 0x93, 0x37, - 0xed, 0xea, 0x77, 0x3b, 0xe1, 0xad, 0xbd, 0x84, 0x88, 0xb4, 0xaa, 0xb9, 0x5d, 0xfd, 0x1a, 0x27, - 0xe9, 0xee, 0x6a, 0x81, 0xc0, 0x92, 0x89, 0xea, 0x14, 0x26, 0xf7, 0x66, 0xfa, 0x74, 0x4a, 0x57, - 0x7b, 0xd3, 0x4e, 0x61, 0x72, 0x2e, 0x65, 0xc5, 0xe4, 0x5b, 0x7b, 0x2b, 0x4c, 0xc2, 0x20, 0x23, - 0x5b, 0x67, 0x8b, 0xe5, 0x5b, 0x23, 0x87, 0xbe, 0x5b, 0xbe, 0xe5, 0x51, 0xe1, 0xdc, 0xba, 0x90, - 0x0b, 0x53, 0xed, 0x30, 0x4a, 0x6e, 0x87, 0x91, 0x9c, 0x4b, 0xa8, 0x87, 0xa1, 0x64, 0x50, 0x8a, - 0x1a, 0x59, 0x2c, 0xad, 0x89, 0xc1, 0x19, 0x9e, 0x74, 0x48, 0xe2, 0x96, 0xe3, 0x93, 0xb5, 0xeb, - 0xd5, 0x13, 0xc5, 0x43, 0xd2, 0xe4, 0x24, 0xdd, 0x43, 0x22, 0x10, 0x58, 0x32, 0xa1, 0x92, 0x86, - 0x65, 0xe8, 0x66, 0x79, 0x60, 0x0b, 0x24, 0x4d, 0x57, 0x94, 0x29, 0x97, 0x34, 0x0c, 0x8c, 0x79, - 0x71, 0xf4, 0x39, 0xa8, 0x08, 0xfd, 0x2f, 0x8c, 0xab, 0xa7, 0xba, 0xb4, 0xd1, 0xb4, 0x65, 0x9c, - 0xe8, 0x7a, 0x33, 0x7f, 0x8b, 0x14, 0x97, 0xc9, 0x24, 0x11, 0x4e, 0x99, 0xda, 0x5f, 0x1c, 0xe9, - 0xd6, 0x0c, 0x98, 0x9e, 0xff, 0x45, 0xab, 0xeb, 0xa8, 0xf4, 0xe3, 0x83, 0x1a, 0xa7, 0x47, 0x78, - 0x68, 0xfa, 0x05, 0x0b, 0x4e, 0xb7, 0x73, 0x3f, 0x4a, 0x6c, 0xb3, 0x83, 0xd9, 0xb8, 0xbc, 0x1b, - 0x54, 0x86, 0xe5, 0x7c, 0x3c, 0x2e, 0xa8, 0x29, 0xab, 0x0f, 0x97, 0xdf, 0xb7, 0x3e, 0x7c, 0x15, - 0xc6, 0x98, 0x2a, 0x97, 0x66, 0x73, 0x19, 0x28, 0xe0, 0x88, 0x6d, 0xd8, 0xcb, 0xa2, 0x20, 0x56, - 0x2c, 0xd0, 0xcf, 0x5b, 0xf0, 0x70, 0xb6, 0xe9, 0x98, 0x30, 0xb4, 0xc8, 0x0e, 0xc8, 0x4d, 0x8c, - 0x55, 0xf1, 0xfd, 0x0f, 0x37, 0x7a, 0x11, 0xdf, 0xed, 0x47, 0x80, 0x7b, 0x57, 0x86, 0xea, 0x39, - 0x36, 0xce, 0x88, 0x79, 0x92, 0xd2, 0xdf, 0xce, 0x39, 0x5e, 0x2d, 0xfd, 0x6b, 0x56, 0x8e, 0x7a, - 0xc9, 0xed, 0xa9, 0x97, 0x4d, 0x7b, 0xea, 0xb1, 0xac, 0x3d, 0xd5, 0xe5, 0x1d, 0x31, 0x4c, 0xa9, - 0xc1, 0xf3, 0x97, 0x0e, 0x9a, 0xb8, 0xc6, 0xf6, 0xe1, 0x5c, 0x3f, 0x31, 0xcb, 0xc2, 0xa7, 0x5c, - 0x75, 0xae, 0x98, 0x86, 0x4f, 0xb9, 0x6b, 0x75, 0xcc, 0x30, 0x83, 0xa6, 0x40, 0xb0, 0xff, 0xab, - 0x05, 0xe5, 0x46, 0xe8, 0x1e, 0x83, 0xb7, 0xe7, 0x53, 0x86, 0xb7, 0xe7, 0xc1, 0x82, 0x57, 0xe4, - 0x0a, 0x7d, 0x3b, 0x2b, 0x19, 0xdf, 0xce, 0xc3, 0x45, 0x0c, 0x7a, 0x7b, 0x72, 0xfe, 0x5e, 0x19, - 0xf4, 0x37, 0xef, 0xd0, 0xbf, 0xba, 0x97, 0x38, 0xdc, 0x72, 0xaf, 0x67, 0xf0, 0x04, 0x67, 0x16, - 0x75, 0x25, 0xaf, 0xf8, 0xfd, 0x88, 0x85, 0xe3, 0xbe, 0x4e, 0xbc, 0xcd, 0xad, 0x84, 0xb8, 0xd9, - 0xcf, 0x39, 0xbe, 0x70, 0xdc, 0xff, 0x64, 0xc1, 0x74, 0xa6, 0x76, 0xe4, 0xe7, 0xdd, 0x17, 0xba, - 0x47, 0xff, 0xcd, 0x6c, 0xdf, 0x0b, 0x46, 0xf3, 0x00, 0xca, 0x95, 0x2e, 0x7d, 0x24, 0x4c, 0x77, - 0x55, 0xbe, 0xf6, 0x18, 0x6b, 0x14, 0xe8, 0x05, 0x18, 0x4f, 0xc2, 0x76, 0xe8, 0x87, 0x9b, 0x7b, - 0x97, 0x89, 0x4c, 0xba, 0xa1, 0x0e, 0x3c, 0xd6, 0x53, 0x14, 0xd6, 0xe9, 0xec, 0x5f, 0x2b, 0x43, - 0xf6, 0x9d, 0xc4, 0xff, 0x3f, 0x27, 0x3f, 0x9c, 0x73, 0xf2, 0xbb, 0x16, 0xcc, 0xd0, 0xda, 0x59, - 0x44, 0x8b, 0x0c, 0x64, 0x55, 0x0f, 0x1d, 0x58, 0x3d, 0x1e, 0x3a, 0x78, 0x8c, 0xca, 0x2e, 0x37, - 0xec, 0x24, 0xc2, 0x97, 0xa3, 0x09, 0x27, 0x0a, 0xc5, 0x02, 0x2b, 0xe8, 0x48, 0x14, 0x89, 0x5b, - 0x40, 0x3a, 0x1d, 0x89, 0x22, 0x2c, 0xb0, 0xf2, 0x1d, 0x84, 0xa1, 0x82, 0x77, 0x10, 0x58, 0xbe, - 0x2a, 0x11, 0x45, 0x21, 0x54, 0x03, 0x2d, 0x5f, 0x95, 0x0c, 0xaf, 0x48, 0x69, 0xec, 0x6f, 0x94, - 0x61, 0xa2, 0x11, 0xba, 0x69, 0xec, 0xfb, 0xf3, 0x46, 0xec, 0xfb, 0xb9, 0x4c, 0xec, 0xfb, 0x8c, - 0x4e, 0x7b, 0x34, 0xa1, 0xef, 0x22, 0x9b, 0x19, 0x7b, 0x95, 0xe3, 0x1e, 0xc3, 0xde, 0x8d, 0x6c, - 0x66, 0x8a, 0x11, 0x36, 0xf9, 0xfe, 0x38, 0x85, 0xbb, 0xff, 0xb9, 0x05, 0x53, 0x8d, 0xd0, 0xa5, - 0x13, 0xf4, 0xc7, 0x69, 0x36, 0xea, 0xd9, 0xd0, 0x46, 0x7a, 0x64, 0x43, 0xfb, 0xfb, 0x16, 0x8c, - 0x36, 0x42, 0xf7, 0x18, 0xfc, 0x9c, 0x2f, 0x9b, 0x7e, 0xce, 0x07, 0x0a, 0xa4, 0x6c, 0x81, 0x6b, - 0xf3, 0xb7, 0xca, 0x30, 0x49, 0xdb, 0x19, 0x6e, 0xca, 0x51, 0x32, 0x7a, 0xc4, 0x1a, 0xa0, 0x47, - 0xa8, 0x32, 0x17, 0xfa, 0x7e, 0x78, 0x3b, 0x3b, 0x62, 0xab, 0x0c, 0x8a, 0x05, 0x16, 0x3d, 0x0d, - 0x63, 0xed, 0x88, 0xec, 0x7a, 0x61, 0x27, 0xce, 0xde, 0x23, 0x6c, 0x08, 0x38, 0x56, 0x14, 0xe8, - 0x79, 0x98, 0x88, 0xbd, 0xa0, 0x45, 0x64, 0x64, 0xc5, 0x10, 0x8b, 0xac, 0xe0, 0x09, 0x25, 0x35, - 0x38, 0x36, 0xa8, 0xd0, 0xeb, 0x50, 0x61, 0xff, 0xd9, 0xba, 0x39, 0xfc, 0x33, 0x07, 0xdc, 0x54, - 0x95, 0x0c, 0x70, 0xca, 0x0b, 0x5d, 0x00, 0x48, 0x64, 0x0c, 0x48, 0x2c, 0xae, 0xb9, 0x2a, 0x8d, - 0x52, 0x45, 0x87, 0xc4, 0x58, 0xa3, 0x42, 0x4f, 0x41, 0x25, 0x71, 0x3c, 0xff, 0x8a, 0x17, 0x90, - 0x58, 0xc4, 0xd0, 0x88, 0x24, 0xcd, 0x02, 0x88, 0x53, 0x3c, 0xdd, 0xd1, 0xd9, 0x25, 0x6a, 0xfe, - 0x48, 0xca, 0x18, 0xa3, 0x66, 0x3b, 0xfa, 0x15, 0x05, 0xc5, 0x1a, 0x85, 0xfd, 0x22, 0x9c, 0x6a, - 0x84, 0x6e, 0x23, 0x8c, 0x92, 0xd5, 0x30, 0xba, 0xed, 0x44, 0xae, 0x1c, 0xbf, 0x9a, 0xcc, 0x17, - 0x4c, 0x77, 0xdd, 0x61, 0x6e, 0xd7, 0x1b, 0x99, 0x80, 0x9f, 0x63, 0x7b, 0xfa, 0x21, 0x2f, 0x3c, - 0xfc, 0xdb, 0x12, 0xa0, 0x06, 0x8b, 0x52, 0x31, 0x5e, 0xd2, 0x79, 0x1b, 0xa6, 0x62, 0x72, 0xc5, - 0x0b, 0x3a, 0x77, 0x04, 0xab, 0x5e, 0xb7, 0x49, 0x9a, 0x2b, 0x3a, 0x25, 0xf7, 0x8d, 0x98, 0x30, - 0x9c, 0xe1, 0x46, 0xbb, 0x30, 0xea, 0x04, 0x8b, 0xf1, 0x8d, 0x98, 0x44, 0xe2, 0xe5, 0x18, 0xd6, - 0x85, 0x58, 0x02, 0x71, 0x8a, 0xa7, 0x53, 0x86, 0xfd, 0xb9, 0x16, 0x06, 0x38, 0x0c, 0x13, 0x39, - 0xc9, 0xd8, 0xdb, 0x03, 0x1a, 0x1c, 0x1b, 0x54, 0x68, 0x15, 0x50, 0xdc, 0x69, 0xb7, 0x7d, 0x76, - 0xa8, 0xe7, 0xf8, 0x17, 0xa3, 0xb0, 0xd3, 0xe6, 0x61, 0xc6, 0x22, 0x6d, 0x7f, 0xb3, 0x0b, 0x8b, - 0x73, 0x4a, 0x50, 0xc1, 0xb0, 0x11, 0xb3, 0xdf, 0xe2, 0x1e, 0x35, 0xf7, 0x4d, 0x36, 0x19, 0x08, - 0x4b, 0x9c, 0xfd, 0x79, 0xb6, 0x99, 0xb1, 0x07, 0x3f, 0x92, 0x4e, 0x44, 0xd0, 0x0e, 0x4c, 0xb6, - 0xd9, 0x86, 0x95, 0x44, 0xa1, 0xef, 0x13, 0xa9, 0x37, 0xde, 0x5b, 0xc4, 0x0c, 0x7f, 0x00, 0x40, - 0x67, 0x87, 0x4d, 0xee, 0xf6, 0x17, 0xa7, 0x99, 0x5c, 0x6a, 0x72, 0xa3, 0x65, 0x54, 0xc4, 0xc1, - 0x0a, 0x0d, 0x6d, 0xae, 0xf8, 0x81, 0xad, 0x54, 0xd2, 0x8b, 0x58, 0x5a, 0x2c, 0xcb, 0xa2, 0xd7, - 0x58, 0x7c, 0x36, 0x17, 0x06, 0xfd, 0x9e, 0xf6, 0xe3, 0x54, 0x46, 0x6c, 0xb6, 0x28, 0x88, 0x35, - 0x26, 0xe8, 0x0a, 0x4c, 0x8a, 0xf7, 0x21, 0x84, 0x0b, 0xa1, 0x6c, 0x98, 0xbf, 0x93, 0x58, 0x47, - 0xde, 0xcd, 0x02, 0xb0, 0x59, 0x18, 0x6d, 0xc2, 0xc3, 0xda, 0x6b, 0x46, 0x39, 0x51, 0x5b, 0x5c, - 0xb6, 0x3c, 0x72, 0xb0, 0x5f, 0x7b, 0x78, 0xbd, 0x17, 0x21, 0xee, 0xcd, 0x07, 0x5d, 0x87, 0x53, - 0x4e, 0x2b, 0xf1, 0x76, 0x49, 0x9d, 0x38, 0xae, 0xef, 0x05, 0xc4, 0xbc, 0x58, 0x7f, 0xe6, 0x60, - 0xbf, 0x76, 0x6a, 0x31, 0x8f, 0x00, 0xe7, 0x97, 0x43, 0x2f, 0x43, 0xc5, 0x0d, 0x62, 0xd1, 0x07, - 0x23, 0xc6, 0x43, 0x5d, 0x95, 0xfa, 0xb5, 0xa6, 0xfa, 0xfe, 0xf4, 0x0f, 0x4e, 0x0b, 0xa0, 0x4d, - 0xfe, 0xa0, 0xbb, 0xb2, 0x48, 0x46, 0xbb, 0xb2, 0x25, 0x64, 0x6d, 0x5b, 0xe3, 0xc6, 0x09, 0xf7, - 0x9f, 0xa9, 0x98, 0x48, 0xe3, 0x32, 0x8a, 0xc1, 0x18, 0xbd, 0x0a, 0x28, 0x26, 0xd1, 0xae, 0xd7, - 0x22, 0x8b, 0x2d, 0x96, 0x71, 0x95, 0x79, 0x5d, 0xc6, 0x8c, 0x00, 0x7f, 0xd4, 0xec, 0xa2, 0xc0, - 0x39, 0xa5, 0xd0, 0x25, 0x2a, 0x51, 0x74, 0xa8, 0x08, 0x61, 0x95, 0x6a, 0x5e, 0xb5, 0x4e, 0xda, - 0x11, 0x69, 0x39, 0x09, 0x71, 0x4d, 0x8e, 0x38, 0x53, 0x8e, 0xee, 0x37, 0x2a, 0x91, 0x3d, 0x98, - 0x81, 0x97, 0xdd, 0xc9, 0xec, 0xa9, 0x85, 0xb4, 0x15, 0xc6, 0xc9, 0x35, 0x92, 0xdc, 0x0e, 0xa3, - 0x6d, 0x91, 0xd4, 0x2a, 0xcd, 0x62, 0x97, 0xa2, 0xb0, 0x4e, 0x47, 0x35, 0x22, 0x76, 0x74, 0xb5, - 0x56, 0x67, 0xe7, 0x0c, 0x63, 0xe9, 0x3a, 0xb9, 0xc4, 0xc1, 0x58, 0xe2, 0x25, 0xe9, 0x5a, 0x63, - 0x99, 0x9d, 0x1e, 0x64, 0x48, 0xd7, 0x1a, 0xcb, 0x58, 0xe2, 0x11, 0xe9, 0x7e, 0x04, 0x6d, 0xaa, - 0xf8, 0x84, 0xa6, 0x5b, 0x2e, 0x0f, 0xf8, 0x0e, 0x5a, 0x00, 0x33, 0xea, 0xf9, 0x35, 0x9e, 0xed, - 0x2b, 0xae, 0x4e, 0x17, 0xbf, 0x2c, 0x9f, 0x9b, 0x2a, 0x4c, 0x79, 0xd5, 0xd6, 0x32, 0x9c, 0x70, - 0x17, 0x6f, 0x23, 0x61, 0xc3, 0x4c, 0xdf, 0x87, 0x08, 0x16, 0xa0, 0x12, 0x77, 0x6e, 0xb9, 0xe1, - 0x8e, 0xe3, 0x05, 0xcc, 0xed, 0xaf, 0x3f, 0x7a, 0x2e, 0x11, 0x38, 0xa5, 0x41, 0xab, 0x30, 0xe6, - 0x08, 0xe3, 0x4b, 0x38, 0xea, 0x73, 0x6f, 0x70, 0x4b, 0x03, 0x8d, 0x7b, 0x34, 0xd5, 0xfb, 0xff, - 0xaa, 0x2c, 0x7a, 0x09, 0x26, 0xc5, 0x25, 0x23, 0x11, 0x1f, 0x78, 0xc2, 0x8c, 0x47, 0x6f, 0xea, - 0x48, 0x6c, 0xd2, 0xa2, 0x9f, 0x86, 0x29, 0xca, 0x25, 0x15, 0x6c, 0xd5, 0x93, 0x83, 0x48, 0x44, - 0x2d, 0xc1, 0xb4, 0x5e, 0x18, 0x67, 0x98, 0x21, 0x17, 0x1e, 0x72, 0x3a, 0x49, 0xb8, 0x43, 0x67, - 0xb8, 0x39, 0xff, 0xd7, 0xc3, 0x6d, 0x12, 0x30, 0x3f, 0xfd, 0xd8, 0xd2, 0xb9, 0x83, 0xfd, 0xda, - 0x43, 0x8b, 0x3d, 0xe8, 0x70, 0x4f, 0x2e, 0xe8, 0x06, 0x8c, 0x27, 0xa1, 0x2f, 0x02, 0x7b, 0xe3, - 0xea, 0xe9, 0xe2, 0x84, 0x33, 0xeb, 0x8a, 0x4c, 0x77, 0x27, 0xa8, 0xa2, 0x58, 0xe7, 0x83, 0xd6, - 0xf9, 0x1a, 0x63, 0x79, 0x0b, 0x49, 0x5c, 0x7d, 0xa0, 0xb8, 0x63, 0x54, 0x7a, 0x43, 0x73, 0x09, - 0x8a, 0x92, 0x58, 0x67, 0x83, 0x2e, 0xc2, 0x6c, 0x3b, 0xf2, 0x42, 0x36, 0xb1, 0x95, 0xcb, 0xb7, - 0x6a, 0xa4, 0x22, 0x9b, 0x6d, 0x64, 0x09, 0x70, 0x77, 0x19, 0x74, 0x9e, 0x2a, 0xa8, 0x1c, 0x58, - 0x3d, 0xc3, 0x1f, 0xa8, 0xe0, 0xca, 0x29, 0x87, 0x61, 0x85, 0x9d, 0xfb, 0x34, 0xcc, 0x76, 0x49, - 0xca, 0x43, 0x05, 0x59, 0xfe, 0xfa, 0x30, 0x54, 0x94, 0x3b, 0x10, 0x2d, 0x98, 0x5e, 0xde, 0x33, - 0x59, 0x2f, 0xef, 0x18, 0xd5, 0xd7, 0x74, 0xc7, 0xee, 0x7a, 0xce, 0x1b, 0xdb, 0xe7, 0x0a, 0x44, - 0xc3, 0xe0, 0x37, 0xa2, 0x0e, 0xf1, 0xfe, 0x78, 0x6a, 0x30, 0x0e, 0xf5, 0x34, 0x18, 0x07, 0x7c, - 0xef, 0x8e, 0x9a, 0x86, 0xed, 0xd0, 0x5d, 0x6b, 0x64, 0x1f, 0x80, 0x6a, 0x50, 0x20, 0xe6, 0x38, - 0xa6, 0xdc, 0xd3, 0x6d, 0x9d, 0x29, 0xf7, 0xa3, 0xf7, 0xa8, 0xdc, 0x4b, 0x06, 0x38, 0xe5, 0x85, - 0x7c, 0x98, 0x6d, 0x99, 0x6f, 0x77, 0xa9, 0x5b, 0x50, 0x8f, 0xf6, 0x7d, 0x45, 0xab, 0xa3, 0x3d, - 0xe8, 0xb1, 0x9c, 0xe5, 0x82, 0xbb, 0x19, 0xa3, 0x97, 0x60, 0xec, 0xdd, 0x30, 0x66, 0xd3, 0x4e, - 0xec, 0x6d, 0xf2, 0xde, 0xc9, 0xd8, 0x6b, 0xd7, 0x9b, 0x0c, 0x7e, 0x77, 0xbf, 0x36, 0xde, 0x08, - 0x5d, 0xf9, 0x17, 0xab, 0x02, 0xe8, 0x0e, 0x9c, 0x32, 0x24, 0x82, 0x6a, 0x2e, 0x0c, 0xde, 0xdc, - 0x87, 0x45, 0x75, 0xa7, 0xd6, 0xf2, 0x38, 0xe1, 0xfc, 0x0a, 0xec, 0x6f, 0x72, 0xa7, 0xa7, 0x70, - 0x8d, 0x90, 0xb8, 0xe3, 0x1f, 0x47, 0xd6, 0xfe, 0x15, 0xc3, 0x6b, 0x73, 0xcf, 0x8e, 0xf5, 0xdf, - 0xb7, 0x98, 0x63, 0x7d, 0x9d, 0xec, 0xb4, 0x7d, 0x27, 0x39, 0x8e, 0xd0, 0xda, 0xd7, 0x60, 0x2c, - 0x11, 0xb5, 0xf5, 0x7a, 0x68, 0x40, 0x6b, 0x14, 0x3b, 0x5c, 0x50, 0x1b, 0xa2, 0x84, 0x62, 0xc5, - 0xc6, 0xfe, 0xa7, 0x7c, 0x04, 0x24, 0xe6, 0x18, 0x7c, 0x0b, 0x75, 0xd3, 0xb7, 0x50, 0xeb, 0xf3, - 0x05, 0x05, 0x3e, 0x86, 0x7f, 0x62, 0xb6, 0x9b, 0xd9, 0x1e, 0x1f, 0xf6, 0x13, 0x1d, 0xfb, 0x97, - 0x2d, 0x38, 0x99, 0x77, 0xa4, 0x4f, 0x95, 0x18, 0x6e, 0xf9, 0xa8, 0x13, 0x2e, 0xd5, 0x83, 0x37, - 0x05, 0x1c, 0x2b, 0x8a, 0x81, 0x93, 0x7d, 0x1f, 0x2e, 0xc9, 0xd2, 0x75, 0x30, 0x9f, 0x79, 0x43, - 0xaf, 0xf0, 0x58, 0x79, 0x4b, 0xbd, 0xc3, 0x76, 0xb8, 0x38, 0x79, 0xfb, 0xeb, 0x25, 0x38, 0xc9, - 0x5d, 0xd4, 0x8b, 0xbb, 0xa1, 0xe7, 0x36, 0x42, 0x57, 0xdc, 0x1c, 0x78, 0x13, 0x26, 0xda, 0x9a, - 0xb9, 0xda, 0x2b, 0xcd, 0x8b, 0x6e, 0xd6, 0xa6, 0x66, 0x83, 0x0e, 0xc5, 0x06, 0x2f, 0xe4, 0xc2, - 0x04, 0xd9, 0xf5, 0x5a, 0xca, 0xcf, 0x59, 0x3a, 0xb4, 0x48, 0x57, 0xb5, 0xac, 0x68, 0x7c, 0xb0, - 0xc1, 0xf5, 0x3e, 0x3c, 0xc9, 0x61, 0x7f, 0xc5, 0x82, 0x07, 0x0a, 0x92, 0xc2, 0xd0, 0xea, 0x6e, - 0xb3, 0xc3, 0x00, 0xf1, 0x66, 0xa0, 0xaa, 0x8e, 0x1f, 0x11, 0x60, 0x81, 0x45, 0x3f, 0x05, 0xc0, - 0x5d, 0xfc, 0xec, 0x85, 0xf6, 0x52, 0xef, 0x5b, 0xe7, 0x46, 0xb2, 0x04, 0xed, 0x46, 0xbd, 0x7a, - 0x93, 0x5d, 0xe3, 0x65, 0xff, 0x6a, 0x19, 0x86, 0xf9, 0x03, 0xd2, 0xab, 0x30, 0xba, 0xc5, 0x53, - 0xd0, 0x0e, 0x92, 0xed, 0x36, 0x35, 0x47, 0x38, 0x00, 0xcb, 0xc2, 0xe8, 0x2a, 0x9c, 0x10, 0xb7, - 0x53, 0xea, 0xc4, 0x77, 0xf6, 0xa4, 0x55, 0xcb, 0xdf, 0x69, 0x90, 0x39, 0xc4, 0x4f, 0xac, 0x75, - 0x93, 0xe0, 0xbc, 0x72, 0xe8, 0x95, 0xae, 0xc4, 0x73, 0x3c, 0x79, 0xaf, 0xd2, 0x81, 0xfb, 0x24, - 0x9f, 0x7b, 0x09, 0x26, 0xdb, 0x5d, 0xf6, 0xbb, 0xf6, 0x76, 0xaf, 0x69, 0xb3, 0x9b, 0xb4, 0x2c, - 0x3e, 0xa0, 0xc3, 0xa2, 0x21, 0xd6, 0xb7, 0x22, 0x12, 0x6f, 0x85, 0xbe, 0x2b, 0x1e, 0xaa, 0x4c, - 0xe3, 0x03, 0x32, 0x78, 0xdc, 0x55, 0x82, 0x72, 0xd9, 0x70, 0x3c, 0xbf, 0x13, 0x91, 0x94, 0xcb, - 0x88, 0xc9, 0x65, 0x35, 0x83, 0xc7, 0x5d, 0x25, 0xe8, 0x3c, 0x3a, 0x25, 0x5e, 0x39, 0x94, 0x77, - 0x96, 0x55, 0xd0, 0xc7, 0xa8, 0x8c, 0x4a, 0xef, 0x91, 0x47, 0x43, 0x1c, 0xf9, 0xab, 0x77, 0x12, - 0xb5, 0xf7, 0xb3, 0x44, 0x3c, 0xba, 0xe4, 0x72, 0x2f, 0x6f, 0xed, 0xfd, 0xa9, 0x05, 0x27, 0x72, - 0x02, 0xc1, 0xb8, 0xa8, 0xda, 0xf4, 0xe2, 0x44, 0x3d, 0x0f, 0xa0, 0x89, 0x2a, 0x0e, 0xc7, 0x8a, - 0x82, 0xae, 0x07, 0x2e, 0x0c, 0xb3, 0x02, 0x50, 0x04, 0x6f, 0x08, 0xec, 0xe1, 0x04, 0x20, 0x3a, - 0x07, 0x43, 0x9d, 0x98, 0x44, 0xf2, 0x81, 0x3a, 0x29, 0xbf, 0x99, 0x47, 0x90, 0x61, 0xa8, 0x46, - 0xb9, 0xa9, 0x9c, 0x71, 0x9a, 0x46, 0xc9, 0xdd, 0x71, 0x1c, 0x67, 0x7f, 0xb9, 0x0c, 0xd3, 0x99, - 0xb0, 0x4d, 0xda, 0x90, 0x9d, 0x30, 0xf0, 0x92, 0x50, 0xe5, 0x3d, 0xe3, 0x69, 0x1e, 0x48, 0x7b, - 0xeb, 0xaa, 0x80, 0x63, 0x45, 0x81, 0x1e, 0x93, 0x2f, 0x97, 0x66, 0x9f, 0x3d, 0x58, 0xaa, 0x1b, - 0x8f, 0x97, 0x0e, 0xfa, 0x7e, 0xc9, 0xa3, 0x30, 0xd4, 0x0e, 0xd5, 0xb3, 0xd2, 0x6a, 0x3c, 0xf1, - 0x52, 0xbd, 0x11, 0x86, 0x3e, 0x66, 0x48, 0xf4, 0x31, 0xf1, 0xf5, 0x99, 0xf3, 0x0a, 0xec, 0xb8, - 0x61, 0xac, 0x75, 0xc1, 0x13, 0x30, 0xba, 0x4d, 0xf6, 0x22, 0x2f, 0xd8, 0xcc, 0x9e, 0xd6, 0x5c, - 0xe6, 0x60, 0x2c, 0xf1, 0x66, 0xb6, 0xf0, 0xd1, 0xfb, 0xf2, 0x04, 0xc9, 0x58, 0xdf, 0x5d, 0xed, - 0xb7, 0x2c, 0x98, 0x66, 0x39, 0x46, 0xc5, 0xed, 0x78, 0x2f, 0x0c, 0x8e, 0x41, 0x4f, 0x78, 0x14, - 0x86, 0x23, 0x5a, 0x69, 0xf6, 0x5d, 0x01, 0xd6, 0x12, 0xcc, 0x71, 0xe8, 0x21, 0x18, 0x62, 0x4d, - 0xa0, 0x83, 0x37, 0xc1, 0xb3, 0x8c, 0xd7, 0x9d, 0xc4, 0xc1, 0x0c, 0xca, 0xae, 0x29, 0x61, 0xd2, - 0xf6, 0x3d, 0xde, 0xe8, 0xd4, 0xdd, 0xfa, 0xe1, 0xb8, 0xa6, 0x94, 0xdb, 0xb4, 0xf7, 0x77, 0x4d, - 0x29, 0x9f, 0x65, 0x6f, 0x1d, 0xfc, 0xbf, 0x95, 0xe0, 0x6c, 0x6e, 0xb9, 0xf4, 0x64, 0x77, 0xd5, - 0x38, 0xd9, 0xbd, 0x90, 0x39, 0xd9, 0xb5, 0x7b, 0x97, 0x3e, 0x9a, 0xb3, 0xde, 0xfc, 0x23, 0xd8, - 0xf2, 0x31, 0x1e, 0xc1, 0x0e, 0x0d, 0xaa, 0xa6, 0x0c, 0xf7, 0x51, 0x53, 0xbe, 0x63, 0xc1, 0x99, - 0xdc, 0x2e, 0xfb, 0x90, 0xdc, 0x0b, 0xcb, 0x6d, 0x5b, 0x81, 0x0d, 0xf1, 0xc3, 0x52, 0xc1, 0xb7, - 0x30, 0x6b, 0xe2, 0x3c, 0x95, 0x33, 0x0c, 0x19, 0x0b, 0xb5, 0x6b, 0x82, 0xcb, 0x18, 0x0e, 0xc3, - 0x0a, 0x8b, 0x3c, 0xed, 0x86, 0x15, 0x6f, 0xda, 0x4b, 0x87, 0x5a, 0x32, 0xf3, 0xa6, 0x77, 0x5c, - 0xbf, 0xca, 0x9f, 0xbd, 0x6d, 0x75, 0x55, 0xb3, 0x00, 0xcb, 0x83, 0x5b, 0x80, 0x13, 0xf9, 0xd6, - 0x1f, 0x5a, 0x84, 0xe9, 0x1d, 0x2f, 0x60, 0x8f, 0x83, 0x9a, 0x7a, 0x8f, 0xba, 0x96, 0x7a, 0xd5, - 0x44, 0xe3, 0x2c, 0xfd, 0xdc, 0x4b, 0x30, 0x79, 0xef, 0x2e, 0xab, 0xef, 0x96, 0xe1, 0xc1, 0x1e, - 0xcb, 0x9e, 0xcb, 0x7a, 0x63, 0x0c, 0x34, 0x59, 0xdf, 0x35, 0x0e, 0x0d, 0x38, 0xb9, 0xd1, 0xf1, - 0xfd, 0x3d, 0x16, 0xe5, 0x44, 0x5c, 0x49, 0x21, 0x14, 0x13, 0x95, 0x40, 0x78, 0x35, 0x87, 0x06, - 0xe7, 0x96, 0x44, 0xaf, 0x02, 0x0a, 0x6f, 0xb1, 0xa4, 0xb6, 0x6e, 0x9a, 0xa0, 0x80, 0x75, 0x7c, - 0x39, 0x5d, 0x8c, 0xd7, 0xbb, 0x28, 0x70, 0x4e, 0x29, 0xaa, 0x61, 0xb2, 0x27, 0xcd, 0x55, 0xb3, - 0x32, 0x1a, 0x26, 0xd6, 0x91, 0xd8, 0xa4, 0x45, 0x17, 0x61, 0xd6, 0xd9, 0x75, 0x3c, 0x9e, 0xb0, - 0x4a, 0x32, 0xe0, 0x2a, 0xa6, 0x72, 0x14, 0x2d, 0x66, 0x09, 0x70, 0x77, 0x19, 0xb4, 0x61, 0x78, - 0xf9, 0x78, 0xbe, 0xfc, 0x0b, 0x03, 0xcf, 0xd6, 0x81, 0xfd, 0x7e, 0xf6, 0x7f, 0xb4, 0xe8, 0xf6, - 0x95, 0xf3, 0x1a, 0x25, 0xed, 0x07, 0xe5, 0xbf, 0xd2, 0x6e, 0x87, 0xa9, 0x7e, 0x58, 0xd6, 0x91, - 0xd8, 0xa4, 0xe5, 0x13, 0x22, 0x4e, 0xc3, 0xa5, 0x0d, 0x3d, 0x51, 0x5c, 0xa7, 0x54, 0x14, 0xe8, - 0x0d, 0x18, 0x75, 0xbd, 0x5d, 0x2f, 0x0e, 0x23, 0xb1, 0x58, 0x0e, 0xfb, 0x0a, 0xb3, 0x92, 0x83, - 0x75, 0xce, 0x06, 0x4b, 0x7e, 0xf6, 0x97, 0x4b, 0x30, 0x29, 0x6b, 0x7c, 0xad, 0x13, 0x26, 0xce, - 0x31, 0x6c, 0xcb, 0x17, 0x8d, 0x6d, 0xf9, 0x63, 0xbd, 0xee, 0x94, 0xb2, 0x26, 0x15, 0x6e, 0xc7, - 0xd7, 0x33, 0xdb, 0xf1, 0xe3, 0xfd, 0x59, 0xf5, 0xde, 0x86, 0x7f, 0xd7, 0x82, 0x59, 0x83, 0xfe, - 0x18, 0x76, 0x83, 0x55, 0x73, 0x37, 0x78, 0xa4, 0xef, 0x37, 0x14, 0xec, 0x02, 0x5f, 0x2b, 0x65, - 0xda, 0xce, 0xa4, 0xff, 0xbb, 0x30, 0xb4, 0xe5, 0x44, 0x6e, 0xaf, 0xb4, 0x8b, 0x5d, 0x85, 0xe6, - 0x2f, 0x39, 0x91, 0xcb, 0x65, 0xf8, 0xd3, 0xea, 0xa1, 0x2c, 0x27, 0x72, 0xfb, 0xde, 0x0e, 0x60, - 0x55, 0xa1, 0x17, 0x61, 0x24, 0x6e, 0x85, 0x6d, 0x15, 0x7b, 0x79, 0x8e, 0x3f, 0xa2, 0x45, 0x21, - 0x77, 0xf7, 0x6b, 0xc8, 0xac, 0x8e, 0x82, 0xb1, 0xa0, 0x9f, 0xdb, 0x84, 0x8a, 0xaa, 0xfa, 0xbe, - 0x46, 0x95, 0xff, 0x51, 0x19, 0x4e, 0xe4, 0xcc, 0x0b, 0x14, 0x1b, 0xbd, 0xf5, 0xec, 0x80, 0xd3, - 0xe9, 0x7d, 0xf6, 0x57, 0xcc, 0x2c, 0x16, 0x57, 0x8c, 0xff, 0xc0, 0x95, 0xde, 0x88, 0x49, 0xb6, - 0x52, 0x0a, 0xea, 0x5f, 0x29, 0xad, 0xec, 0xd8, 0xba, 0x9a, 0x56, 0xa4, 0x5a, 0x7a, 0x5f, 0xc7, - 0xf4, 0x7f, 0x95, 0xe1, 0x64, 0xde, 0x55, 0x74, 0xf4, 0xb3, 0x99, 0x47, 0x1c, 0x9e, 0x1f, 0xf4, - 0x12, 0x3b, 0x7f, 0xd9, 0x41, 0x64, 0x78, 0x99, 0x37, 0x9f, 0x75, 0xe8, 0xdb, 0xcd, 0xa2, 0x4e, - 0x76, 0x5d, 0x27, 0xe2, 0x8f, 0x6f, 0xc8, 0x25, 0xfe, 0xf1, 0x81, 0x1b, 0x20, 0x5e, 0xed, 0x88, - 0x33, 0xd7, 0x75, 0x24, 0xb8, 0xff, 0x75, 0x1d, 0x59, 0xf3, 0x9c, 0x07, 0xe3, 0xda, 0xd7, 0xdc, - 0xd7, 0x11, 0xdf, 0xa6, 0x3b, 0x8a, 0xd6, 0xee, 0xfb, 0x3a, 0xea, 0x5f, 0xb1, 0x20, 0x13, 0x27, - 0xa5, 0xfc, 0x1f, 0x56, 0xa1, 0xff, 0xe3, 0x1c, 0x0c, 0x45, 0xa1, 0x4f, 0xb2, 0x79, 0xfd, 0x71, - 0xe8, 0x13, 0xcc, 0x30, 0xea, 0x55, 0xdc, 0x72, 0xd1, 0xab, 0xb8, 0xd4, 0x34, 0xf6, 0xc9, 0x2e, - 0x91, 0xde, 0x08, 0x25, 0x93, 0xaf, 0x50, 0x20, 0xe6, 0x38, 0xfb, 0x37, 0x86, 0xe0, 0x44, 0xce, - 0xe5, 0x34, 0x6a, 0xa8, 0x6c, 0x3a, 0x09, 0xb9, 0xed, 0xec, 0x65, 0x73, 0x8d, 0x5e, 0xe4, 0x60, - 0x2c, 0xf1, 0x2c, 0x96, 0x93, 0xa7, 0x2b, 0xcb, 0xf8, 0x88, 0x44, 0x96, 0x32, 0x81, 0xbd, 0x5f, - 0x0f, 0xa5, 0x5e, 0x00, 0x88, 0x63, 0x7f, 0x25, 0xa0, 0xca, 0x97, 0x2b, 0x22, 0x45, 0xd3, 0xdc, - 0x76, 0xcd, 0x2b, 0x02, 0x83, 0x35, 0x2a, 0x54, 0x87, 0x99, 0x76, 0x14, 0x26, 0xdc, 0xef, 0x56, - 0xe7, 0x31, 0x0a, 0xc3, 0xe6, 0x35, 0xa3, 0x46, 0x06, 0x8f, 0xbb, 0x4a, 0xa0, 0x17, 0x60, 0x5c, - 0x5c, 0x3d, 0x6a, 0x84, 0xa1, 0x2f, 0xbc, 0x34, 0xea, 0xc4, 0xbb, 0x99, 0xa2, 0xb0, 0x4e, 0xa7, - 0x15, 0x63, 0xce, 0xbc, 0xd1, 0xdc, 0x62, 0xdc, 0xa1, 0xa7, 0xd1, 0x65, 0x32, 0x52, 0x8c, 0x0d, - 0x94, 0x91, 0x22, 0xf5, 0x5b, 0x55, 0x06, 0x3e, 0xbf, 0x80, 0xbe, 0x9e, 0x9e, 0x6f, 0x96, 0x61, - 0x84, 0x0f, 0xc5, 0x31, 0xa8, 0x62, 0xab, 0xc2, 0x77, 0xd3, 0x23, 0x0f, 0x00, 0x6f, 0xcb, 0x7c, - 0xdd, 0x49, 0x1c, 0x2e, 0x86, 0xd4, 0x6a, 0x48, 0xbd, 0x3c, 0x68, 0xde, 0x58, 0x2f, 0x73, 0x19, - 0xe7, 0x04, 0x70, 0x1e, 0xda, 0xea, 0x79, 0x1b, 0x20, 0x66, 0x8f, 0x75, 0x52, 0x1e, 0x22, 0x6f, - 0xe9, 0x93, 0x3d, 0x6a, 0x6f, 0x2a, 0x62, 0xde, 0x86, 0x74, 0x0a, 0x2a, 0x04, 0xd6, 0x38, 0xce, - 0x7d, 0x02, 0x2a, 0x8a, 0xb8, 0x9f, 0x25, 0x37, 0xa1, 0x0b, 0xaf, 0x4f, 0xc1, 0x74, 0xa6, 0xae, - 0x43, 0x19, 0x82, 0xbf, 0x6d, 0xc1, 0x34, 0x6f, 0xf2, 0x4a, 0xb0, 0x2b, 0x16, 0xfb, 0x7b, 0x70, - 0xd2, 0xcf, 0x59, 0x74, 0x62, 0x44, 0x07, 0x5f, 0xa4, 0xca, 0xf0, 0xcb, 0xc3, 0xe2, 0xdc, 0x3a, - 0xa8, 0xf1, 0xcf, 0x9f, 0x19, 0x76, 0x7c, 0x11, 0x81, 0x3c, 0xc1, 0xf3, 0x39, 0x73, 0x18, 0x56, - 0x58, 0xfb, 0x7b, 0x16, 0xcc, 0x76, 0x3d, 0x52, 0xff, 0x81, 0xb6, 0x5d, 0xa4, 0xab, 0x2e, 0x15, - 0xa4, 0xab, 0xd6, 0x3f, 0xad, 0xdc, 0xf3, 0xd3, 0xbe, 0x6e, 0x81, 0x98, 0x81, 0xc7, 0xa0, 0xce, - 0x7f, 0xda, 0x54, 0xe7, 0xe7, 0x8a, 0x27, 0x75, 0x81, 0x1e, 0xff, 0xe7, 0x16, 0xcc, 0x70, 0x82, - 0xf4, 0xf0, 0xe2, 0x03, 0x1d, 0x87, 0x41, 0xde, 0x50, 0x51, 0x8f, 0x56, 0xe6, 0x7f, 0x94, 0x31, - 0x58, 0x43, 0x3d, 0x07, 0xeb, 0xbf, 0x58, 0x80, 0xf8, 0xe7, 0x67, 0x5f, 0x5e, 0xe6, 0x9b, 0x92, - 0x66, 0x6a, 0xa7, 0x42, 0x40, 0x61, 0xb0, 0x46, 0x75, 0x24, 0x0d, 0xcf, 0x9c, 0x0d, 0x95, 0xfb, - 0x9f, 0x0d, 0x1d, 0xe2, 0x5b, 0xff, 0xda, 0x10, 0x64, 0x03, 0x11, 0xd1, 0x4d, 0x98, 0x68, 0x39, - 0x6d, 0xe7, 0x96, 0xe7, 0x7b, 0x89, 0x47, 0xe2, 0x5e, 0x87, 0xca, 0xcb, 0x1a, 0x9d, 0x38, 0x88, - 0xd1, 0x20, 0xd8, 0xe0, 0x83, 0xe6, 0x01, 0xda, 0x91, 0xb7, 0xeb, 0xf9, 0x64, 0x93, 0xd9, 0x1a, - 0xec, 0x36, 0x02, 0x3f, 0x29, 0x95, 0x50, 0xac, 0x51, 0xe4, 0x44, 0xaf, 0x97, 0xef, 0x5f, 0xf4, - 0xfa, 0xd0, 0x21, 0xa3, 0xd7, 0x87, 0x07, 0x8a, 0x5e, 0xc7, 0x70, 0x5a, 0xee, 0xaa, 0xf4, 0xff, - 0xaa, 0xe7, 0x13, 0xa1, 0x4a, 0xf1, 0x3b, 0x0a, 0x73, 0x07, 0xfb, 0xb5, 0xd3, 0x38, 0x97, 0x02, - 0x17, 0x94, 0x44, 0x3f, 0x05, 0x55, 0xc7, 0xf7, 0xc3, 0xdb, 0xaa, 0xd7, 0x56, 0xe2, 0x96, 0xe3, - 0xa7, 0xa9, 0x40, 0xc7, 0x96, 0x1e, 0x3a, 0xd8, 0xaf, 0x55, 0x17, 0x0b, 0x68, 0x70, 0x61, 0x69, - 0x7b, 0x1b, 0x4e, 0x34, 0x49, 0x24, 0x1f, 0x02, 0x53, 0xab, 0x6f, 0x1d, 0x2a, 0x51, 0x66, 0xb9, - 0x0f, 0x74, 0x25, 0x5d, 0x4b, 0xc0, 0x25, 0x97, 0x77, 0xca, 0xc8, 0xfe, 0x33, 0x0b, 0x46, 0x45, - 0x70, 0xe3, 0x31, 0x68, 0x19, 0x8b, 0x86, 0xc3, 0xa7, 0x96, 0x2f, 0x12, 0x59, 0x63, 0x0a, 0x5d, - 0x3d, 0x6b, 0x19, 0x57, 0xcf, 0x23, 0xbd, 0x98, 0xf4, 0x76, 0xf2, 0xfc, 0x52, 0x19, 0xa6, 0xcc, - 0xc0, 0xce, 0x63, 0xe8, 0x82, 0x6b, 0x30, 0x1a, 0x8b, 0x28, 0xe2, 0x52, 0x71, 0x34, 0x5a, 0x76, - 0x10, 0xd3, 0x33, 0x6b, 0x11, 0x37, 0x2c, 0x99, 0xe4, 0x86, 0x27, 0x97, 0xef, 0x63, 0x78, 0x72, - 0xbf, 0xd8, 0xda, 0xa1, 0xa3, 0x88, 0xad, 0xb5, 0xbf, 0xc5, 0x84, 0xbf, 0x0e, 0x3f, 0x86, 0x1d, - 0xfb, 0xa2, 0xb9, 0x4d, 0xd8, 0x3d, 0x66, 0x96, 0x68, 0x54, 0xc1, 0xce, 0xfd, 0x8f, 0x2c, 0x18, - 0x17, 0x84, 0xc7, 0xd0, 0xec, 0xcf, 0x98, 0xcd, 0x7e, 0xb0, 0x47, 0xb3, 0x0b, 0xda, 0xfb, 0xb7, - 0x4b, 0xaa, 0xbd, 0x0d, 0xf1, 0x46, 0x7e, 0xdf, 0xd4, 0xd0, 0x63, 0xd4, 0x4e, 0x0b, 0x5b, 0xa1, - 0x2f, 0xf4, 0xb2, 0x87, 0xd2, 0x6b, 0x6a, 0x1c, 0x7e, 0x57, 0xfb, 0x8d, 0x15, 0x35, 0xbb, 0x45, - 0x15, 0x46, 0x89, 0xd8, 0x40, 0xf3, 0x5e, 0xe8, 0x77, 0x01, 0xd2, 0x87, 0xce, 0xc5, 0xbd, 0xce, - 0xc3, 0xbf, 0xfd, 0x9f, 0xde, 0x3b, 0x53, 0xbc, 0xb0, 0xc6, 0x57, 0x5e, 0x7c, 0x60, 0x75, 0x0c, - 0x9b, 0x27, 0x31, 0xd7, 0x04, 0x1c, 0x2b, 0x0a, 0xfb, 0x13, 0x4c, 0x26, 0xb3, 0x0e, 0x3a, 0xdc, - 0x95, 0xb0, 0x3f, 0x1e, 0x55, 0x5d, 0xcb, 0xdc, 0xb0, 0x75, 0xfd, 0xe2, 0x59, 0x6f, 0x11, 0x48, - 0x2b, 0xd6, 0x83, 0x7c, 0xd3, 0xdb, 0x69, 0xe8, 0xb3, 0x5d, 0x07, 0x74, 0xcf, 0xf4, 0x91, 0xa5, - 0x87, 0x38, 0x92, 0x63, 0x99, 0xee, 0x58, 0x46, 0xb0, 0xb5, 0x46, 0x36, 0x79, 0xf7, 0xb2, 0x44, - 0xe0, 0x94, 0x06, 0x2d, 0x08, 0x9b, 0x8f, 0x3b, 0x40, 0x1e, 0xcc, 0xd8, 0x7c, 0xf2, 0xf3, 0x35, - 0xa3, 0xef, 0x59, 0x18, 0x57, 0x0f, 0xa2, 0x34, 0xf8, 0xbb, 0x12, 0x15, 0xae, 0x4b, 0xad, 0xa4, - 0x60, 0xac, 0xd3, 0xa0, 0x75, 0x98, 0x8e, 0xf9, 0x6b, 0x2d, 0xf2, 0x2e, 0x82, 0xb0, 0xe8, 0x9f, - 0xcc, 0x3c, 0xa9, 0x2e, 0xd1, 0x77, 0x19, 0x88, 0x2f, 0x56, 0x79, 0x7b, 0x21, 0xcb, 0x02, 0xbd, - 0x02, 0x53, 0xbe, 0xfe, 0x6a, 0x65, 0x43, 0x18, 0xfc, 0x2a, 0xc8, 0xca, 0x78, 0xd3, 0xb2, 0x81, - 0x33, 0xd4, 0x54, 0x09, 0xd0, 0x21, 0x22, 0x49, 0x8d, 0x13, 0x6c, 0x92, 0x58, 0x3c, 0xe7, 0xc0, - 0x94, 0x80, 0x2b, 0x05, 0x34, 0xb8, 0xb0, 0x34, 0x7a, 0x11, 0x26, 0xe4, 0xe7, 0x6b, 0x77, 0x73, - 0xd2, 0x50, 0x3e, 0x0d, 0x87, 0x0d, 0x4a, 0x74, 0x1b, 0x4e, 0xc9, 0xff, 0xeb, 0x91, 0xb3, 0xb1, - 0xe1, 0xb5, 0xc4, 0xd5, 0xa8, 0x71, 0xc6, 0x62, 0x51, 0xc6, 0x35, 0xaf, 0xe4, 0x11, 0xdd, 0xdd, - 0xaf, 0x9d, 0x13, 0xbd, 0x96, 0x8b, 0x67, 0x83, 0x98, 0xcf, 0x1f, 0x5d, 0x85, 0x13, 0x5b, 0xc4, - 0xf1, 0x93, 0xad, 0xe5, 0x2d, 0xd2, 0xda, 0x96, 0x8b, 0x88, 0xdd, 0xf8, 0xd1, 0x02, 0xe0, 0x2e, - 0x75, 0x93, 0xe0, 0xbc, 0x72, 0xe8, 0x2d, 0xa8, 0xb6, 0x3b, 0xb7, 0x7c, 0x2f, 0xde, 0xba, 0x16, - 0x26, 0xec, 0x2c, 0x51, 0xbd, 0x27, 0x22, 0xae, 0x06, 0xa9, 0xdb, 0x4e, 0x8d, 0x02, 0x3a, 0x5c, - 0xc8, 0xe1, 0xfd, 0x9d, 0xf2, 0xbe, 0x4b, 0x0b, 0x6b, 0x1a, 0x06, 0xfa, 0x1c, 0x4c, 0xe8, 0x23, - 0x29, 0x84, 0xfc, 0x63, 0xfd, 0x5e, 0x49, 0x15, 0xfa, 0x89, 0x1a, 0x55, 0x1d, 0x87, 0x0d, 0x8e, - 0xf6, 0xbf, 0x28, 0x41, 0xad, 0x4f, 0x0e, 0xa9, 0x8c, 0xeb, 0xca, 0x1a, 0xc8, 0x75, 0xb5, 0x28, - 0x9f, 0x0e, 0xb9, 0x96, 0x49, 0x4c, 0x9d, 0x79, 0x16, 0x24, 0x4d, 0x4f, 0x9d, 0xa5, 0x1f, 0x38, - 0x6a, 0x4b, 0xf7, 0x7e, 0x0d, 0xf5, 0x0d, 0x5e, 0x6b, 0xe8, 0x6e, 0xcc, 0xe1, 0xc1, 0xd5, 0xdd, - 0x42, 0x0f, 0xa6, 0xfd, 0xad, 0x12, 0x9c, 0x52, 0x5d, 0xf8, 0xe3, 0xdb, 0x71, 0x37, 0xba, 0x3b, - 0xee, 0x08, 0xfc, 0xbf, 0xf6, 0x75, 0x18, 0x69, 0xee, 0xc5, 0xad, 0xc4, 0x1f, 0x40, 0x3b, 0x78, - 0xd4, 0x58, 0x39, 0xe9, 0x1e, 0xc6, 0x5e, 0xff, 0x12, 0x0b, 0xc9, 0xfe, 0xcb, 0x16, 0x4c, 0xaf, - 0x2f, 0x37, 0x9a, 0x61, 0x6b, 0x9b, 0x24, 0x8b, 0xdc, 0xbb, 0x81, 0x85, 0x72, 0x60, 0xdd, 0xe3, - 0xa6, 0x9f, 0xa7, 0x4e, 0x9c, 0x83, 0xa1, 0xad, 0x30, 0x4e, 0xb2, 0x3e, 0xfe, 0x4b, 0x61, 0x9c, - 0x60, 0x86, 0xb1, 0xff, 0xc4, 0x82, 0x61, 0xf6, 0xe0, 0x55, 0xbf, 0x87, 0xd1, 0x06, 0xf9, 0x2e, - 0xf4, 0x02, 0x8c, 0x90, 0x8d, 0x0d, 0xd2, 0x4a, 0xc4, 0xa8, 0xca, 0x8b, 0x24, 0x23, 0x2b, 0x0c, - 0x4a, 0x77, 0x44, 0x56, 0x19, 0xff, 0x8b, 0x05, 0x31, 0xfa, 0x2c, 0x54, 0x12, 0x6f, 0x87, 0x2c, - 0xba, 0xae, 0x70, 0xaf, 0x1f, 0x2e, 0x92, 0x4a, 0xed, 0xd0, 0xeb, 0x92, 0x09, 0x4e, 0xf9, 0xd9, - 0xbf, 0x50, 0x02, 0x48, 0x2f, 0x9c, 0xf5, 0xfb, 0xcc, 0xa5, 0xae, 0xf7, 0xdf, 0x1e, 0xcb, 0x79, - 0xff, 0x0d, 0xa5, 0x0c, 0x73, 0x5e, 0x7f, 0x53, 0x5d, 0x55, 0x1e, 0xa8, 0xab, 0x86, 0x0e, 0xd3, - 0x55, 0xcb, 0x30, 0x9b, 0x5e, 0x98, 0x33, 0x6f, 0x0f, 0xb3, 0xdc, 0xb0, 0xeb, 0x59, 0x24, 0xee, - 0xa6, 0xb7, 0xbf, 0x64, 0x81, 0x88, 0xae, 0x1d, 0x60, 0x42, 0xbf, 0x29, 0x9f, 0x6a, 0x32, 0x12, - 0xdb, 0x9d, 0x2b, 0x0e, 0x37, 0x16, 0xe9, 0xec, 0x94, 0x64, 0x37, 0x92, 0xd8, 0x19, 0xbc, 0xec, - 0xdf, 0xb5, 0x60, 0x9c, 0xa3, 0xaf, 0x32, 0x13, 0xb4, 0x7f, 0x6b, 0x0e, 0x95, 0x59, 0x98, 0xbd, - 0x62, 0x44, 0x19, 0xab, 0x04, 0xb4, 0xfa, 0x2b, 0x46, 0x12, 0x81, 0x53, 0x1a, 0xf4, 0x04, 0x8c, - 0xc6, 0x9d, 0x5b, 0x8c, 0x3c, 0x13, 0x60, 0xdb, 0xe4, 0x60, 0x2c, 0xf1, 0x74, 0x5e, 0xcd, 0x64, - 0xe3, 0xab, 0xd1, 0x25, 0x18, 0xe1, 0x62, 0x43, 0x2c, 0xe3, 0x1e, 0x87, 0x09, 0x5a, 0x54, 0x36, - 0xf0, 0x67, 0xb7, 0x99, 0xb8, 0x11, 0xe5, 0xd1, 0x5b, 0x30, 0xee, 0x86, 0xb7, 0x83, 0xdb, 0x4e, - 0xe4, 0x2e, 0x36, 0xd6, 0x44, 0xaf, 0xe7, 0x46, 0xc9, 0xd5, 0x53, 0x32, 0x3d, 0xd2, 0x9b, 0xb9, - 0xe7, 0x52, 0x14, 0xd6, 0xd9, 0xa1, 0x75, 0x96, 0xc3, 0x83, 0x3f, 0x06, 0xda, 0x2b, 0x6e, 0x44, - 0xbd, 0x1f, 0xaa, 0x71, 0x9e, 0x14, 0x89, 0x3e, 0xc4, 0x53, 0xa2, 0x29, 0x23, 0xfb, 0x0b, 0x27, - 0xc0, 0x18, 0x6d, 0x23, 0xff, 0xaf, 0x75, 0x44, 0xf9, 0x7f, 0x31, 0x8c, 0x91, 0x9d, 0x76, 0xb2, - 0x57, 0xf7, 0xa2, 0x5e, 0x09, 0xd9, 0x57, 0x04, 0x4d, 0x37, 0x4f, 0x89, 0xc1, 0x8a, 0x4f, 0x7e, - 0x92, 0xe6, 0xf2, 0x07, 0x98, 0xa4, 0x79, 0xe8, 0x18, 0x93, 0x34, 0x5f, 0x83, 0xd1, 0x4d, 0x2f, - 0xc1, 0xa4, 0x1d, 0x8a, 0x2d, 0x33, 0x77, 0x26, 0x5c, 0xe4, 0x24, 0xdd, 0xe9, 0x45, 0x05, 0x02, - 0x4b, 0x26, 0xe8, 0x55, 0xb5, 0x06, 0x46, 0x8a, 0x55, 0xc1, 0x6e, 0xef, 0x76, 0xee, 0x2a, 0x10, - 0x49, 0x99, 0x47, 0xef, 0x35, 0x29, 0xb3, 0x4a, 0xaa, 0x3c, 0xf6, 0xfe, 0x92, 0x2a, 0x1b, 0x49, - 0xa7, 0x2b, 0x47, 0x97, 0x74, 0xfa, 0x4b, 0x16, 0x9c, 0x6a, 0xe7, 0xe5, 0x5f, 0x17, 0x89, 0x92, - 0x5f, 0x18, 0x38, 0x0f, 0xbd, 0x51, 0x21, 0x4b, 0x24, 0x91, 0x4b, 0x86, 0xf3, 0xab, 0x93, 0xd9, - 0xab, 0xc7, 0xef, 0x35, 0x7b, 0xf5, 0xfd, 0xc9, 0xa8, 0x9c, 0xe6, 0xb2, 0x9e, 0x3c, 0xc2, 0x5c, - 0xd6, 0x53, 0xef, 0x3b, 0x97, 0xb5, 0x96, 0x8f, 0x7a, 0xfa, 0x28, 0xf2, 0x51, 0xbf, 0x6d, 0x0a, - 0x7b, 0x9e, 0x26, 0xf9, 0xa9, 0x3e, 0xc2, 0xde, 0xe0, 0xdb, 0x5b, 0xdc, 0xf3, 0xdc, 0xdb, 0xb3, - 0xf7, 0x94, 0x7b, 0xdb, 0xc8, 0x6a, 0x8d, 0x8e, 0x2e, 0xab, 0xf5, 0x4d, 0x7d, 0x0b, 0x3a, 0x51, - 0xcc, 0x57, 0xed, 0x34, 0xdd, 0x7c, 0xf3, 0x36, 0xa1, 0xee, 0x6c, 0xd9, 0x27, 0x8f, 0x27, 0x5b, - 0xf6, 0xa9, 0x23, 0xcf, 0x96, 0x7d, 0xfa, 0x18, 0xb2, 0x65, 0x3f, 0xf0, 0x81, 0x66, 0xcb, 0xae, - 0xde, 0xdf, 0x6c, 0xd9, 0x67, 0x8e, 0x22, 0x5b, 0xf6, 0x4d, 0xa8, 0xb4, 0xe5, 0x15, 0xbc, 0xea, - 0x5c, 0xf1, 0x90, 0xe4, 0xde, 0xd3, 0xe3, 0x43, 0xa2, 0x50, 0x38, 0x65, 0x45, 0xf9, 0xa6, 0xd9, - 0xb3, 0x1f, 0x2c, 0xe6, 0x9b, 0x6b, 0xb6, 0xf7, 0xc8, 0x99, 0xfd, 0x57, 0x4a, 0x70, 0xb6, 0xf7, - 0xbc, 0x4e, 0x6d, 0xfe, 0x46, 0xea, 0xc0, 0xcd, 0xd8, 0xfc, 0x4c, 0xe9, 0xd2, 0xa8, 0x06, 0xbe, - 0xa7, 0x7c, 0x11, 0x66, 0x55, 0x24, 0x92, 0xef, 0xb5, 0xf6, 0xb4, 0xc7, 0x6d, 0x54, 0x70, 0x7b, - 0x33, 0x4b, 0x80, 0xbb, 0xcb, 0xa0, 0x45, 0x98, 0x36, 0x80, 0x6b, 0x75, 0xa1, 0x92, 0x2b, 0x27, - 0x43, 0xd3, 0x44, 0xe3, 0x2c, 0xbd, 0xfd, 0x35, 0x0b, 0x1e, 0x28, 0x48, 0xbc, 0x39, 0xf0, 0x35, - 0xdc, 0x0d, 0x98, 0x6e, 0x9b, 0x45, 0xfb, 0xdc, 0xd6, 0x37, 0xd2, 0x7b, 0xaa, 0xb6, 0x66, 0x10, - 0x38, 0xcb, 0x74, 0xe9, 0xfc, 0xb7, 0xbf, 0x7f, 0xf6, 0x23, 0x7f, 0xf8, 0xfd, 0xb3, 0x1f, 0xf9, - 0xde, 0xf7, 0xcf, 0x7e, 0xe4, 0x2f, 0x1c, 0x9c, 0xb5, 0xbe, 0x7d, 0x70, 0xd6, 0xfa, 0xc3, 0x83, - 0xb3, 0xd6, 0xf7, 0x0e, 0xce, 0x5a, 0x7f, 0x7a, 0x70, 0xd6, 0xfa, 0x85, 0x1f, 0x9c, 0xfd, 0xc8, - 0x9b, 0xa5, 0xdd, 0x67, 0xff, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x81, 0xfc, 0x41, 0x31, 0xf8, - 0xc9, 0x00, 0x00, + // 11739 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x5b, 0x90, 0x24, 0xc7, + 0x71, 0x18, 0x7b, 0x66, 0x5f, 0x93, 0xfb, 0xae, 0xbb, 0x03, 0xe6, 0x16, 0xc0, 0xed, 0xa1, 0x41, + 0x02, 0x87, 0xd7, 0x2e, 0x71, 0x00, 0x08, 0x88, 0x00, 0x41, 0xee, 0xee, 0xec, 0xde, 0x0d, 0xee, + 0x35, 0xa8, 0xd9, 0x3b, 0x08, 0x20, 0x04, 0xb1, 0x6f, 0xba, 0x76, 0xb7, 0xb1, 0xbd, 0xdd, 0x83, + 0xee, 0x9e, 0xbd, 0x5b, 0x84, 0x18, 0x61, 0xd3, 0x14, 0xfd, 0xa0, 0x3e, 0x14, 0x0e, 0x85, 0x2d, + 0x8b, 0x0c, 0x39, 0xc2, 0x8f, 0x90, 0x68, 0xd9, 0x0e, 0xc9, 0x94, 0xf5, 0x20, 0xe5, 0xb0, 0x2c, + 0x3f, 0x82, 0xfc, 0xa1, 0x25, 0xff, 0x90, 0x11, 0x0e, 0xaf, 0xc4, 0xa5, 0xc3, 0x0e, 0x7d, 0xd8, + 0x61, 0x5b, 0x5f, 0x5a, 0xcb, 0xa6, 0xa3, 0x9e, 0x5d, 0xd5, 0xd3, 0x3d, 0x33, 0x7b, 0xb8, 0x5b, + 0x80, 0x0c, 0xff, 0xcd, 0x64, 0x66, 0x65, 0x55, 0xd7, 0x23, 0x2b, 0x2b, 0x2b, 0x33, 0x0b, 0x5e, + 0xda, 0x7e, 0x31, 0x5e, 0xf0, 0xc2, 0xc5, 0xed, 0xce, 0x4d, 0x12, 0x05, 0x24, 0x21, 0xf1, 0xe2, + 0x2e, 0x09, 0xdc, 0x30, 0x5a, 0x14, 0x08, 0xa7, 0xed, 0x2d, 0xb6, 0xc2, 0x88, 0x2c, 0xee, 0x3e, + 0xb3, 0xb8, 0x49, 0x02, 0x12, 0x39, 0x09, 0x71, 0x17, 0xda, 0x51, 0x98, 0x84, 0x08, 0x71, 0x9a, + 0x05, 0xa7, 0xed, 0x2d, 0x50, 0x9a, 0x85, 0xdd, 0x67, 0xe6, 0x9e, 0xde, 0xf4, 0x92, 0xad, 0xce, + 0xcd, 0x85, 0x56, 0xb8, 0xb3, 0xb8, 0x19, 0x6e, 0x86, 0x8b, 0x8c, 0xf4, 0x66, 0x67, 0x83, 0xfd, + 0x63, 0x7f, 0xd8, 0x2f, 0xce, 0x62, 0xee, 0xb9, 0xb4, 0x9a, 0x1d, 0xa7, 0xb5, 0xe5, 0x05, 0x24, + 0xda, 0x5b, 0x6c, 0x6f, 0x6f, 0xb2, 0x7a, 0x23, 0x12, 0x87, 0x9d, 0xa8, 0x45, 0xb2, 0x15, 0xf7, + 0x2c, 0x15, 0x2f, 0xee, 0x90, 0xc4, 0xc9, 0x69, 0xee, 0xdc, 0x62, 0x51, 0xa9, 0xa8, 0x13, 0x24, + 0xde, 0x4e, 0x77, 0x35, 0x9f, 0xe8, 0x57, 0x20, 0x6e, 0x6d, 0x91, 0x1d, 0xa7, 0xab, 0xdc, 0xb3, + 0x45, 0xe5, 0x3a, 0x89, 0xe7, 0x2f, 0x7a, 0x41, 0x12, 0x27, 0x51, 0xb6, 0x90, 0xfd, 0x5d, 0x0b, + 0xce, 0x2e, 0xbd, 0xde, 0x5c, 0xf5, 0x9d, 0x38, 0xf1, 0x5a, 0xcb, 0x7e, 0xd8, 0xda, 0x6e, 0x26, + 0x61, 0x44, 0x6e, 0x84, 0x7e, 0x67, 0x87, 0x34, 0x59, 0x47, 0xa0, 0xa7, 0x60, 0x6c, 0x97, 0xfd, + 0xaf, 0xd7, 0xaa, 0xd6, 0x59, 0xeb, 0x5c, 0x65, 0x79, 0xe6, 0x5b, 0xfb, 0xf3, 0x1f, 0x39, 0xd8, + 0x9f, 0x1f, 0xbb, 0x21, 0xe0, 0x58, 0x51, 0xa0, 0x47, 0x61, 0x64, 0x23, 0x5e, 0xdf, 0x6b, 0x93, + 0x6a, 0x89, 0xd1, 0x4e, 0x09, 0xda, 0x91, 0xb5, 0x26, 0x85, 0x62, 0x81, 0x45, 0x8b, 0x50, 0x69, + 0x3b, 0x51, 0xe2, 0x25, 0x5e, 0x18, 0x54, 0xcb, 0x67, 0xad, 0x73, 0xc3, 0xcb, 0xb3, 0x82, 0xb4, + 0xd2, 0x90, 0x08, 0x9c, 0xd2, 0xd0, 0x66, 0x44, 0xc4, 0x71, 0xaf, 0x05, 0xfe, 0x5e, 0x75, 0xe8, + 0xac, 0x75, 0x6e, 0x2c, 0x6d, 0x06, 0x16, 0x70, 0xac, 0x28, 0xec, 0x5f, 0x2a, 0xc1, 0xd8, 0xd2, + 0xc6, 0x86, 0x17, 0x78, 0xc9, 0x1e, 0xba, 0x01, 0x13, 0x41, 0xe8, 0x12, 0xf9, 0x9f, 0x7d, 0xc5, + 0xf8, 0xf9, 0xb3, 0x0b, 0xdd, 0x53, 0x69, 0xe1, 0xaa, 0x46, 0xb7, 0x3c, 0x73, 0xb0, 0x3f, 0x3f, + 0xa1, 0x43, 0xb0, 0xc1, 0x07, 0x61, 0x18, 0x6f, 0x87, 0xae, 0x62, 0x5b, 0x62, 0x6c, 0xe7, 0xf3, + 0xd8, 0x36, 0x52, 0xb2, 0xe5, 0xe9, 0x83, 0xfd, 0xf9, 0x71, 0x0d, 0x80, 0x75, 0x26, 0xe8, 0x26, + 0x4c, 0xd3, 0xbf, 0x41, 0xe2, 0x29, 0xbe, 0x65, 0xc6, 0xf7, 0x91, 0x22, 0xbe, 0x1a, 0xe9, 0xf2, + 0x89, 0x83, 0xfd, 0xf9, 0xe9, 0x0c, 0x10, 0x67, 0x19, 0xda, 0xef, 0xc1, 0xd4, 0x52, 0x92, 0x38, + 0xad, 0x2d, 0xe2, 0xf2, 0x11, 0x44, 0xcf, 0xc1, 0x50, 0xe0, 0xec, 0x10, 0x31, 0xbe, 0x67, 0x45, + 0xc7, 0x0e, 0x5d, 0x75, 0x76, 0xc8, 0xe1, 0xfe, 0xfc, 0xcc, 0xf5, 0xc0, 0x7b, 0xb7, 0x23, 0x66, + 0x05, 0x85, 0x61, 0x46, 0x8d, 0xce, 0x03, 0xb8, 0x64, 0xd7, 0x6b, 0x91, 0x86, 0x93, 0x6c, 0x89, + 0xf1, 0x46, 0xa2, 0x2c, 0xd4, 0x14, 0x06, 0x6b, 0x54, 0xf6, 0x6d, 0xa8, 0x2c, 0xed, 0x86, 0x9e, + 0xdb, 0x08, 0xdd, 0x18, 0x6d, 0xc3, 0x74, 0x3b, 0x22, 0x1b, 0x24, 0x52, 0xa0, 0xaa, 0x75, 0xb6, + 0x7c, 0x6e, 0xfc, 0xfc, 0xb9, 0xdc, 0x8f, 0x35, 0x49, 0x57, 0x83, 0x24, 0xda, 0x5b, 0xbe, 0x5f, + 0xd4, 0x37, 0x9d, 0xc1, 0xe2, 0x2c, 0x67, 0xfb, 0xdf, 0x94, 0xe0, 0xd4, 0xd2, 0x7b, 0x9d, 0x88, + 0xd4, 0xbc, 0x78, 0x3b, 0x3b, 0xc3, 0x5d, 0x2f, 0xde, 0xbe, 0x9a, 0xf6, 0x80, 0x9a, 0x5a, 0x35, + 0x01, 0xc7, 0x8a, 0x02, 0x3d, 0x0d, 0xa3, 0xf4, 0xf7, 0x75, 0x5c, 0x17, 0x9f, 0x7c, 0x42, 0x10, + 0x8f, 0xd7, 0x9c, 0xc4, 0xa9, 0x71, 0x14, 0x96, 0x34, 0xe8, 0x0a, 0x8c, 0xb7, 0xd8, 0x82, 0xdc, + 0xbc, 0x12, 0xba, 0x84, 0x0d, 0x66, 0x65, 0xf9, 0x49, 0x4a, 0xbe, 0x92, 0x82, 0x0f, 0xf7, 0xe7, + 0xab, 0xbc, 0x6d, 0x82, 0x85, 0x86, 0xc3, 0x7a, 0x79, 0x64, 0xab, 0xf5, 0x35, 0xc4, 0x38, 0x41, + 0xce, 0xda, 0x3a, 0xa7, 0x2d, 0x95, 0x61, 0xb6, 0x54, 0x26, 0xf2, 0x97, 0x09, 0x7a, 0x06, 0x86, + 0xb6, 0xbd, 0xc0, 0xad, 0x8e, 0x30, 0x5e, 0x0f, 0xd1, 0x31, 0xbf, 0xe4, 0x05, 0xee, 0xe1, 0xfe, + 0xfc, 0xac, 0xd1, 0x1c, 0x0a, 0xc4, 0x8c, 0xd4, 0xfe, 0x33, 0x0b, 0xe6, 0x19, 0x6e, 0xcd, 0xf3, + 0x49, 0x83, 0x44, 0xb1, 0x17, 0x27, 0x24, 0x48, 0x8c, 0x0e, 0x3d, 0x0f, 0x10, 0x93, 0x56, 0x44, + 0x12, 0xad, 0x4b, 0xd5, 0xc4, 0x68, 0x2a, 0x0c, 0xd6, 0xa8, 0xa8, 0x40, 0x88, 0xb7, 0x9c, 0x88, + 0xcd, 0x2f, 0xd1, 0xb1, 0x4a, 0x20, 0x34, 0x25, 0x02, 0xa7, 0x34, 0x86, 0x40, 0x28, 0xf7, 0x13, + 0x08, 0xe8, 0x53, 0x30, 0x9d, 0x56, 0x16, 0xb7, 0x9d, 0x96, 0xec, 0x40, 0xb6, 0x64, 0x9a, 0x26, + 0x0a, 0x67, 0x69, 0xed, 0x7f, 0x64, 0x89, 0xc9, 0x43, 0xbf, 0xfa, 0x43, 0xfe, 0xad, 0xf6, 0xef, + 0x58, 0x30, 0xba, 0xec, 0x05, 0xae, 0x17, 0x6c, 0xa2, 0xcf, 0xc1, 0x18, 0xdd, 0x9b, 0x5c, 0x27, + 0x71, 0x84, 0xdc, 0xfb, 0xb8, 0xb6, 0xb6, 0xd4, 0x56, 0xb1, 0xd0, 0xde, 0xde, 0xa4, 0x80, 0x78, + 0x81, 0x52, 0xd3, 0xd5, 0x76, 0xed, 0xe6, 0x3b, 0xa4, 0x95, 0x5c, 0x21, 0x89, 0x93, 0x7e, 0x4e, + 0x0a, 0xc3, 0x8a, 0x2b, 0xba, 0x04, 0x23, 0x89, 0x13, 0x6d, 0x92, 0x44, 0x08, 0xc0, 0x5c, 0x41, + 0xc5, 0x4b, 0x62, 0xba, 0x22, 0x49, 0xd0, 0x22, 0xe9, 0xb6, 0xb0, 0xce, 0x8a, 0x62, 0xc1, 0xc2, + 0x6e, 0xc1, 0xc4, 0x8a, 0xd3, 0x76, 0x6e, 0x7a, 0xbe, 0x97, 0x78, 0x24, 0x46, 0x8f, 0x41, 0xd9, + 0x71, 0x5d, 0x26, 0x15, 0x2a, 0xcb, 0xa7, 0x0e, 0xf6, 0xe7, 0xcb, 0x4b, 0x2e, 0x9d, 0x9e, 0xa0, + 0xa8, 0xf6, 0x30, 0xa5, 0x40, 0x4f, 0xc0, 0x90, 0x1b, 0x85, 0xed, 0x6a, 0x89, 0x51, 0xde, 0x47, + 0x67, 0x72, 0x2d, 0x0a, 0xdb, 0x19, 0x52, 0x46, 0x63, 0xff, 0x7e, 0x09, 0x1e, 0x5c, 0x21, 0xed, + 0xad, 0xb5, 0x66, 0xc1, 0xfc, 0x3d, 0x07, 0x63, 0x3b, 0x61, 0xe0, 0x25, 0x61, 0x14, 0x8b, 0xaa, + 0xd9, 0x02, 0xba, 0x22, 0x60, 0x58, 0x61, 0xd1, 0x59, 0x18, 0x6a, 0xa7, 0xc2, 0x6f, 0x42, 0x0a, + 0x4e, 0x26, 0xf6, 0x18, 0x86, 0x52, 0x74, 0x62, 0x12, 0x89, 0x85, 0xaf, 0x28, 0xae, 0xc7, 0x24, + 0xc2, 0x0c, 0x93, 0xce, 0x20, 0x3a, 0xb7, 0xc4, 0xac, 0xcc, 0xcc, 0x20, 0x8a, 0xc1, 0x1a, 0x15, + 0x6a, 0x40, 0x85, 0xff, 0xc3, 0x64, 0x83, 0xad, 0xf1, 0x82, 0x7e, 0x6f, 0x4a, 0x22, 0xd1, 0xef, + 0x93, 0x6c, 0x8a, 0x49, 0x20, 0x4e, 0x99, 0x18, 0x53, 0x6c, 0xa4, 0xef, 0x14, 0xfb, 0x66, 0x09, + 0x10, 0xef, 0xc2, 0x1f, 0xb1, 0x8e, 0xbb, 0xde, 0xdd, 0x71, 0xb9, 0x9b, 0xcd, 0xe5, 0xb0, 0xe5, + 0xf8, 0xd9, 0x59, 0x7b, 0xb7, 0x7a, 0xef, 0x17, 0x2d, 0x40, 0x2b, 0x5e, 0xe0, 0x92, 0xe8, 0x18, + 0x34, 0xad, 0xa3, 0xc9, 0x8e, 0xcb, 0x30, 0xb5, 0xe2, 0x7b, 0x24, 0x48, 0xea, 0x8d, 0x95, 0x30, + 0xd8, 0xf0, 0x36, 0xd1, 0x27, 0x61, 0x8a, 0x2a, 0x9e, 0x61, 0x27, 0x69, 0x92, 0x56, 0x18, 0xb0, + 0x3d, 0x9a, 0xaa, 0x6b, 0xe8, 0x60, 0x7f, 0x7e, 0x6a, 0xdd, 0xc0, 0xe0, 0x0c, 0xa5, 0xfd, 0x1f, + 0xe9, 0x87, 0x86, 0x3b, 0xed, 0x30, 0x20, 0x41, 0xb2, 0x12, 0x06, 0x2e, 0xd7, 0xe5, 0x3e, 0x09, + 0x43, 0x09, 0x6d, 0x38, 0xff, 0xc8, 0x47, 0xe5, 0xd0, 0xd2, 0xe6, 0x1e, 0xee, 0xcf, 0xdf, 0xd7, + 0x5d, 0x82, 0x7d, 0x10, 0x2b, 0x83, 0x7e, 0x02, 0x46, 0xe2, 0xc4, 0x49, 0x3a, 0xb1, 0xf8, 0xec, + 0x87, 0xe5, 0x67, 0x37, 0x19, 0xf4, 0x70, 0x7f, 0x7e, 0x5a, 0x15, 0xe3, 0x20, 0x2c, 0x0a, 0xa0, + 0xc7, 0x61, 0x74, 0x87, 0xc4, 0xb1, 0xb3, 0x29, 0xb7, 0xe1, 0x69, 0x51, 0x76, 0xf4, 0x0a, 0x07, + 0x63, 0x89, 0x47, 0x8f, 0xc0, 0x30, 0x89, 0xa2, 0x30, 0x12, 0xb3, 0x6a, 0x52, 0x10, 0x0e, 0xaf, + 0x52, 0x20, 0xe6, 0x38, 0xfb, 0xdf, 0x5b, 0x30, 0xad, 0xda, 0xca, 0xeb, 0x3a, 0x06, 0x79, 0xfb, + 0x26, 0x40, 0x4b, 0x7e, 0x60, 0xcc, 0xe4, 0xdd, 0xf8, 0xf9, 0x47, 0xf3, 0xa6, 0x70, 0x77, 0x37, + 0xa6, 0x9c, 0x15, 0x28, 0xc6, 0x1a, 0x37, 0xfb, 0x5f, 0x58, 0x70, 0x22, 0xf3, 0x45, 0x97, 0xbd, + 0x38, 0x41, 0x6f, 0x75, 0x7d, 0xd5, 0xc2, 0x60, 0x5f, 0x45, 0x4b, 0xb3, 0x6f, 0x52, 0x73, 0x4e, + 0x42, 0xb4, 0x2f, 0xba, 0x08, 0xc3, 0x5e, 0x42, 0x76, 0xe4, 0xc7, 0x3c, 0xd2, 0xf3, 0x63, 0x78, + 0xab, 0xd2, 0x11, 0xa9, 0xd3, 0x92, 0x98, 0x33, 0xb0, 0xff, 0xa7, 0x05, 0x15, 0x3e, 0x6d, 0xaf, + 0x38, 0xed, 0x63, 0x18, 0x8b, 0x3a, 0x0c, 0x31, 0xee, 0xbc, 0xe1, 0x8f, 0xe5, 0x37, 0x5c, 0x34, + 0x67, 0x81, 0x2a, 0x53, 0x5c, 0x69, 0x55, 0xc2, 0x8c, 0x82, 0x30, 0x63, 0x31, 0xf7, 0x02, 0x54, + 0x14, 0x01, 0x9a, 0x81, 0xf2, 0x36, 0xe1, 0x07, 0x95, 0x0a, 0xa6, 0x3f, 0xd1, 0x49, 0x18, 0xde, + 0x75, 0xfc, 0x8e, 0x58, 0xec, 0x98, 0xff, 0xf9, 0x64, 0xe9, 0x45, 0xcb, 0xfe, 0x06, 0x5b, 0x63, + 0xa2, 0x92, 0xd5, 0x60, 0x57, 0x08, 0x93, 0xf7, 0xe0, 0xa4, 0x9f, 0x23, 0xc3, 0x44, 0x47, 0x0c, + 0x2e, 0xf3, 0x1e, 0x14, 0x6d, 0x3d, 0x99, 0x87, 0xc5, 0xb9, 0x75, 0xd0, 0x6d, 0x20, 0x6c, 0xd3, + 0x19, 0xe5, 0xf8, 0xac, 0xbd, 0x42, 0x01, 0xbd, 0x26, 0x60, 0x58, 0x61, 0xa9, 0x80, 0x38, 0xa9, + 0x1a, 0x7f, 0x89, 0xec, 0x35, 0x89, 0x4f, 0x5a, 0x49, 0x18, 0x7d, 0xa0, 0xcd, 0x7f, 0x88, 0xf7, + 0x3e, 0x97, 0x2f, 0xe3, 0x82, 0x41, 0xf9, 0x12, 0xd9, 0xe3, 0x43, 0xa1, 0x7f, 0x5d, 0xb9, 0xe7, + 0xd7, 0xfd, 0x86, 0x05, 0x93, 0xea, 0xeb, 0x8e, 0x61, 0x21, 0x2d, 0x9b, 0x0b, 0xe9, 0xa1, 0x9e, + 0xf3, 0xb1, 0x60, 0x09, 0xfd, 0x90, 0x89, 0x00, 0x41, 0xd3, 0x88, 0x42, 0xda, 0x35, 0x54, 0x66, + 0x7f, 0x90, 0x03, 0x32, 0xc8, 0x77, 0x5d, 0x22, 0x7b, 0xeb, 0x21, 0x55, 0x1f, 0xf2, 0xbf, 0xcb, + 0x18, 0xb5, 0xa1, 0x9e, 0xa3, 0xf6, 0x9b, 0x25, 0x38, 0xa5, 0x7a, 0xc0, 0xd8, 0xa0, 0x7f, 0xd4, + 0xfb, 0xe0, 0x19, 0x18, 0x77, 0xc9, 0x86, 0xd3, 0xf1, 0x13, 0x75, 0x16, 0x1d, 0xe6, 0xf6, 0x88, + 0x5a, 0x0a, 0xc6, 0x3a, 0xcd, 0x11, 0xba, 0xed, 0xdf, 0x02, 0x93, 0xbd, 0x89, 0x43, 0x67, 0x30, + 0xd5, 0xde, 0x34, 0x8b, 0xc2, 0x84, 0x6e, 0x51, 0x10, 0xd6, 0x83, 0x47, 0x60, 0xd8, 0xdb, 0xa1, + 0x7b, 0x71, 0xc9, 0xdc, 0x62, 0xeb, 0x14, 0x88, 0x39, 0x0e, 0x7d, 0x0c, 0x46, 0x5b, 0xe1, 0xce, + 0x8e, 0x13, 0xb8, 0xd5, 0x32, 0xd3, 0x27, 0xc7, 0xe9, 0x76, 0xbd, 0xc2, 0x41, 0x58, 0xe2, 0xd0, + 0x83, 0x30, 0xe4, 0x44, 0x9b, 0x71, 0x75, 0x88, 0xd1, 0x8c, 0xd1, 0x9a, 0x96, 0xa2, 0xcd, 0x18, + 0x33, 0x28, 0xd5, 0x13, 0x6f, 0x85, 0xd1, 0xb6, 0x17, 0x6c, 0xd6, 0xbc, 0x88, 0x29, 0x7d, 0x9a, + 0x9e, 0xf8, 0xba, 0xc2, 0x60, 0x8d, 0x0a, 0xad, 0xc1, 0x70, 0x3b, 0x8c, 0x92, 0xb8, 0x3a, 0xc2, + 0xba, 0xfb, 0xe1, 0x82, 0xa5, 0xc4, 0xbf, 0xb6, 0x11, 0x46, 0x49, 0xfa, 0x01, 0xf4, 0x5f, 0x8c, + 0x79, 0x71, 0xf4, 0x13, 0x50, 0x26, 0xc1, 0x6e, 0x75, 0x94, 0x71, 0x99, 0xcb, 0xe3, 0xb2, 0x1a, + 0xec, 0xde, 0x70, 0xa2, 0x54, 0xce, 0xac, 0x06, 0xbb, 0x98, 0x96, 0x41, 0x6f, 0x40, 0x45, 0x5a, + 0x23, 0xe3, 0xea, 0x58, 0xf1, 0x14, 0xc3, 0x82, 0x08, 0x93, 0x77, 0x3b, 0x5e, 0x44, 0x76, 0x48, + 0x90, 0xc4, 0xe9, 0x79, 0x52, 0x62, 0x63, 0x9c, 0x72, 0x43, 0x6f, 0xc0, 0x04, 0xd7, 0x23, 0xaf, + 0x84, 0x9d, 0x20, 0x89, 0xab, 0x15, 0xd6, 0xbc, 0x5c, 0xd3, 0xd5, 0x8d, 0x94, 0x6e, 0xf9, 0xa4, + 0x60, 0x3a, 0xa1, 0x01, 0x63, 0x6c, 0xb0, 0x42, 0x18, 0x26, 0x7d, 0x6f, 0x97, 0x04, 0x24, 0x8e, + 0x1b, 0x51, 0x78, 0x93, 0x54, 0x81, 0xb5, 0xfc, 0x74, 0xbe, 0x45, 0x27, 0xbc, 0x49, 0x96, 0x67, + 0x0f, 0xf6, 0xe7, 0x27, 0x2f, 0xeb, 0x65, 0xb0, 0xc9, 0x02, 0x5d, 0x87, 0x29, 0xaa, 0xa0, 0x7a, + 0x29, 0xd3, 0xf1, 0x7e, 0x4c, 0x99, 0x76, 0x8a, 0x8d, 0x42, 0x38, 0xc3, 0x04, 0xbd, 0x0a, 0x15, + 0xdf, 0xdb, 0x20, 0xad, 0xbd, 0x96, 0x4f, 0xaa, 0x13, 0x8c, 0x63, 0xee, 0xb2, 0xba, 0x2c, 0x89, + 0xf8, 0x01, 0x40, 0xfd, 0xc5, 0x69, 0x71, 0x74, 0x03, 0xee, 0x4b, 0x48, 0xb4, 0xe3, 0x05, 0x0e, + 0x5d, 0x0e, 0x42, 0x9f, 0x64, 0x76, 0xb1, 0x49, 0x36, 0xdf, 0xce, 0x88, 0xae, 0xbb, 0x6f, 0x3d, + 0x97, 0x0a, 0x17, 0x94, 0x46, 0xd7, 0x60, 0x9a, 0xad, 0x84, 0x46, 0xc7, 0xf7, 0x1b, 0xa1, 0xef, + 0xb5, 0xf6, 0xaa, 0x53, 0x8c, 0xe1, 0xc7, 0xa4, 0xe1, 0xab, 0x6e, 0xa2, 0xe9, 0x89, 0x37, 0xfd, + 0x87, 0xb3, 0xa5, 0xd1, 0x4d, 0x66, 0x08, 0xe9, 0x44, 0x5e, 0xb2, 0x47, 0xe7, 0x2f, 0xb9, 0x9d, + 0x54, 0xa7, 0x7b, 0x9e, 0x1f, 0x75, 0x52, 0x65, 0x2d, 0xd1, 0x81, 0x38, 0xcb, 0x90, 0x2e, 0xed, + 0x38, 0x71, 0xbd, 0xa0, 0x3a, 0xc3, 0x24, 0x86, 0x5a, 0x19, 0x4d, 0x0a, 0xc4, 0x1c, 0xc7, 0x8c, + 0x20, 0xf4, 0xc7, 0x35, 0x2a, 0x41, 0x67, 0x19, 0x61, 0x6a, 0x04, 0x91, 0x08, 0x9c, 0xd2, 0xd0, + 0x6d, 0x39, 0x49, 0xf6, 0xaa, 0x88, 0x91, 0xaa, 0xe5, 0xb2, 0xbe, 0xfe, 0x06, 0xa6, 0x70, 0x74, + 0x19, 0x46, 0x49, 0xb0, 0xbb, 0x16, 0x85, 0x3b, 0xd5, 0x13, 0xc5, 0x6b, 0x76, 0x95, 0x93, 0x70, + 0x81, 0x9e, 0x1e, 0x00, 0x04, 0x18, 0x4b, 0x16, 0xe8, 0x36, 0x54, 0x73, 0x46, 0x84, 0x0f, 0xc0, + 0x49, 0x36, 0x00, 0x2f, 0x8b, 0xb2, 0xd5, 0xf5, 0x02, 0xba, 0xc3, 0x1e, 0x38, 0x5c, 0xc8, 0xdd, + 0xbe, 0x09, 0x53, 0x4a, 0xb0, 0xb0, 0xb1, 0x45, 0xf3, 0x30, 0x4c, 0x25, 0xa6, 0x3c, 0x52, 0x57, + 0x68, 0x57, 0x32, 0xd3, 0x14, 0xe6, 0x70, 0xd6, 0x95, 0xde, 0x7b, 0x64, 0x79, 0x2f, 0x21, 0xfc, + 0x58, 0x54, 0xd6, 0xba, 0x52, 0x22, 0x70, 0x4a, 0x63, 0xff, 0x5f, 0xae, 0x98, 0xa4, 0xd2, 0x6b, + 0x00, 0x79, 0xfd, 0x14, 0x8c, 0x6d, 0x85, 0x71, 0x42, 0xa9, 0x59, 0x1d, 0xc3, 0xa9, 0x2a, 0x72, + 0x51, 0xc0, 0xb1, 0xa2, 0x40, 0x2f, 0xc1, 0x64, 0x4b, 0xaf, 0x40, 0x6c, 0x36, 0xa7, 0x44, 0x11, + 0xb3, 0x76, 0x6c, 0xd2, 0xa2, 0x17, 0x61, 0x8c, 0x5d, 0x50, 0xb4, 0x42, 0x5f, 0x1c, 0xc0, 0xe4, + 0x8e, 0x39, 0xd6, 0x10, 0xf0, 0x43, 0xed, 0x37, 0x56, 0xd4, 0xf4, 0x50, 0x4c, 0x9b, 0x50, 0x6f, + 0x08, 0x31, 0xaf, 0x0e, 0xc5, 0x17, 0x19, 0x14, 0x0b, 0xac, 0xfd, 0x37, 0x4b, 0x5a, 0x2f, 0xd3, + 0x23, 0x05, 0x41, 0x0d, 0x18, 0xbd, 0xe5, 0x78, 0x89, 0x17, 0x6c, 0x8a, 0xfd, 0xfc, 0xf1, 0x9e, + 0x32, 0x9f, 0x15, 0x7a, 0x9d, 0x17, 0xe0, 0xbb, 0x92, 0xf8, 0x83, 0x25, 0x1b, 0xca, 0x31, 0xea, + 0x04, 0x01, 0xe5, 0x58, 0x1a, 0x94, 0x23, 0xe6, 0x05, 0x38, 0x47, 0xf1, 0x07, 0x4b, 0x36, 0xe8, + 0x2d, 0x00, 0x39, 0x6f, 0x88, 0x2b, 0x2e, 0x06, 0x9e, 0xea, 0xcf, 0x74, 0x5d, 0x95, 0x59, 0x9e, + 0xa2, 0x7b, 0x5e, 0xfa, 0x1f, 0x6b, 0xfc, 0xec, 0x84, 0xe9, 0x3d, 0xdd, 0x8d, 0x41, 0x9f, 0xa5, + 0x4b, 0xd5, 0x89, 0x12, 0xe2, 0x2e, 0x25, 0xa2, 0x73, 0x9e, 0x18, 0x4c, 0x6d, 0x5d, 0xf7, 0x76, + 0x88, 0xbe, 0xac, 0x05, 0x13, 0x9c, 0xf2, 0xb3, 0x7f, 0xbb, 0x0c, 0xd5, 0xa2, 0xe6, 0xd2, 0x49, + 0x47, 0x6e, 0x7b, 0xc9, 0x0a, 0x55, 0x57, 0x2c, 0x73, 0xd2, 0xad, 0x0a, 0x38, 0x56, 0x14, 0x74, + 0xf4, 0x63, 0x6f, 0x53, 0x9e, 0x3a, 0x86, 0xd3, 0xd1, 0x6f, 0x32, 0x28, 0x16, 0x58, 0x4a, 0x17, + 0x11, 0x27, 0x16, 0x37, 0x4f, 0xda, 0x2c, 0xc1, 0x0c, 0x8a, 0x05, 0x56, 0x37, 0x18, 0x0c, 0xf5, + 0x31, 0x18, 0x18, 0x5d, 0x34, 0x7c, 0x77, 0xbb, 0x08, 0xbd, 0x0d, 0xb0, 0xe1, 0x05, 0x5e, 0xbc, + 0xc5, 0xb8, 0x8f, 0x1c, 0x99, 0xbb, 0x52, 0x76, 0xd6, 0x14, 0x17, 0xac, 0x71, 0x44, 0xcf, 0xc3, + 0xb8, 0x5a, 0x80, 0xf5, 0x5a, 0x75, 0xd4, 0xbc, 0xd6, 0x48, 0xa5, 0x51, 0x0d, 0xeb, 0x74, 0xf6, + 0x3b, 0xd9, 0xf9, 0x22, 0x56, 0x80, 0xd6, 0xbf, 0xd6, 0xa0, 0xfd, 0x5b, 0xea, 0xdd, 0xbf, 0xf6, + 0x1f, 0x94, 0x61, 0xda, 0xa8, 0xac, 0x13, 0x0f, 0x20, 0xb3, 0x2e, 0xd0, 0x8d, 0xc8, 0x49, 0x88, + 0x58, 0x7f, 0x76, 0xff, 0xa5, 0xa2, 0x6f, 0x56, 0x74, 0x05, 0xf0, 0xf2, 0xe8, 0x6d, 0xa8, 0xf8, + 0x4e, 0xcc, 0x8c, 0x0f, 0x44, 0xac, 0xbb, 0x41, 0x98, 0xa5, 0x8a, 0xbe, 0x13, 0x27, 0xda, 0x5e, + 0xc0, 0x79, 0xa7, 0x2c, 0xe9, 0x8e, 0x49, 0x95, 0x13, 0x79, 0xb5, 0xa9, 0x1a, 0x41, 0x35, 0x98, + 0x3d, 0xcc, 0x71, 0xe8, 0x45, 0x98, 0x88, 0x08, 0x9b, 0x15, 0x2b, 0x54, 0xd7, 0x62, 0xd3, 0x6c, + 0x38, 0x55, 0xca, 0xb0, 0x86, 0xc3, 0x06, 0x65, 0xaa, 0x6b, 0x8f, 0xf4, 0xd0, 0xb5, 0x1f, 0x87, + 0x51, 0xf6, 0x43, 0xcd, 0x00, 0x35, 0x1a, 0x75, 0x0e, 0xc6, 0x12, 0x9f, 0x9d, 0x30, 0x63, 0x03, + 0x4e, 0x98, 0x27, 0x60, 0xaa, 0xe6, 0x90, 0x9d, 0x30, 0x58, 0x0d, 0xdc, 0x76, 0xe8, 0x05, 0x09, + 0xaa, 0xc2, 0x10, 0xdb, 0x1d, 0xf8, 0xda, 0x1e, 0xa2, 0x1c, 0xf0, 0x10, 0xd5, 0x9c, 0xed, 0x3f, + 0x2a, 0xc1, 0x64, 0x8d, 0xf8, 0x24, 0x21, 0xfc, 0xac, 0x11, 0xa3, 0x35, 0x40, 0x9b, 0x91, 0xd3, + 0x22, 0x0d, 0x12, 0x79, 0xa1, 0xab, 0x1b, 0x23, 0xcb, 0xcc, 0xe0, 0x8f, 0x2e, 0x74, 0x61, 0x71, + 0x4e, 0x09, 0xf4, 0x26, 0x4c, 0xb6, 0x23, 0x62, 0xd8, 0xd0, 0xac, 0x22, 0x75, 0xa1, 0xa1, 0x13, + 0x72, 0x4d, 0xd5, 0x00, 0x61, 0x93, 0x15, 0xfa, 0x0c, 0xcc, 0x84, 0x51, 0x7b, 0xcb, 0x09, 0x6a, + 0xa4, 0x4d, 0x02, 0x97, 0xaa, 0xe2, 0xc2, 0x46, 0x70, 0xf2, 0x60, 0x7f, 0x7e, 0xe6, 0x5a, 0x06, + 0x87, 0xbb, 0xa8, 0xd1, 0x9b, 0x30, 0xdb, 0x8e, 0xc2, 0xb6, 0xb3, 0xc9, 0x26, 0x8a, 0xd0, 0x38, + 0xb8, 0xf4, 0x79, 0xea, 0x60, 0x7f, 0x7e, 0xb6, 0x91, 0x45, 0x1e, 0xee, 0xcf, 0x9f, 0x60, 0x1d, + 0x45, 0x21, 0x29, 0x12, 0x77, 0xb3, 0xb1, 0x37, 0xe1, 0x54, 0x2d, 0xbc, 0x15, 0xdc, 0x72, 0x22, + 0x77, 0xa9, 0x51, 0xd7, 0x0e, 0xf7, 0x57, 0xe5, 0xe1, 0x92, 0x5f, 0xbf, 0xe6, 0xee, 0x53, 0x5a, + 0x49, 0xae, 0xfe, 0xaf, 0x79, 0x3e, 0x29, 0x30, 0x22, 0xfc, 0xed, 0x92, 0x51, 0x53, 0x4a, 0xaf, + 0xec, 0xfe, 0x56, 0xa1, 0xdd, 0xff, 0x35, 0x18, 0xdb, 0xf0, 0x88, 0xef, 0x62, 0xb2, 0x21, 0x46, + 0xe6, 0xb1, 0xe2, 0x1b, 0xa5, 0x35, 0x4a, 0x29, 0x8d, 0x46, 0xfc, 0x68, 0xba, 0x26, 0x0a, 0x63, + 0xc5, 0x06, 0x6d, 0xc3, 0x8c, 0x3c, 0xfb, 0x48, 0xac, 0x58, 0xc4, 0x8f, 0xf7, 0x3a, 0x50, 0x99, + 0xcc, 0xd9, 0x00, 0xe2, 0x0c, 0x1b, 0xdc, 0xc5, 0x98, 0x9e, 0x45, 0x77, 0xe8, 0x76, 0x35, 0xc4, + 0xa6, 0x34, 0x3b, 0x8b, 0xb2, 0x63, 0x35, 0x83, 0xda, 0x5f, 0xb5, 0xe0, 0xfe, 0xae, 0x9e, 0x11, + 0xe6, 0x85, 0xbb, 0x3c, 0x0a, 0xd9, 0xe3, 0x7e, 0xa9, 0xff, 0x71, 0xdf, 0xfe, 0x75, 0x0b, 0x4e, + 0xae, 0xee, 0xb4, 0x93, 0xbd, 0x9a, 0x67, 0xde, 0x4d, 0xbc, 0x00, 0x23, 0x3b, 0xc4, 0xf5, 0x3a, + 0x3b, 0x62, 0xe4, 0xe6, 0xa5, 0x48, 0xbf, 0xc2, 0xa0, 0x87, 0xfb, 0xf3, 0x93, 0xcd, 0x24, 0x8c, + 0x9c, 0x4d, 0xc2, 0x01, 0x58, 0x90, 0xa3, 0x9f, 0xe6, 0xba, 0xe9, 0x65, 0x6f, 0xc7, 0x93, 0x37, + 0x84, 0x3d, 0x4d, 0x5e, 0x0b, 0xb2, 0x43, 0x17, 0x5e, 0xeb, 0x38, 0x41, 0xe2, 0x25, 0x7b, 0xa6, + 0x2e, 0xcb, 0x18, 0xe1, 0x94, 0xa7, 0xfd, 0x5d, 0x0b, 0xa6, 0xa5, 0x3c, 0x59, 0x72, 0xdd, 0x88, + 0xc4, 0x31, 0x9a, 0x83, 0x92, 0xd7, 0x16, 0x2d, 0x05, 0x51, 0xba, 0x54, 0x6f, 0xe0, 0x92, 0xd7, + 0x46, 0x0d, 0xa8, 0xf0, 0xcb, 0xc6, 0x74, 0x82, 0x0d, 0x74, 0x65, 0xc9, 0xce, 0x7e, 0xeb, 0xb2, + 0x24, 0x4e, 0x99, 0x48, 0xcd, 0x98, 0xed, 0x45, 0x65, 0xf3, 0xde, 0xe6, 0xa2, 0x80, 0x63, 0x45, + 0x81, 0xce, 0xc1, 0x58, 0x10, 0xba, 0xfc, 0xee, 0x97, 0xaf, 0x6b, 0x36, 0x6d, 0xaf, 0x0a, 0x18, + 0x56, 0x58, 0xfb, 0xe7, 0x2c, 0x98, 0x90, 0x5f, 0x36, 0xa0, 0x92, 0x4e, 0x97, 0x57, 0xaa, 0xa0, + 0xa7, 0xcb, 0x8b, 0x2a, 0xd9, 0x0c, 0x63, 0xe8, 0xd6, 0xe5, 0xa3, 0xe8, 0xd6, 0xf6, 0x57, 0x4a, + 0x30, 0x25, 0x9b, 0xd3, 0xec, 0xdc, 0x8c, 0x49, 0x82, 0xd6, 0xa1, 0xe2, 0xf0, 0x2e, 0x27, 0x72, + 0xd6, 0x3e, 0x92, 0x7f, 0xea, 0x32, 0xc6, 0x27, 0x1d, 0xd1, 0x25, 0x59, 0x1a, 0xa7, 0x8c, 0x90, + 0x0f, 0xb3, 0x41, 0x98, 0xb0, 0xad, 0x4f, 0xe1, 0x7b, 0xdd, 0x0d, 0x64, 0xb9, 0x9f, 0x16, 0xdc, + 0x67, 0xaf, 0x66, 0xb9, 0xe0, 0x6e, 0xc6, 0x68, 0x55, 0x5a, 0x7a, 0xca, 0xac, 0x86, 0xb3, 0xbd, + 0x6a, 0x28, 0x36, 0xf4, 0xd8, 0xbf, 0x67, 0x41, 0x45, 0x92, 0x1d, 0xc7, 0x35, 0xd0, 0x15, 0x18, + 0x8d, 0xd9, 0x20, 0xc8, 0xae, 0xb1, 0x7b, 0x35, 0x9c, 0x8f, 0x57, 0xba, 0xa3, 0xf3, 0xff, 0x31, + 0x96, 0x3c, 0x98, 0xa9, 0x5a, 0x35, 0xff, 0x43, 0x62, 0xaa, 0x56, 0xed, 0x29, 0xd8, 0x65, 0xfe, + 0x2b, 0x6b, 0xb3, 0x76, 0x9e, 0xa7, 0x8a, 0x67, 0x3b, 0x22, 0x1b, 0xde, 0xed, 0xac, 0xe2, 0xd9, + 0x60, 0x50, 0x2c, 0xb0, 0xe8, 0x2d, 0x98, 0x68, 0x49, 0x0b, 0x6f, 0x2a, 0x06, 0x1e, 0xed, 0x69, + 0x2f, 0x57, 0x57, 0x2b, 0xdc, 0x2f, 0x6c, 0x45, 0x2b, 0x8f, 0x0d, 0x6e, 0xe6, 0xe5, 0x7c, 0xb9, + 0xdf, 0xe5, 0x7c, 0xca, 0xb7, 0xf0, 0x7a, 0xd9, 0xfe, 0x65, 0x0b, 0x46, 0xb8, 0x9d, 0x70, 0x30, + 0xc3, 0xaa, 0x76, 0x55, 0x94, 0xf6, 0xdd, 0x0d, 0x0a, 0x14, 0x37, 0x47, 0xe8, 0x0a, 0x54, 0xd8, + 0x0f, 0x66, 0x2f, 0x29, 0x17, 0x3b, 0xc4, 0xf1, 0x5a, 0xf5, 0x06, 0xde, 0x90, 0xc5, 0x70, 0xca, + 0xc1, 0xfe, 0x85, 0x32, 0x15, 0x55, 0x29, 0xa9, 0xb1, 0x8b, 0x5b, 0xf7, 0x6e, 0x17, 0x2f, 0xdd, + 0xab, 0x5d, 0x7c, 0x13, 0xa6, 0x5b, 0xda, 0xbd, 0x54, 0x3a, 0x92, 0xe7, 0x7a, 0x4e, 0x12, 0xed, + 0x0a, 0x8b, 0xdb, 0xca, 0x56, 0x4c, 0x26, 0x38, 0xcb, 0x15, 0x7d, 0x16, 0x26, 0xf8, 0x38, 0x8b, + 0x5a, 0x86, 0x58, 0x2d, 0x1f, 0x2b, 0x9e, 0x2f, 0x7a, 0x15, 0x6c, 0x26, 0x36, 0xb5, 0xe2, 0xd8, + 0x60, 0x66, 0x7f, 0x69, 0x18, 0x86, 0x57, 0x77, 0x49, 0x90, 0x1c, 0x83, 0x40, 0x6a, 0xc1, 0x94, + 0x17, 0xec, 0x86, 0xfe, 0x2e, 0x71, 0x39, 0xfe, 0x28, 0x9b, 0xeb, 0x7d, 0x82, 0xf5, 0x54, 0xdd, + 0x60, 0x81, 0x33, 0x2c, 0xef, 0xc5, 0xc9, 0xfd, 0x02, 0x8c, 0xf0, 0xb1, 0x17, 0xc7, 0xf6, 0x5c, + 0x2b, 0x38, 0xeb, 0x44, 0xb1, 0x0a, 0x52, 0xab, 0x02, 0x37, 0xbb, 0x8b, 0xe2, 0xe8, 0x1d, 0x98, + 0xda, 0xf0, 0xa2, 0x38, 0xa1, 0x47, 0xee, 0x38, 0x71, 0x76, 0xda, 0x77, 0x70, 0x52, 0x57, 0xfd, + 0xb0, 0x66, 0x70, 0xc2, 0x19, 0xce, 0x68, 0x13, 0x26, 0xe9, 0xe1, 0x31, 0xad, 0x6a, 0xf4, 0xc8, + 0x55, 0x29, 0x53, 0xdc, 0x65, 0x9d, 0x11, 0x36, 0xf9, 0x52, 0x61, 0xd2, 0x62, 0x87, 0xcd, 0x31, + 0xa6, 0x51, 0x28, 0x61, 0xc2, 0x4f, 0x99, 0x1c, 0x47, 0x65, 0x12, 0xf3, 0xe7, 0xa8, 0x98, 0x32, + 0x29, 0xf5, 0xda, 0xb0, 0xbf, 0x46, 0x77, 0x47, 0xda, 0x87, 0xc7, 0xb0, 0xb5, 0xbc, 0x62, 0x6e, + 0x2d, 0xa7, 0x0b, 0xc7, 0xb3, 0x60, 0x5b, 0xf9, 0x1c, 0x8c, 0x6b, 0xc3, 0x8d, 0x16, 0xa1, 0xd2, + 0x92, 0xce, 0x07, 0x42, 0xea, 0x2a, 0xf5, 0x45, 0x79, 0x25, 0xe0, 0x94, 0x86, 0xf6, 0x06, 0x55, + 0xf6, 0xb2, 0xae, 0x4d, 0x54, 0x15, 0xc4, 0x0c, 0x63, 0x3f, 0x0b, 0xb0, 0x7a, 0x9b, 0xb4, 0x96, + 0xf8, 0xe1, 0x4b, 0xbb, 0xe3, 0xb2, 0x8a, 0xef, 0xb8, 0xec, 0xff, 0x60, 0xc1, 0xd4, 0xda, 0x8a, + 0xa1, 0x94, 0x2f, 0x00, 0x70, 0x2d, 0xf4, 0xf5, 0xd7, 0xaf, 0x4a, 0xeb, 0x30, 0x37, 0xf0, 0x29, + 0x28, 0xd6, 0x28, 0xd0, 0x69, 0x28, 0xfb, 0x9d, 0x40, 0x28, 0x87, 0xa3, 0x07, 0xfb, 0xf3, 0xe5, + 0xcb, 0x9d, 0x00, 0x53, 0x98, 0xe6, 0x4d, 0x54, 0x1e, 0xd8, 0x9b, 0xa8, 0xaf, 0x1b, 0x36, 0x9a, + 0x87, 0xe1, 0x5b, 0xb7, 0x3c, 0x37, 0xae, 0x0e, 0xa7, 0x96, 0xeb, 0xd7, 0x5f, 0xaf, 0xd7, 0x62, + 0xcc, 0xe1, 0xf6, 0x5f, 0x2e, 0xc3, 0xcc, 0x9a, 0x4f, 0x6e, 0x1b, 0x9f, 0xf5, 0x28, 0x8c, 0xb8, + 0x91, 0xb7, 0x4b, 0xa2, 0xec, 0x2e, 0x5e, 0x63, 0x50, 0x2c, 0xb0, 0x03, 0x7b, 0x40, 0x5d, 0xef, + 0xde, 0x8f, 0xef, 0xb6, 0xcf, 0x57, 0xff, 0xae, 0x78, 0x0b, 0x46, 0xf9, 0x55, 0x29, 0xef, 0x8c, + 0xf1, 0xf3, 0xcf, 0xe4, 0x35, 0x21, 0xdb, 0x17, 0x0b, 0xc2, 0xf8, 0xc1, 0xfd, 0x46, 0x94, 0x10, + 0x13, 0x50, 0x2c, 0x59, 0xce, 0x7d, 0x12, 0x26, 0x74, 0xca, 0x23, 0x39, 0x90, 0xfc, 0x15, 0x0b, + 0x4e, 0xac, 0xf9, 0x61, 0x6b, 0x3b, 0xe3, 0x8e, 0xf6, 0x3c, 0x8c, 0xd3, 0xf5, 0x14, 0x1b, 0xae, + 0xad, 0x86, 0xb3, 0xb3, 0x40, 0x61, 0x9d, 0x4e, 0x2b, 0x76, 0xfd, 0x7a, 0xbd, 0x96, 0xe7, 0x23, + 0x2d, 0x50, 0x58, 0xa7, 0xb3, 0xbf, 0x63, 0xc1, 0x43, 0x17, 0x56, 0x56, 0x53, 0x8f, 0xcc, 0x2e, + 0x37, 0x6d, 0xaa, 0xdc, 0xb9, 0x5a, 0x53, 0x52, 0xe5, 0xae, 0xc6, 0x5a, 0x21, 0xb0, 0x1f, 0x96, + 0x10, 0x84, 0x5f, 0xb5, 0xe0, 0xc4, 0x05, 0x2f, 0xc1, 0xa4, 0x1d, 0x66, 0x1d, 0x86, 0x23, 0xd2, + 0x0e, 0x63, 0x2f, 0x09, 0xa3, 0xbd, 0xac, 0xc3, 0x30, 0x56, 0x18, 0xac, 0x51, 0xf1, 0x9a, 0x77, + 0xbd, 0x98, 0xb6, 0xb4, 0x64, 0x9e, 0x30, 0xb1, 0x80, 0x63, 0x45, 0x41, 0x3f, 0xcc, 0xf5, 0x22, + 0xa6, 0x21, 0xec, 0x89, 0xe5, 0xac, 0x3e, 0xac, 0x26, 0x11, 0x38, 0xa5, 0xb1, 0xbf, 0x6a, 0xc1, + 0xa9, 0x0b, 0x7e, 0x27, 0x4e, 0x48, 0xb4, 0x11, 0x1b, 0x8d, 0x7d, 0x16, 0x2a, 0x44, 0x6a, 0xe1, + 0xa2, 0xad, 0x6a, 0xdf, 0x50, 0xea, 0x39, 0xf7, 0x56, 0x56, 0x74, 0x03, 0xf8, 0x76, 0x1e, 0xcd, + 0x27, 0xf1, 0xeb, 0x25, 0x98, 0xbc, 0xb8, 0xbe, 0xde, 0xb8, 0x40, 0x12, 0x21, 0x32, 0xfb, 0x5b, + 0x91, 0xb0, 0x76, 0x10, 0xee, 0xa5, 0xeb, 0x74, 0x12, 0xcf, 0x5f, 0xe0, 0xe1, 0x31, 0x0b, 0xf5, + 0x20, 0xb9, 0x16, 0x35, 0x93, 0xc8, 0x0b, 0x36, 0x73, 0x8f, 0xce, 0x52, 0xb0, 0x97, 0x8b, 0x04, + 0x3b, 0x7a, 0x16, 0x46, 0x58, 0x7c, 0x8e, 0xd4, 0x3a, 0x1e, 0x50, 0xaa, 0x02, 0x83, 0x1e, 0xee, + 0xcf, 0x57, 0xae, 0xe3, 0x3a, 0xff, 0x83, 0x05, 0x29, 0xba, 0x0e, 0xe3, 0x5b, 0x49, 0xd2, 0xbe, + 0x48, 0x1c, 0x97, 0x44, 0x52, 0x3a, 0x9c, 0xc9, 0x93, 0x0e, 0xb4, 0x13, 0x38, 0x59, 0xba, 0xa0, + 0x52, 0x58, 0x8c, 0x75, 0x3e, 0x76, 0x13, 0x20, 0xc5, 0xdd, 0xa5, 0x63, 0x83, 0xfd, 0x03, 0x0b, + 0x46, 0x2f, 0x3a, 0x81, 0xeb, 0x93, 0x08, 0xbd, 0x0c, 0x43, 0xe4, 0x36, 0x69, 0x89, 0x1d, 0x3c, + 0xb7, 0xc1, 0xe9, 0x2e, 0xc7, 0x0d, 0x61, 0xf4, 0x3f, 0x66, 0xa5, 0xd0, 0x45, 0x18, 0xa5, 0xad, + 0xbd, 0xa0, 0xfc, 0xc6, 0x1f, 0x2e, 0xfa, 0x62, 0x35, 0xec, 0x7c, 0x63, 0x14, 0x20, 0x2c, 0x8b, + 0x33, 0x83, 0x4e, 0xab, 0xdd, 0xa4, 0x02, 0x2c, 0xe9, 0x75, 0xdc, 0x5a, 0x5f, 0x69, 0x70, 0x22, + 0xc1, 0x8d, 0x1b, 0x74, 0x24, 0x10, 0xa7, 0x4c, 0xec, 0x75, 0xa8, 0xd0, 0x41, 0x5d, 0xf2, 0x3d, + 0xa7, 0xb7, 0x2d, 0xe9, 0x49, 0xa8, 0x48, 0xbb, 0x4e, 0x2c, 0x5c, 0xcf, 0x19, 0x57, 0x69, 0xf6, + 0x89, 0x71, 0x8a, 0xb7, 0x37, 0xe0, 0x24, 0xbb, 0x28, 0x75, 0x92, 0x2d, 0x63, 0x8d, 0xf5, 0x9f, + 0xcc, 0x4f, 0x09, 0xfd, 0x8a, 0x8f, 0x4c, 0x55, 0xf3, 0x95, 0x9d, 0x90, 0x1c, 0x35, 0x5d, 0xeb, + 0x3f, 0x0f, 0xc1, 0x6c, 0xbd, 0xb9, 0xd2, 0x34, 0x8d, 0x8b, 0x2f, 0xc2, 0x04, 0xd7, 0x04, 0xe8, + 0x84, 0x76, 0x7c, 0x51, 0x9b, 0xba, 0x3c, 0x58, 0xd7, 0x70, 0xd8, 0xa0, 0x44, 0x0f, 0x41, 0xd9, + 0x7b, 0x37, 0xc8, 0xba, 0xc3, 0xd5, 0x5f, 0xbb, 0x8a, 0x29, 0x9c, 0xa2, 0xa9, 0x52, 0xc1, 0x05, + 0xa8, 0x42, 0x2b, 0xc5, 0xe2, 0x15, 0x98, 0xf2, 0xe2, 0x56, 0xec, 0xd5, 0x03, 0x2a, 0x5d, 0xd2, + 0xb8, 0x8b, 0x54, 0xe3, 0xa7, 0x4d, 0x55, 0x58, 0x9c, 0xa1, 0xd6, 0xa4, 0xf9, 0xf0, 0xc0, 0x8a, + 0x49, 0x5f, 0x0f, 0x6c, 0xaa, 0x73, 0xb5, 0xd9, 0xd7, 0xc5, 0xcc, 0x35, 0x47, 0xe8, 0x5c, 0xfc, + 0x83, 0x63, 0x2c, 0x71, 0xe8, 0x02, 0xcc, 0xb6, 0xb6, 0x9c, 0xf6, 0x52, 0x27, 0xd9, 0xaa, 0x79, + 0x71, 0x2b, 0xdc, 0x25, 0xd1, 0x1e, 0xd3, 0x84, 0xc7, 0x52, 0x23, 0x93, 0x42, 0xac, 0x5c, 0x5c, + 0x6a, 0x50, 0x4a, 0xdc, 0x5d, 0xc6, 0x54, 0x41, 0xe0, 0xae, 0xa9, 0x20, 0x4b, 0x30, 0x2d, 0xeb, + 0x6a, 0x92, 0x98, 0x6d, 0x0f, 0xe3, 0xac, 0x75, 0x2a, 0x2c, 0x4a, 0x80, 0x55, 0xdb, 0xb2, 0xf4, + 0xe8, 0x05, 0x98, 0xf4, 0x02, 0x2f, 0xf1, 0x9c, 0x24, 0x8c, 0xd8, 0xe6, 0x3a, 0xc1, 0x37, 0x0c, + 0x2a, 0xe1, 0xeb, 0x3a, 0x02, 0x9b, 0x74, 0xf6, 0x3b, 0x50, 0x51, 0xfe, 0x66, 0xd2, 0x65, 0xd2, + 0x2a, 0x70, 0x99, 0xec, 0xbf, 0x23, 0x48, 0xab, 0x79, 0x39, 0xd7, 0x6a, 0xfe, 0x77, 0x2c, 0x48, + 0xdd, 0x6e, 0xd0, 0x45, 0xa8, 0xb4, 0x43, 0x76, 0x73, 0x16, 0xc9, 0xeb, 0xe8, 0x07, 0x72, 0x85, + 0x07, 0x17, 0x54, 0xbc, 0xff, 0x1a, 0xb2, 0x04, 0x4e, 0x0b, 0xa3, 0x65, 0x18, 0x6d, 0x47, 0xa4, + 0x99, 0xb0, 0xc0, 0x91, 0xbe, 0x7c, 0xf8, 0x1c, 0xe1, 0xf4, 0x58, 0x16, 0xb4, 0x7f, 0xd3, 0x02, + 0xe0, 0x46, 0x69, 0x27, 0xd8, 0x24, 0xc7, 0x70, 0xd0, 0xae, 0xc1, 0x50, 0xdc, 0x26, 0xad, 0x5e, + 0x77, 0x9a, 0x69, 0x7b, 0x9a, 0x6d, 0xd2, 0x4a, 0x3b, 0x9c, 0xfe, 0xc3, 0xac, 0xb4, 0xfd, 0xb3, + 0x00, 0x53, 0x29, 0x19, 0x3d, 0x00, 0xa1, 0xa7, 0x0d, 0xb7, 0xfc, 0xd3, 0x19, 0xb7, 0xfc, 0x0a, + 0xa3, 0xd6, 0x3c, 0xf1, 0xdf, 0x81, 0xf2, 0x8e, 0x73, 0x5b, 0x9c, 0xb2, 0x9e, 0xec, 0xdd, 0x0c, + 0xca, 0x7f, 0xe1, 0x8a, 0x73, 0x9b, 0xeb, 0xb1, 0x4f, 0xca, 0x09, 0x72, 0xc5, 0xb9, 0x7d, 0xc8, + 0x6f, 0x2e, 0x99, 0x90, 0xa2, 0x87, 0xb9, 0x2f, 0xfc, 0x71, 0xfa, 0x9f, 0x4d, 0x3b, 0x5a, 0x09, + 0xab, 0xcb, 0x0b, 0x84, 0x89, 0x76, 0xa0, 0xba, 0xbc, 0x20, 0x5b, 0x97, 0x17, 0x0c, 0x50, 0x97, + 0x17, 0xa0, 0xf7, 0x60, 0x54, 0x5c, 0x89, 0x30, 0x7f, 0xc2, 0xf1, 0xf3, 0x8b, 0x03, 0xd4, 0x27, + 0x6e, 0x54, 0x78, 0x9d, 0x8b, 0x52, 0x4f, 0x17, 0xd0, 0xbe, 0xf5, 0xca, 0x0a, 0xd1, 0xdf, 0xb2, + 0x60, 0x4a, 0xfc, 0xc6, 0xe4, 0xdd, 0x0e, 0x89, 0x13, 0xa1, 0x0f, 0x7c, 0x62, 0xf0, 0x36, 0x88, + 0x82, 0xbc, 0x29, 0x9f, 0x90, 0x62, 0xd6, 0x44, 0xf6, 0x6d, 0x51, 0xa6, 0x15, 0xe8, 0x9f, 0x5a, + 0x70, 0x72, 0xc7, 0xb9, 0xcd, 0x6b, 0xe4, 0x30, 0xec, 0x24, 0x5e, 0x28, 0xfc, 0x23, 0x5f, 0x1e, + 0x6c, 0xf8, 0xbb, 0x8a, 0xf3, 0x46, 0x4a, 0x57, 0xaa, 0x93, 0x79, 0x24, 0x7d, 0x9b, 0x9a, 0xdb, + 0xae, 0xb9, 0x0d, 0x18, 0x93, 0xf3, 0x2d, 0xe7, 0x34, 0x54, 0xd3, 0x95, 0x9d, 0x23, 0xdf, 0x48, + 0x69, 0xa7, 0x27, 0x56, 0x8f, 0x98, 0x6b, 0xf7, 0xb4, 0x9e, 0x77, 0x60, 0x42, 0x9f, 0x63, 0xf7, + 0xb4, 0xae, 0x77, 0xe1, 0x44, 0xce, 0x5c, 0xba, 0xa7, 0x55, 0xde, 0x82, 0xd3, 0x85, 0xf3, 0xe3, + 0x5e, 0x56, 0x6c, 0x7f, 0xdd, 0xd2, 0xe5, 0xe0, 0x31, 0x98, 0xa7, 0x56, 0x4c, 0xf3, 0xd4, 0x99, + 0xde, 0x2b, 0xa7, 0xc0, 0x46, 0xf5, 0x96, 0xde, 0x68, 0x2a, 0xd5, 0xd1, 0xab, 0x30, 0xe2, 0x53, + 0x88, 0xbc, 0x87, 0xb3, 0xfb, 0xaf, 0xc8, 0x54, 0x97, 0x62, 0xf0, 0x18, 0x0b, 0x0e, 0xf6, 0xef, + 0x58, 0x30, 0x74, 0x0c, 0x3d, 0x81, 0xcd, 0x9e, 0x78, 0xba, 0x90, 0xb5, 0xc8, 0x7d, 0xb0, 0x80, + 0x9d, 0x5b, 0xab, 0xb7, 0x13, 0x12, 0xc4, 0x4c, 0x7d, 0xcf, 0xed, 0x98, 0xff, 0x53, 0x82, 0x71, + 0x5a, 0x95, 0x74, 0x1a, 0x79, 0x09, 0x26, 0x7d, 0xe7, 0x26, 0xf1, 0xa5, 0xc9, 0x3c, 0x7b, 0x88, + 0xbd, 0xac, 0x23, 0xb1, 0x49, 0x4b, 0x0b, 0x6f, 0xe8, 0xb7, 0x07, 0x42, 0x7f, 0x51, 0x85, 0x8d, + 0xab, 0x05, 0x6c, 0xd2, 0xd2, 0xf3, 0xd4, 0x2d, 0x27, 0x69, 0x6d, 0x89, 0x03, 0xae, 0x6a, 0xee, + 0xeb, 0x14, 0x88, 0x39, 0x8e, 0x2a, 0x70, 0x72, 0x76, 0xde, 0x20, 0x11, 0x53, 0xe0, 0xb8, 0x7a, + 0xac, 0x14, 0x38, 0x6c, 0xa2, 0x71, 0x96, 0x3e, 0x27, 0x3e, 0x6f, 0x98, 0xb9, 0xc4, 0x0c, 0x10, + 0x9f, 0x87, 0x1a, 0x70, 0xd2, 0x0b, 0x5a, 0x7e, 0xc7, 0x25, 0xd7, 0x03, 0xae, 0xdd, 0xf9, 0xde, + 0x7b, 0xc4, 0x15, 0x0a, 0xb4, 0xf2, 0x5e, 0xaa, 0xe7, 0xd0, 0xe0, 0xdc, 0x92, 0xf6, 0x4f, 0xc3, + 0x89, 0xcb, 0xa1, 0xe3, 0x2e, 0x3b, 0xbe, 0x13, 0xb4, 0x48, 0x54, 0x0f, 0x36, 0xfb, 0x5e, 0xc8, + 0xeb, 0xd7, 0xe7, 0xa5, 0x7e, 0xd7, 0xe7, 0xf6, 0x16, 0x20, 0xbd, 0x02, 0xe1, 0x0a, 0x86, 0x61, + 0xd4, 0xe3, 0x55, 0x89, 0xe9, 0xff, 0x58, 0xbe, 0x76, 0xdd, 0xd5, 0x32, 0xcd, 0xc9, 0x89, 0x03, + 0xb0, 0x64, 0x64, 0xbf, 0x08, 0xb9, 0xf1, 0x19, 0xfd, 0x8f, 0xd2, 0xf6, 0xf3, 0x30, 0xcb, 0x4a, + 0x1e, 0xed, 0x98, 0x67, 0xff, 0x75, 0x0b, 0xa6, 0xaf, 0x66, 0x22, 0x6a, 0x1f, 0x85, 0x91, 0x98, + 0x44, 0x39, 0xb6, 0xd0, 0x26, 0x83, 0x62, 0x81, 0xbd, 0xeb, 0x36, 0x97, 0x1f, 0x5a, 0x50, 0x51, + 0xe1, 0xef, 0xc7, 0xa0, 0xd4, 0xae, 0x18, 0x4a, 0x6d, 0xae, 0x2d, 0x40, 0x35, 0xa7, 0x48, 0xa7, + 0x45, 0x97, 0x54, 0x6c, 0x68, 0x0f, 0x33, 0x40, 0xca, 0x86, 0x47, 0x12, 0x4e, 0x99, 0x01, 0xa4, + 0x32, 0x5a, 0x94, 0xdd, 0x88, 0x2b, 0xda, 0x0f, 0xc9, 0x8d, 0xb8, 0x6a, 0x4f, 0x81, 0xf4, 0x6b, + 0x68, 0x4d, 0x66, 0xbb, 0xc2, 0xa7, 0x99, 0xe7, 0x28, 0x5b, 0x9b, 0x2a, 0x24, 0x7b, 0x5e, 0x78, + 0x82, 0x0a, 0xe8, 0x21, 0x13, 0x64, 0xe2, 0x1f, 0x4f, 0x55, 0x90, 0x16, 0xb1, 0x2f, 0xc2, 0x74, + 0xa6, 0xc3, 0xd0, 0xf3, 0x30, 0xdc, 0xde, 0x72, 0x62, 0x92, 0xf1, 0x04, 0x1a, 0x6e, 0x50, 0xe0, + 0xe1, 0xfe, 0xfc, 0x94, 0x2a, 0xc0, 0x20, 0x98, 0x53, 0xdb, 0xff, 0xc3, 0x82, 0xa1, 0xab, 0xa1, + 0x7b, 0x1c, 0x93, 0xe9, 0x15, 0x63, 0x32, 0x3d, 0x58, 0x94, 0xe8, 0xa5, 0x70, 0x1e, 0xad, 0x65, + 0xe6, 0xd1, 0x99, 0x42, 0x0e, 0xbd, 0xa7, 0xd0, 0x0e, 0x8c, 0xb3, 0xf4, 0x31, 0xc2, 0x2b, 0xe9, + 0x59, 0xe3, 0x7c, 0x35, 0x9f, 0x39, 0x5f, 0x4d, 0x6b, 0xa4, 0xda, 0x29, 0xeb, 0x71, 0x18, 0x15, + 0x9e, 0x31, 0x59, 0x1f, 0x59, 0x41, 0x8b, 0x25, 0xde, 0xfe, 0xe5, 0x32, 0x18, 0xe9, 0x6a, 0xd0, + 0xef, 0x59, 0xb0, 0x10, 0xf1, 0xa8, 0x20, 0xb7, 0xd6, 0x89, 0xbc, 0x60, 0xb3, 0xd9, 0xda, 0x22, + 0x6e, 0xc7, 0xf7, 0x82, 0xcd, 0xfa, 0x66, 0x10, 0x2a, 0xf0, 0xea, 0x6d, 0xd2, 0xea, 0x30, 0x3b, + 0x78, 0x9f, 0xdc, 0x38, 0xea, 0xe6, 0xf9, 0xfc, 0xc1, 0xfe, 0xfc, 0x02, 0x3e, 0x12, 0x6f, 0x7c, + 0xc4, 0xb6, 0xa0, 0xef, 0x58, 0xb0, 0xc8, 0xb3, 0xb8, 0x0c, 0xde, 0xfe, 0x1e, 0xa7, 0xd1, 0x86, + 0x64, 0x95, 0x32, 0x59, 0x27, 0xd1, 0xce, 0xf2, 0x0b, 0xa2, 0x43, 0x17, 0x1b, 0x47, 0xab, 0x0b, + 0x1f, 0xb5, 0x71, 0xf6, 0xbf, 0x2a, 0xc3, 0x24, 0xed, 0xc5, 0x34, 0x12, 0xfe, 0x79, 0x63, 0x4a, + 0x3c, 0x9c, 0x99, 0x12, 0xb3, 0x06, 0xf1, 0xdd, 0x09, 0x82, 0x8f, 0x61, 0xd6, 0x77, 0xe2, 0xe4, + 0x22, 0x71, 0xa2, 0xe4, 0x26, 0x71, 0xd8, 0x55, 0xaf, 0x98, 0xe6, 0x47, 0xb9, 0x3d, 0x56, 0xe6, + 0xaf, 0xcb, 0x59, 0x66, 0xb8, 0x9b, 0x3f, 0xda, 0x05, 0xc4, 0xae, 0x95, 0x23, 0x27, 0x88, 0xf9, + 0xb7, 0x78, 0xc2, 0x46, 0x7e, 0xb4, 0x5a, 0xe7, 0x44, 0xad, 0xe8, 0x72, 0x17, 0x37, 0x9c, 0x53, + 0x83, 0xe6, 0x2e, 0x30, 0x3c, 0xa8, 0xbb, 0xc0, 0x48, 0x1f, 0x47, 0xf4, 0x1d, 0x98, 0x11, 0xa3, + 0xb2, 0xe1, 0x6d, 0x8a, 0x4d, 0xfa, 0x8d, 0x8c, 0x3b, 0x91, 0x35, 0xb8, 0xe3, 0x43, 0x1f, 0x5f, + 0x22, 0xfb, 0x67, 0xe0, 0x04, 0xad, 0xce, 0x74, 0x9b, 0x8e, 0x11, 0x81, 0xe9, 0xed, 0xce, 0x4d, + 0xe2, 0x93, 0x44, 0xc2, 0x44, 0xa5, 0xb9, 0x6a, 0xbf, 0x59, 0x3a, 0xd5, 0x2d, 0x2f, 0x99, 0x2c, + 0x70, 0x96, 0xa7, 0xfd, 0x2b, 0x16, 0x30, 0xc7, 0xc4, 0x63, 0xd8, 0xfe, 0x3e, 0x65, 0x6e, 0x7f, + 0xd5, 0x22, 0x09, 0x54, 0xb0, 0xf3, 0x3d, 0xc7, 0x87, 0xa5, 0x11, 0x85, 0xb7, 0xf7, 0xa4, 0xee, + 0xdf, 0x5f, 0xe3, 0xfa, 0xdf, 0x16, 0x5f, 0x90, 0x2a, 0x48, 0x12, 0x7d, 0x1e, 0xc6, 0x5a, 0x4e, + 0xdb, 0x69, 0xf1, 0x3c, 0x61, 0x85, 0xd6, 0x1f, 0xa3, 0xd0, 0xc2, 0x8a, 0x28, 0xc1, 0xad, 0x19, + 0x1f, 0x97, 0x5f, 0x29, 0xc1, 0x7d, 0x2d, 0x18, 0xaa, 0xca, 0xb9, 0x6d, 0x98, 0x34, 0x98, 0xdd, + 0xd3, 0xa3, 0xef, 0xe7, 0xf9, 0x76, 0xa1, 0x4e, 0x2c, 0x3b, 0x30, 0x1b, 0x68, 0xff, 0xa9, 0x70, + 0x94, 0xea, 0xf4, 0x47, 0xfb, 0x6d, 0x08, 0x4c, 0x92, 0x6a, 0x8e, 0x97, 0x19, 0x36, 0xb8, 0x9b, + 0xb3, 0xfd, 0xf7, 0x2c, 0xb8, 0x5f, 0x27, 0xd4, 0xe2, 0x57, 0xfb, 0xd9, 0x93, 0x6b, 0x30, 0x16, + 0xb6, 0x49, 0xe4, 0xa4, 0x67, 0xb2, 0x73, 0xb2, 0xd3, 0xaf, 0x09, 0xf8, 0xe1, 0xfe, 0xfc, 0x49, + 0x9d, 0xbb, 0x84, 0x63, 0x55, 0x12, 0xd9, 0x30, 0xc2, 0x3a, 0x23, 0x16, 0xb1, 0xc5, 0x2c, 0x97, + 0x16, 0xbb, 0xee, 0x8a, 0xb1, 0xc0, 0xd8, 0x3f, 0x6b, 0xf1, 0x89, 0xa5, 0x37, 0x1d, 0xbd, 0x0b, + 0x33, 0x3b, 0xf4, 0xf8, 0xb6, 0x7a, 0xbb, 0x1d, 0x71, 0x33, 0xba, 0xec, 0xa7, 0x27, 0xfb, 0xf5, + 0x93, 0xf6, 0x91, 0xcb, 0x55, 0xd1, 0xe6, 0x99, 0x2b, 0x19, 0x66, 0xb8, 0x8b, 0xbd, 0xfd, 0xe7, + 0x25, 0xbe, 0x12, 0x99, 0x56, 0xf7, 0x38, 0x8c, 0xb6, 0x43, 0x77, 0xa5, 0x5e, 0xc3, 0xa2, 0x87, + 0x94, 0xb8, 0x6a, 0x70, 0x30, 0x96, 0x78, 0x74, 0x1e, 0x80, 0xdc, 0x4e, 0x48, 0x14, 0x38, 0xbe, + 0xba, 0x8c, 0x57, 0xca, 0xd3, 0xaa, 0xc2, 0x60, 0x8d, 0x8a, 0x96, 0x69, 0x47, 0xe1, 0xae, 0xe7, + 0xb2, 0xe0, 0x8e, 0xb2, 0x59, 0xa6, 0xa1, 0x30, 0x58, 0xa3, 0xa2, 0x47, 0xe5, 0x4e, 0x10, 0xf3, + 0x0d, 0xd0, 0xb9, 0x29, 0xd2, 0xf1, 0x8c, 0xa5, 0x47, 0xe5, 0xeb, 0x3a, 0x12, 0x9b, 0xb4, 0x68, + 0x09, 0x46, 0x12, 0x87, 0x5d, 0x31, 0x0f, 0x17, 0xbb, 0xec, 0xac, 0x53, 0x0a, 0x3d, 0x71, 0x14, + 0x2d, 0x80, 0x45, 0x41, 0xf4, 0xa6, 0x14, 0xc1, 0x5c, 0x24, 0x0b, 0xd7, 0xab, 0xc2, 0x69, 0xab, + 0x8b, 0x6f, 0x5d, 0x06, 0x0b, 0x97, 0x2e, 0x83, 0x97, 0xfd, 0xc5, 0x0a, 0x40, 0xaa, 0xed, 0xa1, + 0xf7, 0xba, 0x44, 0xc4, 0x53, 0xbd, 0xf5, 0xc3, 0xbb, 0x27, 0x1f, 0xd0, 0x97, 0x2c, 0x18, 0x77, + 0x7c, 0x3f, 0x6c, 0x39, 0x09, 0xeb, 0xe5, 0x52, 0x6f, 0x11, 0x25, 0xea, 0x5f, 0x4a, 0x4b, 0xf0, + 0x26, 0x3c, 0x2b, 0x6f, 0x8f, 0x35, 0x4c, 0xdf, 0x56, 0xe8, 0x15, 0xa3, 0x8f, 0xcb, 0x43, 0x00, + 0x9f, 0x1e, 0x73, 0xd9, 0x43, 0x40, 0x85, 0x49, 0x63, 0x4d, 0xff, 0x47, 0xd7, 0x8d, 0xbc, 0x35, + 0x43, 0xc5, 0x21, 0xba, 0x86, 0xd2, 0xd3, 0x2f, 0x65, 0x0d, 0x6a, 0xe8, 0x2e, 0xe8, 0xc3, 0xc5, + 0x71, 0xec, 0x9a, 0x76, 0xdd, 0xc7, 0xfd, 0xfc, 0x1d, 0x98, 0x76, 0xcd, 0xed, 0x56, 0xcc, 0xa6, + 0xc7, 0x8a, 0xf8, 0x66, 0x76, 0xe7, 0x74, 0x83, 0xcd, 0x20, 0x70, 0x96, 0x31, 0x6a, 0xf0, 0x60, + 0x80, 0x7a, 0xb0, 0x11, 0x0a, 0x17, 0x3e, 0xbb, 0x70, 0x2c, 0xf7, 0xe2, 0x84, 0xec, 0x50, 0xca, + 0x74, 0x1f, 0xbd, 0x2a, 0xca, 0x62, 0xc5, 0x05, 0xbd, 0x0a, 0x23, 0x2c, 0x4a, 0x2b, 0xae, 0x8e, + 0x15, 0xdb, 0x01, 0xcd, 0x00, 0xe3, 0x74, 0x51, 0xb1, 0xbf, 0x31, 0x16, 0x1c, 0xd0, 0x45, 0x99, + 0x26, 0x20, 0xae, 0x07, 0xd7, 0x63, 0xc2, 0xd2, 0x04, 0x54, 0x96, 0x3f, 0x9a, 0x66, 0x00, 0xe0, + 0xf0, 0xdc, 0x14, 0x91, 0x46, 0x49, 0xaa, 0xaf, 0x88, 0xff, 0x32, 0xf3, 0x64, 0x15, 0x8a, 0x9b, + 0x67, 0x66, 0xa7, 0x4c, 0xbb, 0xf3, 0x86, 0xc9, 0x02, 0x67, 0x79, 0x1e, 0xeb, 0xf6, 0x39, 0x17, + 0xc0, 0x4c, 0x76, 0x61, 0xdd, 0xd3, 0xed, 0xfa, 0x07, 0x43, 0x30, 0x65, 0x4e, 0x04, 0xb4, 0x08, + 0x15, 0xc1, 0x44, 0xa5, 0x0c, 0x53, 0x73, 0xfb, 0x8a, 0x44, 0xe0, 0x94, 0x86, 0xa5, 0x4c, 0x63, + 0xc5, 0x35, 0xdf, 0xac, 0x34, 0x65, 0x9a, 0xc2, 0x60, 0x8d, 0x8a, 0x2a, 0xd1, 0x37, 0xc3, 0x30, + 0x51, 0x5b, 0x81, 0x9a, 0x2d, 0xcb, 0x0c, 0x8a, 0x05, 0x96, 0x6e, 0x01, 0xdb, 0x24, 0x0a, 0x88, + 0x6f, 0x5a, 0x32, 0xd5, 0x16, 0x70, 0x49, 0x47, 0x62, 0x93, 0x96, 0x6e, 0x69, 0x61, 0xcc, 0xa6, + 0x9f, 0x50, 0xd5, 0x53, 0x5f, 0xb7, 0x26, 0x8f, 0x52, 0x94, 0x78, 0xf4, 0x06, 0xdc, 0xaf, 0x82, + 0x0a, 0x31, 0xb7, 0x0c, 0xcb, 0x1a, 0x47, 0x8c, 0x93, 0xf5, 0xfd, 0x2b, 0xf9, 0x64, 0xb8, 0xa8, + 0x3c, 0x7a, 0x05, 0xa6, 0x84, 0x0a, 0x2c, 0x39, 0x8e, 0x9a, 0xce, 0x0a, 0x97, 0x0c, 0x2c, 0xce, + 0x50, 0xa3, 0x1a, 0xcc, 0x50, 0x08, 0xd3, 0x42, 0x25, 0x07, 0x1e, 0x1c, 0xa9, 0xf6, 0xfa, 0x4b, + 0x19, 0x3c, 0xee, 0x2a, 0x81, 0x96, 0x60, 0x9a, 0xeb, 0x28, 0xf4, 0x4c, 0xc9, 0xc6, 0x41, 0x78, + 0xd6, 0xaa, 0x85, 0x70, 0xcd, 0x44, 0xe3, 0x2c, 0x3d, 0x7a, 0x11, 0x26, 0x9c, 0xa8, 0xb5, 0xe5, + 0x25, 0xa4, 0x95, 0x74, 0x22, 0x9e, 0x84, 0x43, 0xf3, 0xf6, 0x58, 0xd2, 0x70, 0xd8, 0xa0, 0xb4, + 0xdf, 0x83, 0x13, 0x39, 0x4e, 0xf9, 0x74, 0xe2, 0x38, 0x6d, 0x4f, 0x7e, 0x53, 0xc6, 0x6b, 0x6d, + 0xa9, 0x51, 0x97, 0x5f, 0xa3, 0x51, 0xd1, 0xd9, 0xc9, 0x4c, 0xe2, 0x5a, 0x7a, 0x58, 0x35, 0x3b, + 0xd7, 0x24, 0x02, 0xa7, 0x34, 0xf6, 0xb7, 0x01, 0x34, 0x83, 0xce, 0x00, 0x3e, 0x4b, 0x2f, 0xc2, + 0x84, 0xcc, 0x69, 0xac, 0xe5, 0xd2, 0x54, 0x9f, 0x79, 0x41, 0xc3, 0x61, 0x83, 0x92, 0xb6, 0x2d, + 0x50, 0x99, 0x40, 0x33, 0x3e, 0x72, 0x69, 0x1e, 0xd0, 0x94, 0x06, 0x3d, 0x05, 0x63, 0x31, 0xf1, + 0x37, 0x2e, 0x7b, 0xc1, 0xb6, 0x98, 0xd8, 0x4a, 0x0a, 0x37, 0x05, 0x1c, 0x2b, 0x0a, 0xb4, 0x0c, + 0xe5, 0x8e, 0xe7, 0x8a, 0xa9, 0x2c, 0x37, 0xfc, 0xf2, 0xf5, 0x7a, 0xed, 0x70, 0x7f, 0xfe, 0xe1, + 0xa2, 0x54, 0xcd, 0xf4, 0x68, 0x1f, 0x2f, 0xd0, 0xe5, 0x47, 0x0b, 0xe7, 0xdd, 0x0d, 0x8c, 0x1c, + 0xf1, 0x6e, 0xe0, 0x3c, 0x80, 0xf8, 0x6a, 0x39, 0x97, 0xcb, 0xe9, 0xa8, 0x5d, 0x50, 0x18, 0xac, + 0x51, 0xa1, 0x18, 0x66, 0x5b, 0x11, 0x71, 0xe4, 0x19, 0x9a, 0xbb, 0x97, 0x8f, 0xdd, 0xb9, 0x81, + 0x60, 0x25, 0xcb, 0x0c, 0x77, 0xf3, 0x47, 0x21, 0xcc, 0xba, 0x22, 0x86, 0x35, 0xad, 0xb4, 0x72, + 0x74, 0x9f, 0x76, 0xe6, 0x90, 0x93, 0x65, 0x84, 0xbb, 0x79, 0xa3, 0xb7, 0x61, 0x4e, 0x02, 0xbb, + 0xc3, 0x86, 0xd9, 0x72, 0x29, 0x2f, 0x9f, 0x39, 0xd8, 0x9f, 0x9f, 0xab, 0x15, 0x52, 0xe1, 0x1e, + 0x1c, 0x10, 0x86, 0x11, 0x76, 0x97, 0x14, 0x57, 0xc7, 0xd9, 0x3e, 0xf7, 0x44, 0xb1, 0x31, 0x80, + 0xce, 0xf5, 0x05, 0x76, 0x0f, 0x25, 0xdc, 0x7c, 0xd3, 0x6b, 0x39, 0x06, 0xc4, 0x82, 0x13, 0xda, + 0x80, 0x71, 0x27, 0x08, 0xc2, 0xc4, 0xe1, 0x2a, 0xd4, 0x44, 0xb1, 0xee, 0xa7, 0x31, 0x5e, 0x4a, + 0x4b, 0x70, 0xee, 0xca, 0x73, 0x50, 0xc3, 0x60, 0x9d, 0x31, 0xba, 0x05, 0xd3, 0xe1, 0x2d, 0x2a, + 0x1c, 0xa5, 0x95, 0x22, 0xae, 0x4e, 0xb2, 0xba, 0x9e, 0x1b, 0xd0, 0x4e, 0x6b, 0x14, 0xd6, 0xa4, + 0x96, 0xc9, 0x14, 0x67, 0x6b, 0x41, 0x0b, 0x86, 0xb5, 0x7a, 0x2a, 0xf5, 0x67, 0x4f, 0xad, 0xd5, + 0xba, 0x71, 0x9a, 0x85, 0xa1, 0x73, 0xb7, 0x55, 0xb6, 0xfa, 0xa7, 0x33, 0x61, 0xe8, 0x29, 0x0a, + 0xeb, 0x74, 0x68, 0x0b, 0x26, 0xd2, 0x2b, 0xab, 0x28, 0x66, 0x59, 0x6a, 0xc6, 0xcf, 0x9f, 0x1f, + 0xec, 0xe3, 0xea, 0x5a, 0x49, 0x7e, 0x72, 0xd0, 0x21, 0xd8, 0xe0, 0x3c, 0xf7, 0x13, 0x30, 0xae, + 0x0d, 0xec, 0x51, 0xbc, 0xb2, 0xe7, 0x5e, 0x81, 0x99, 0xec, 0xd0, 0x1d, 0xc9, 0xab, 0xfb, 0x7f, + 0x95, 0x60, 0x3a, 0xe7, 0xe6, 0x8a, 0xa5, 0x7b, 0xce, 0x08, 0xd4, 0x34, 0xbb, 0xb3, 0x29, 0x16, + 0x4b, 0x03, 0x88, 0x45, 0x29, 0xa3, 0xcb, 0x85, 0x32, 0x5a, 0x88, 0xc2, 0xa1, 0xf7, 0x23, 0x0a, + 0xcd, 0xdd, 0x67, 0x78, 0xa0, 0xdd, 0xe7, 0x2e, 0x88, 0x4f, 0x63, 0x03, 0x1b, 0x1d, 0x60, 0x03, + 0xfb, 0x85, 0x12, 0xcc, 0x64, 0x73, 0x0a, 0x1f, 0xc3, 0x7d, 0xc7, 0xab, 0xc6, 0x7d, 0x47, 0x7e, + 0xf2, 0xf4, 0x6c, 0xa6, 0xe3, 0xa2, 0xbb, 0x0f, 0x9c, 0xb9, 0xfb, 0x78, 0x62, 0x20, 0x6e, 0xbd, + 0xef, 0x41, 0xfe, 0x7e, 0x09, 0x4e, 0x65, 0x8b, 0xac, 0xf8, 0x8e, 0xb7, 0x73, 0x0c, 0x7d, 0x73, + 0xcd, 0xe8, 0x9b, 0xa7, 0x07, 0xf9, 0x1a, 0xd6, 0xb4, 0xc2, 0x0e, 0x7a, 0x3d, 0xd3, 0x41, 0x8b, + 0x83, 0xb3, 0xec, 0xdd, 0x4b, 0xdf, 0xb6, 0xe0, 0x74, 0x6e, 0xb9, 0x63, 0xb0, 0xbe, 0x5e, 0x35, + 0xad, 0xaf, 0x8f, 0x0f, 0xfc, 0x4d, 0x05, 0xe6, 0xd8, 0xaf, 0x96, 0x0b, 0xbe, 0x85, 0xd9, 0xaf, + 0xae, 0xc1, 0xb8, 0xd3, 0x6a, 0x91, 0x38, 0xbe, 0x12, 0xba, 0x2a, 0xad, 0xd5, 0xd3, 0x6c, 0x4f, + 0x4a, 0xc1, 0x87, 0xfb, 0xf3, 0x73, 0x59, 0x16, 0x29, 0x1a, 0xeb, 0x1c, 0xcc, 0x54, 0x79, 0xa5, + 0xbb, 0x9a, 0x2a, 0xef, 0x3c, 0xc0, 0xae, 0x3a, 0xd5, 0x66, 0x8d, 0x61, 0xda, 0x79, 0x57, 0xa3, + 0x42, 0x3f, 0xc5, 0x74, 0x45, 0xee, 0x32, 0xc2, 0x2f, 0x39, 0x9e, 0x1d, 0x70, 0xac, 0x74, 0xf7, + 0x13, 0x1e, 0x08, 0xab, 0x0c, 0x87, 0x8a, 0x25, 0xfa, 0x0c, 0xcc, 0xc4, 0x3c, 0xd7, 0xc2, 0x8a, + 0xef, 0xc4, 0x2c, 0xfc, 0x42, 0xc8, 0x44, 0x16, 0xdd, 0xda, 0xcc, 0xe0, 0x70, 0x17, 0xb5, 0xfd, + 0x8f, 0xcb, 0xf0, 0x40, 0x8f, 0x29, 0x8a, 0x96, 0xcc, 0x2b, 0xde, 0x27, 0xb3, 0xd6, 0x9d, 0xb9, + 0xdc, 0xc2, 0x86, 0xb9, 0x27, 0x33, 0xc6, 0xa5, 0xf7, 0x3d, 0xc6, 0x5f, 0xb6, 0x34, 0xbb, 0x1b, + 0x77, 0x04, 0xfd, 0xd4, 0x11, 0x97, 0xde, 0x8f, 0xaa, 0xa1, 0xfe, 0x0b, 0x16, 0x3c, 0x9c, 0xfb, + 0x59, 0x86, 0xab, 0xc8, 0x22, 0x54, 0x5a, 0x14, 0xa8, 0x85, 0x48, 0xa5, 0x81, 0x8a, 0x12, 0x81, + 0x53, 0x1a, 0xc3, 0x23, 0xa4, 0xd4, 0xd7, 0x23, 0xe4, 0x5f, 0x5a, 0x70, 0x32, 0xdb, 0x88, 0x63, + 0x90, 0x4c, 0x75, 0x53, 0x32, 0x7d, 0x74, 0x90, 0x21, 0x2f, 0x10, 0x4a, 0x7f, 0x3a, 0x05, 0xf7, + 0x15, 0x64, 0xfc, 0xdf, 0x85, 0xd9, 0xcd, 0x16, 0x31, 0x83, 0xcf, 0xc4, 0xc7, 0xe4, 0xc6, 0xe9, + 0xf5, 0x8c, 0x54, 0xe3, 0xc7, 0x90, 0x2e, 0x12, 0xdc, 0x5d, 0x05, 0xfa, 0x82, 0x05, 0x27, 0x9d, + 0x5b, 0x71, 0xd7, 0x0b, 0x3c, 0x62, 0xce, 0x3c, 0x97, 0x6b, 0x1d, 0xeb, 0xf3, 0x62, 0x0f, 0x0b, + 0x10, 0x39, 0x99, 0x47, 0x85, 0x73, 0xeb, 0x42, 0x58, 0x64, 0xf6, 0xa3, 0x5a, 0x4e, 0x8f, 0xf0, + 0xc8, 0xbc, 0xe0, 0x15, 0x2e, 0xa3, 0x24, 0x06, 0x2b, 0x3e, 0xe8, 0x06, 0x54, 0x36, 0x65, 0x44, + 0x99, 0x90, 0x81, 0xb9, 0x9b, 0x4a, 0x6e, 0xd8, 0x19, 0xf7, 0xd8, 0x57, 0x28, 0x9c, 0xb2, 0x42, + 0xaf, 0x40, 0x39, 0xd8, 0x88, 0x7b, 0x3d, 0x79, 0x90, 0xf1, 0xa0, 0xe2, 0x71, 0xae, 0x57, 0xd7, + 0x9a, 0x98, 0x16, 0xa4, 0xe5, 0xa3, 0x9b, 0xae, 0x30, 0xe8, 0xe6, 0x96, 0xc7, 0xcb, 0xb5, 0xee, + 0xf2, 0x78, 0xb9, 0x86, 0x69, 0x41, 0xb4, 0x06, 0xc3, 0x2c, 0x40, 0x45, 0x58, 0x6b, 0x73, 0xe3, + 0xf4, 0xbb, 0x82, 0x6f, 0x78, 0xe0, 0x2b, 0x03, 0x63, 0x5e, 0x1c, 0xbd, 0x0a, 0x23, 0x2d, 0xf6, + 0x02, 0x80, 0x38, 0x5a, 0xe7, 0xe7, 0x9e, 0xe8, 0x7a, 0x23, 0x80, 0xdf, 0x51, 0x71, 0x38, 0x16, + 0x1c, 0xd0, 0x3a, 0x8c, 0xb4, 0x48, 0x7b, 0x6b, 0x23, 0x16, 0x27, 0xe6, 0x8f, 0xe7, 0xf2, 0xea, + 0xf1, 0xe0, 0x85, 0xe0, 0xca, 0x28, 0xb0, 0xe0, 0x85, 0x3e, 0x09, 0xa5, 0x8d, 0x96, 0x88, 0x55, + 0xc9, 0xb5, 0xd2, 0x9a, 0xc1, 0xc8, 0xcb, 0x23, 0x07, 0xfb, 0xf3, 0xa5, 0xb5, 0x15, 0x5c, 0xda, + 0x68, 0xa1, 0xab, 0x30, 0xba, 0xc1, 0x23, 0x4a, 0x45, 0xa6, 0xd6, 0xc7, 0xf2, 0x83, 0x5d, 0xbb, + 0x82, 0x4e, 0x79, 0x8c, 0x85, 0x40, 0x60, 0xc9, 0x04, 0xad, 0x03, 0x6c, 0xa8, 0xc8, 0x58, 0x91, + 0xaa, 0xf5, 0xa3, 0x83, 0xc4, 0xcf, 0x8a, 0xe3, 0xa3, 0x82, 0x62, 0x8d, 0x0f, 0xfa, 0x1c, 0x54, + 0x1c, 0xf9, 0xa6, 0x0b, 0x4b, 0xd3, 0x6a, 0xee, 0xd3, 0xe9, 0x82, 0xeb, 0xfd, 0xdc, 0x0d, 0x9f, + 0xad, 0x8a, 0x08, 0xa7, 0x4c, 0xd1, 0x36, 0x4c, 0xee, 0xc6, 0xed, 0x2d, 0x22, 0x17, 0x28, 0xcb, + 0xdd, 0x6a, 0x1e, 0x35, 0xd3, 0x44, 0xbb, 0x82, 0xd0, 0x8b, 0x92, 0x8e, 0xe3, 0x77, 0xc9, 0x14, + 0x16, 0x90, 0x73, 0x43, 0x67, 0x86, 0x4d, 0xde, 0xb4, 0xd3, 0xdf, 0xed, 0x84, 0x37, 0xf7, 0x12, + 0x22, 0x32, 0xba, 0xe6, 0x76, 0xfa, 0x6b, 0x9c, 0xa4, 0xbb, 0xd3, 0x05, 0x02, 0x4b, 0x26, 0x74, + 0x09, 0x3b, 0xf2, 0xbd, 0x24, 0x71, 0x46, 0x7e, 0xbc, 0xb0, 0x7b, 0xba, 0xda, 0x9b, 0x76, 0x0a, + 0x93, 0x7d, 0x29, 0x2b, 0x26, 0xf3, 0xda, 0x5b, 0x61, 0x12, 0x06, 0x19, 0x79, 0x3b, 0x5b, 0x2c, + 0xf3, 0x1a, 0x39, 0xf4, 0xdd, 0x32, 0x2f, 0x8f, 0x0a, 0xe7, 0xd6, 0x85, 0x5c, 0x98, 0x6a, 0x87, + 0x51, 0x72, 0x2b, 0x8c, 0xe4, 0xac, 0x42, 0x3d, 0x0e, 0x4f, 0x06, 0xa5, 0xa8, 0x91, 0xf9, 0xd7, + 0x9a, 0x18, 0x9c, 0xe1, 0x49, 0x87, 0x24, 0x6e, 0x39, 0x3e, 0xa9, 0x5f, 0xab, 0x9e, 0x28, 0x1e, + 0x92, 0x26, 0x27, 0xe9, 0x1e, 0x12, 0x81, 0xc0, 0x92, 0x09, 0x95, 0x3e, 0x2c, 0x39, 0x38, 0x4b, + 0x41, 0x5b, 0x20, 0x7d, 0xba, 0x3c, 0x4f, 0xb9, 0xf4, 0x61, 0x60, 0xcc, 0x8b, 0xd3, 0x99, 0x2f, + 0x74, 0xc2, 0x30, 0xae, 0x9e, 0x2a, 0x9e, 0xf9, 0x42, 0x95, 0xbc, 0xd6, 0xec, 0x35, 0xf3, 0x15, + 0x11, 0x4e, 0x99, 0xda, 0xdf, 0x18, 0xe9, 0xd6, 0x16, 0x98, 0xee, 0xff, 0x45, 0xab, 0xeb, 0xfa, + 0xf4, 0x13, 0x83, 0x1e, 0x58, 0xef, 0xe2, 0x45, 0xea, 0x17, 0x2c, 0xb8, 0xaf, 0x9d, 0xfb, 0x51, + 0x62, 0xeb, 0x1d, 0xec, 0xdc, 0xcb, 0xbb, 0x41, 0x25, 0x77, 0xce, 0xc7, 0xe3, 0x82, 0x9a, 0xb2, + 0x3a, 0x72, 0xf9, 0x7d, 0xeb, 0xc8, 0x57, 0x60, 0x8c, 0xa9, 0x77, 0x69, 0x22, 0x99, 0x81, 0x9c, + 0x90, 0xd8, 0x26, 0xbe, 0x22, 0x0a, 0x62, 0xc5, 0x02, 0xfd, 0x9c, 0x05, 0x0f, 0x65, 0x9b, 0x8e, + 0x09, 0x43, 0x8b, 0xc4, 0x84, 0xfc, 0xd8, 0xb1, 0x26, 0xbe, 0xff, 0xa1, 0x46, 0x2f, 0xe2, 0xc3, + 0x7e, 0x04, 0xb8, 0x77, 0x65, 0xa8, 0x96, 0x73, 0xee, 0x19, 0x31, 0x6f, 0x57, 0xfa, 0x9f, 0x7d, + 0xd0, 0x73, 0x30, 0xb1, 0x13, 0x76, 0x02, 0x19, 0x21, 0x20, 0xe2, 0x3f, 0x99, 0x25, 0xef, 0x8a, + 0x06, 0xc7, 0x06, 0xd5, 0xf1, 0xea, 0xfb, 0x5f, 0xb3, 0x72, 0x14, 0x55, 0x7e, 0x32, 0x7b, 0xd9, + 0x3c, 0x99, 0x3d, 0x9a, 0x3d, 0x99, 0x75, 0xd9, 0x59, 0x8c, 0x43, 0xd9, 0xe0, 0x09, 0x57, 0x07, + 0xcd, 0xb4, 0x63, 0xfb, 0x70, 0xb6, 0x9f, 0x70, 0x66, 0x8e, 0x58, 0xae, 0xba, 0xa1, 0x4c, 0x1d, + 0xb1, 0xdc, 0x7a, 0x0d, 0x33, 0xcc, 0xa0, 0x39, 0x1b, 0xec, 0xff, 0x66, 0x41, 0xb9, 0x11, 0xba, + 0xc7, 0x60, 0x37, 0xfa, 0x94, 0x61, 0x37, 0x7a, 0xa0, 0xe0, 0xf5, 0xc5, 0x42, 0x2b, 0xd1, 0x6a, + 0xc6, 0x4a, 0xf4, 0x50, 0x11, 0x83, 0xde, 0x36, 0xa1, 0x7f, 0x50, 0x06, 0xfd, 0xad, 0x48, 0xf4, + 0xaf, 0xef, 0xc4, 0xa3, 0xb7, 0xdc, 0xeb, 0xf9, 0x48, 0xc1, 0x99, 0xf9, 0x6f, 0xc9, 0x60, 0xc1, + 0x1f, 0x31, 0xc7, 0xde, 0xd7, 0x89, 0xb7, 0xb9, 0x95, 0x10, 0x37, 0xfb, 0x39, 0xc7, 0xe7, 0xd8, + 0xfb, 0x5f, 0x2c, 0x98, 0xce, 0xd4, 0x8e, 0xfc, 0xbc, 0xc8, 0xa3, 0x3b, 0xb4, 0x04, 0xcd, 0xf6, + 0x0d, 0x55, 0x5a, 0x00, 0x50, 0x46, 0x79, 0x69, 0x6d, 0x61, 0xba, 0xaf, 0xb2, 0xda, 0xc7, 0x58, + 0xa3, 0x40, 0xcf, 0xc3, 0x78, 0x12, 0xb6, 0x43, 0x3f, 0xdc, 0xdc, 0xbb, 0x44, 0x64, 0x96, 0x10, + 0x75, 0x75, 0xb2, 0x9e, 0xa2, 0xb0, 0x4e, 0x67, 0xff, 0x6a, 0x19, 0xb2, 0xef, 0x8b, 0xfe, 0xff, + 0x39, 0xf9, 0xe1, 0x9c, 0x93, 0xdf, 0xb5, 0x60, 0x86, 0xd6, 0xce, 0x7c, 0x63, 0xa4, 0x4b, 0xac, + 0x7a, 0x99, 0xc1, 0xea, 0xf1, 0x32, 0xc3, 0xa3, 0x54, 0x76, 0xb9, 0x61, 0x27, 0x11, 0x56, 0x21, + 0x4d, 0x38, 0x51, 0x28, 0x16, 0x58, 0x41, 0x47, 0xa2, 0x48, 0xc4, 0x13, 0xe9, 0x74, 0x24, 0x8a, + 0xb0, 0xc0, 0xca, 0x87, 0x1b, 0x86, 0x0a, 0x1e, 0x6e, 0x60, 0x09, 0xb6, 0x84, 0x3f, 0x86, 0x50, + 0x28, 0xb4, 0x04, 0x5b, 0xd2, 0x51, 0x23, 0xa5, 0xb1, 0xbf, 0x5e, 0x86, 0x89, 0x46, 0xe8, 0xa6, + 0x5e, 0xf4, 0xcf, 0x19, 0x5e, 0xf4, 0x67, 0x33, 0x5e, 0xf4, 0x33, 0x3a, 0xed, 0xdd, 0x71, 0xa2, + 0x17, 0xe9, 0xd7, 0xd8, 0x33, 0x22, 0x77, 0xe8, 0x40, 0x6f, 0xa4, 0x5f, 0x53, 0x8c, 0xb0, 0xc9, + 0xf7, 0xc7, 0xc9, 0x71, 0xfe, 0x2f, 0x2c, 0x98, 0x6a, 0x84, 0x2e, 0x9d, 0xa0, 0x3f, 0x4e, 0xb3, + 0x51, 0x4f, 0xdf, 0x36, 0xd2, 0x23, 0x7d, 0xdb, 0x3f, 0xb4, 0x60, 0xb4, 0x11, 0xba, 0xc7, 0x60, + 0x31, 0x7d, 0xd9, 0xb4, 0x98, 0xde, 0x5f, 0x20, 0x65, 0x0b, 0x8c, 0xa4, 0xbf, 0x55, 0x86, 0x49, + 0xda, 0xce, 0x70, 0x53, 0x8e, 0x92, 0xd1, 0x23, 0xd6, 0x00, 0x3d, 0x42, 0x95, 0xb9, 0xd0, 0xf7, + 0xc3, 0x5b, 0xd9, 0x11, 0x5b, 0x63, 0x50, 0x2c, 0xb0, 0xe8, 0x29, 0x18, 0x6b, 0x47, 0x64, 0xd7, + 0x0b, 0x3b, 0x71, 0x36, 0x22, 0xb1, 0x21, 0xe0, 0x58, 0x51, 0x50, 0xbd, 0x3d, 0xf6, 0x82, 0x16, + 0x91, 0x3e, 0x1a, 0x43, 0xcc, 0x47, 0x83, 0x67, 0xc0, 0xd4, 0xe0, 0xd8, 0xa0, 0x42, 0xaf, 0x43, + 0x85, 0xfd, 0x67, 0xeb, 0xe6, 0xe8, 0xef, 0x32, 0xf0, 0x03, 0xae, 0x64, 0x80, 0x53, 0x5e, 0xe8, + 0x3c, 0x40, 0x22, 0xbd, 0x49, 0x62, 0x11, 0x30, 0xab, 0x34, 0x4a, 0xe5, 0x67, 0x12, 0x63, 0x8d, + 0x0a, 0x3d, 0x09, 0x95, 0xc4, 0xf1, 0xfc, 0xcb, 0x5e, 0x40, 0x62, 0xe1, 0x8d, 0x23, 0xb2, 0x4a, + 0x0b, 0x20, 0x4e, 0xf1, 0x74, 0x47, 0x67, 0xe1, 0xd8, 0xfc, 0x55, 0x97, 0x31, 0x46, 0xcd, 0x76, + 0xf4, 0xcb, 0x0a, 0x8a, 0x35, 0x0a, 0xfb, 0x45, 0x38, 0xd5, 0x08, 0xdd, 0x46, 0x18, 0x25, 0x6b, + 0x61, 0x74, 0xcb, 0x89, 0x5c, 0x39, 0x7e, 0xf3, 0x32, 0xc1, 0x31, 0xdd, 0x75, 0x87, 0xb9, 0x35, + 0xc0, 0x48, 0x5d, 0xfc, 0x2c, 0xdb, 0xd3, 0x8f, 0x18, 0x3a, 0xf1, 0xef, 0x4a, 0x80, 0x1a, 0xcc, + 0xdf, 0xc5, 0x78, 0xfa, 0xe7, 0x6d, 0x98, 0x8a, 0xc9, 0x65, 0x2f, 0xe8, 0xdc, 0x96, 0xe7, 0xab, + 0x1e, 0x71, 0x29, 0xcd, 0x55, 0x9d, 0x92, 0x5b, 0x54, 0x4c, 0x18, 0xce, 0x70, 0xa3, 0x5d, 0x18, + 0x75, 0x82, 0xa5, 0xf8, 0x7a, 0x4c, 0x22, 0xf1, 0xd4, 0x0d, 0xeb, 0x42, 0x2c, 0x81, 0x38, 0xc5, + 0xd3, 0x29, 0xc3, 0xfe, 0x5c, 0x0d, 0x03, 0x1c, 0x86, 0x89, 0x9c, 0x64, 0xec, 0xb1, 0x04, 0x0d, + 0x8e, 0x0d, 0x2a, 0xb4, 0x06, 0x28, 0xee, 0xb4, 0xdb, 0x3e, 0xbb, 0x1e, 0x74, 0xfc, 0x0b, 0x51, + 0xd8, 0x69, 0x73, 0x87, 0x65, 0xf1, 0xce, 0x40, 0xb3, 0x0b, 0x8b, 0x73, 0x4a, 0x50, 0xc1, 0xb0, + 0x11, 0xb3, 0xdf, 0x22, 0x22, 0x9b, 0xdb, 0x36, 0x9b, 0x0c, 0x84, 0x25, 0xce, 0xfe, 0x3c, 0xdb, + 0xcc, 0xd8, 0x0b, 0x25, 0x49, 0x27, 0x22, 0x68, 0x07, 0x26, 0xdb, 0x6c, 0xc3, 0x4a, 0xa2, 0xd0, + 0xf7, 0x89, 0xd4, 0x1b, 0xef, 0xcc, 0xf7, 0x86, 0xbf, 0x58, 0xa0, 0xb3, 0xc3, 0x26, 0x77, 0xfb, + 0x8b, 0xd3, 0x4c, 0x2e, 0x35, 0xf9, 0xa1, 0x65, 0x54, 0x78, 0xd4, 0x0a, 0x0d, 0x6d, 0xae, 0xf8, + 0x45, 0xb0, 0x54, 0xd2, 0x0b, 0xaf, 0x5c, 0x2c, 0xcb, 0xa2, 0xd7, 0x98, 0xa7, 0x37, 0x17, 0x06, + 0xfd, 0xde, 0x22, 0xe4, 0x54, 0x86, 0x97, 0xb7, 0x28, 0x88, 0x35, 0x26, 0xe8, 0x32, 0x4c, 0x8a, + 0x07, 0x2d, 0x84, 0xe1, 0xa1, 0x6c, 0x1c, 0x7f, 0x27, 0xb1, 0x8e, 0x3c, 0xcc, 0x02, 0xb0, 0x59, + 0x18, 0x6d, 0xc2, 0x43, 0xda, 0xf3, 0x4b, 0x39, 0xfe, 0x5f, 0x5c, 0xb6, 0x3c, 0x7c, 0xb0, 0x3f, + 0xff, 0xd0, 0x7a, 0x2f, 0x42, 0xdc, 0x9b, 0x0f, 0xba, 0x06, 0xa7, 0x9c, 0x56, 0xe2, 0xed, 0x92, + 0x1a, 0x71, 0x5c, 0xdf, 0x0b, 0x88, 0x19, 0xa2, 0x7f, 0xfa, 0x60, 0x7f, 0xfe, 0xd4, 0x52, 0x1e, + 0x01, 0xce, 0x2f, 0x87, 0x5e, 0x86, 0x8a, 0x1b, 0xc4, 0xa2, 0x0f, 0x46, 0x8c, 0x97, 0xc5, 0x2a, + 0xb5, 0xab, 0x4d, 0xf5, 0xfd, 0xe9, 0x1f, 0x9c, 0x16, 0x40, 0x9b, 0x30, 0xa1, 0x87, 0xe1, 0x88, + 0x57, 0xe9, 0x9e, 0xee, 0x71, 0xb6, 0x35, 0x62, 0x57, 0xb8, 0xd5, 0x4d, 0x79, 0x57, 0x1a, 0x61, + 0x2d, 0x06, 0x63, 0xf4, 0x2a, 0xa0, 0x98, 0x44, 0xbb, 0x5e, 0x8b, 0x2c, 0xb5, 0x58, 0x8a, 0x58, + 0x66, 0xab, 0x19, 0x33, 0x42, 0x05, 0x50, 0xb3, 0x8b, 0x02, 0xe7, 0x94, 0x42, 0x17, 0xa9, 0x44, + 0xd1, 0xa1, 0xc2, 0x19, 0x56, 0xaa, 0x79, 0xd5, 0x1a, 0x69, 0x47, 0xa4, 0xe5, 0x24, 0xc4, 0x35, + 0x39, 0xe2, 0x4c, 0x39, 0xba, 0xdf, 0xa8, 0xcc, 0xfb, 0x60, 0xba, 0x70, 0x76, 0x67, 0xdf, 0xa7, + 0x27, 0xa4, 0xad, 0x30, 0x4e, 0xae, 0x92, 0xe4, 0x56, 0x18, 0x6d, 0x8b, 0xbc, 0x5a, 0x69, 0xda, + 0xbd, 0x14, 0x85, 0x75, 0x3a, 0xaa, 0x11, 0xb1, 0x4b, 0xb0, 0x7a, 0x8d, 0xdd, 0x53, 0x8c, 0xa5, + 0xeb, 0xe4, 0x22, 0x07, 0x63, 0x89, 0x97, 0xa4, 0xf5, 0xc6, 0x0a, 0xbb, 0x7d, 0xc8, 0x90, 0xd6, + 0x1b, 0x2b, 0x58, 0xe2, 0x11, 0xe9, 0x7e, 0xb5, 0x6d, 0xaa, 0xf8, 0xde, 0xa8, 0x5b, 0x2e, 0x0f, + 0xf8, 0x70, 0x5b, 0x00, 0x33, 0xea, 0xbd, 0x38, 0x9e, 0x70, 0x2c, 0xae, 0x4e, 0xb3, 0x49, 0x32, + 0x78, 0xb6, 0x32, 0x65, 0x8b, 0xab, 0x67, 0x38, 0xe1, 0x2e, 0xde, 0x46, 0xea, 0x87, 0x99, 0xbe, + 0x2f, 0x27, 0x2c, 0x42, 0x25, 0xee, 0xdc, 0x74, 0xc3, 0x1d, 0xc7, 0x0b, 0xd8, 0x65, 0x81, 0xfe, + 0x6c, 0xbe, 0x44, 0xe0, 0x94, 0x06, 0xad, 0xc1, 0x98, 0x23, 0x0e, 0x5f, 0xc2, 0xbc, 0x9f, 0x1b, + 0x0b, 0x2e, 0x0f, 0x68, 0xdc, 0x0e, 0x2a, 0xff, 0x61, 0x55, 0x16, 0xbd, 0x04, 0x93, 0x22, 0x5c, + 0x49, 0x78, 0x1a, 0x9e, 0x30, 0x3d, 0xdb, 0x9b, 0x3a, 0x12, 0x9b, 0xb4, 0xe8, 0xa7, 0x60, 0x8a, + 0x72, 0x49, 0x05, 0x5b, 0xf5, 0xe4, 0x20, 0x12, 0x51, 0xcb, 0x88, 0xad, 0x17, 0xc6, 0x19, 0x66, + 0xc8, 0x85, 0x07, 0x9d, 0x4e, 0x12, 0x32, 0x63, 0xa5, 0x39, 0xff, 0xd7, 0xc3, 0x6d, 0x12, 0x30, + 0xeb, 0xfe, 0xd8, 0xf2, 0xd9, 0x83, 0xfd, 0xf9, 0x07, 0x97, 0x7a, 0xd0, 0xe1, 0x9e, 0x5c, 0xd0, + 0x75, 0x18, 0x4f, 0x42, 0x5f, 0xb8, 0x08, 0xc7, 0xd5, 0xfb, 0x8a, 0x53, 0xd7, 0xac, 0x2b, 0x32, + 0xdd, 0x9c, 0xa0, 0x8a, 0x62, 0x9d, 0x0f, 0x5a, 0xe7, 0x6b, 0x8c, 0x25, 0x5a, 0x24, 0x71, 0xf5, + 0xfe, 0xe2, 0x8e, 0x51, 0xf9, 0x18, 0xcd, 0x25, 0x28, 0x4a, 0x62, 0x9d, 0x0d, 0xba, 0x00, 0xb3, + 0xed, 0xc8, 0x0b, 0xd9, 0xc4, 0x56, 0x86, 0xe2, 0xaa, 0x91, 0xd4, 0x6c, 0xb6, 0x91, 0x25, 0xc0, + 0xdd, 0x65, 0xd0, 0x39, 0xaa, 0xa0, 0x72, 0x60, 0xf5, 0x34, 0x7f, 0x51, 0x83, 0x2b, 0xa7, 0x1c, + 0x86, 0x15, 0x76, 0xee, 0xd3, 0x30, 0xdb, 0x25, 0x29, 0x8f, 0xe4, 0xae, 0xf9, 0xeb, 0xc3, 0x50, + 0x51, 0xe6, 0x40, 0xb4, 0x68, 0x5a, 0x79, 0x4f, 0x67, 0xad, 0xbc, 0x63, 0x54, 0x5f, 0xd3, 0x0d, + 0xbb, 0xeb, 0x39, 0x8f, 0x82, 0x9f, 0x2d, 0x10, 0x0d, 0x83, 0xc7, 0x56, 0x1d, 0xe1, 0xc1, 0xf4, + 0xf4, 0xc0, 0x38, 0xd4, 0xf3, 0xc0, 0x38, 0xe0, 0x03, 0x7d, 0xf4, 0x68, 0xd8, 0x0e, 0xdd, 0x7a, + 0x23, 0xfb, 0x62, 0x55, 0x83, 0x02, 0x31, 0xc7, 0x31, 0xe5, 0x9e, 0x6e, 0xeb, 0x4c, 0xb9, 0x1f, + 0xbd, 0x43, 0xe5, 0x5e, 0x32, 0xc0, 0x29, 0x2f, 0xe4, 0xc3, 0x6c, 0xcb, 0x7c, 0x6c, 0x4c, 0xc5, + 0x53, 0x3d, 0xd2, 0xf7, 0xd9, 0xaf, 0x8e, 0xf6, 0x02, 0xc9, 0x4a, 0x96, 0x0b, 0xee, 0x66, 0x8c, + 0x5e, 0x82, 0xb1, 0x77, 0xc3, 0x98, 0x4d, 0x3b, 0xb1, 0xb7, 0xc9, 0x08, 0x96, 0xb1, 0xd7, 0xae, + 0x35, 0x19, 0xfc, 0x70, 0x7f, 0x7e, 0xbc, 0x11, 0xba, 0xf2, 0x2f, 0x56, 0x05, 0xd0, 0x6d, 0x38, + 0x65, 0x48, 0x04, 0xd5, 0x5c, 0x18, 0xbc, 0xb9, 0x0f, 0x89, 0xea, 0x4e, 0xd5, 0xf3, 0x38, 0xe1, + 0xfc, 0x0a, 0xec, 0x6f, 0x70, 0xa3, 0xa7, 0x30, 0x8d, 0x90, 0xb8, 0xe3, 0x1f, 0xc7, 0x33, 0x03, + 0xab, 0x86, 0xd5, 0xe6, 0x8e, 0x0d, 0xeb, 0x7f, 0x60, 0x31, 0xc3, 0xfa, 0x3a, 0xd9, 0x69, 0xfb, + 0x4e, 0x72, 0x1c, 0x4e, 0xba, 0xaf, 0xc1, 0x58, 0x22, 0x6a, 0xeb, 0xf5, 0x32, 0x82, 0xd6, 0x28, + 0x76, 0xb9, 0xa0, 0x36, 0x44, 0x09, 0xc5, 0x8a, 0x8d, 0xfd, 0xcf, 0xf9, 0x08, 0x48, 0xcc, 0x31, + 0xd8, 0x16, 0x6a, 0xa6, 0x6d, 0x61, 0xbe, 0xcf, 0x17, 0x14, 0xd8, 0x18, 0xfe, 0x99, 0xd9, 0x6e, + 0x76, 0xf6, 0xf8, 0xb0, 0xdf, 0xe8, 0xd8, 0xbf, 0x64, 0xc1, 0xc9, 0x3c, 0x47, 0x00, 0xaa, 0xc4, + 0xf0, 0x93, 0x8f, 0xba, 0xe1, 0x52, 0x3d, 0x78, 0x43, 0xc0, 0xb1, 0xa2, 0x18, 0x38, 0x3b, 0xf9, + 0xd1, 0xd2, 0x35, 0x5d, 0x03, 0xf3, 0x5d, 0x3a, 0xf4, 0x0a, 0xf7, 0xba, 0xb7, 0xd4, 0xc3, 0x71, + 0x47, 0xf3, 0xb8, 0xb7, 0x7f, 0xad, 0x04, 0x27, 0xb9, 0x89, 0x7a, 0x69, 0x37, 0xf4, 0xdc, 0x46, + 0xe8, 0x8a, 0x18, 0x84, 0x37, 0x61, 0xa2, 0xad, 0x1d, 0x57, 0x7b, 0x25, 0x8c, 0xd1, 0x8f, 0xb5, + 0xe9, 0xb1, 0x41, 0x87, 0x62, 0x83, 0x17, 0x72, 0x61, 0x82, 0xec, 0x7a, 0x2d, 0x65, 0xe7, 0x2c, + 0x1d, 0x59, 0xa4, 0xab, 0x5a, 0x56, 0x35, 0x3e, 0xd8, 0xe0, 0x7a, 0x0f, 0xde, 0x10, 0xb1, 0xbf, + 0x62, 0xc1, 0xfd, 0x05, 0xe9, 0x65, 0x68, 0x75, 0xb7, 0xd8, 0x65, 0x80, 0x78, 0xe4, 0x50, 0x55, + 0xc7, 0xaf, 0x08, 0xb0, 0xc0, 0xa2, 0x9f, 0x04, 0xe0, 0x26, 0x7e, 0xf6, 0xa4, 0x7c, 0xa9, 0x77, + 0xfc, 0xba, 0x91, 0x76, 0x41, 0x8b, 0xcd, 0x57, 0x8f, 0xc8, 0x6b, 0xbc, 0xec, 0x5f, 0x29, 0xc3, + 0x30, 0x7f, 0xf1, 0x7a, 0x0d, 0x46, 0xb7, 0x78, 0x32, 0xdb, 0x41, 0xf2, 0xe6, 0xa6, 0xc7, 0x11, + 0x0e, 0xc0, 0xb2, 0x30, 0xba, 0x02, 0x27, 0x44, 0x9c, 0x4b, 0x8d, 0xf8, 0xce, 0x9e, 0x3c, 0xd5, + 0xf2, 0x87, 0x25, 0x64, 0xd2, 0xf3, 0x13, 0xf5, 0x6e, 0x12, 0x9c, 0x57, 0x0e, 0xbd, 0xd2, 0x95, + 0xc2, 0x8e, 0xa7, 0x01, 0x56, 0x3a, 0x70, 0x9f, 0x34, 0x76, 0x2f, 0xc1, 0x64, 0xbb, 0xeb, 0xfc, + 0xae, 0x3d, 0x36, 0x6c, 0x9e, 0xd9, 0x4d, 0x5a, 0xe6, 0x55, 0xd0, 0x61, 0x3e, 0x14, 0xeb, 0x5b, + 0x11, 0x89, 0xb7, 0x42, 0xdf, 0x15, 0x2f, 0x6b, 0xa6, 0x5e, 0x05, 0x19, 0x3c, 0xee, 0x2a, 0x41, + 0xb9, 0x6c, 0x38, 0x9e, 0xdf, 0x89, 0x48, 0xca, 0x65, 0xc4, 0xe4, 0xb2, 0x96, 0xc1, 0xe3, 0xae, + 0x12, 0x74, 0x1e, 0x9d, 0x12, 0xcf, 0x32, 0xca, 0xe8, 0x67, 0xe5, 0x2a, 0x32, 0x2a, 0xfd, 0xdb, + 0x7b, 0x64, 0xe4, 0x10, 0x57, 0xfe, 0xea, 0x61, 0x47, 0xed, 0xc1, 0x2f, 0xe1, 0xd9, 0x2e, 0xb9, + 0xdc, 0xc9, 0xe3, 0x80, 0x7f, 0x62, 0xc1, 0x89, 0x1c, 0xf7, 0x31, 0x2e, 0xaa, 0x36, 0xbd, 0x38, + 0x51, 0xef, 0x19, 0x68, 0xa2, 0x8a, 0xc3, 0xb1, 0xa2, 0xa0, 0xeb, 0x81, 0x0b, 0xc3, 0xac, 0x00, + 0x14, 0x2e, 0x1f, 0x02, 0x7b, 0x34, 0x01, 0x88, 0xce, 0xc2, 0x50, 0x27, 0x26, 0x91, 0x7c, 0x51, + 0x4f, 0xca, 0x6f, 0x66, 0x11, 0x64, 0x18, 0xaa, 0x51, 0x6e, 0x2a, 0x63, 0x9c, 0xa6, 0x51, 0x72, + 0x73, 0x1c, 0xc7, 0xd9, 0x5f, 0x2e, 0xc3, 0x74, 0xc6, 0x01, 0x94, 0x36, 0x64, 0x27, 0x0c, 0xbc, + 0x24, 0x54, 0x19, 0xd4, 0x78, 0xc2, 0x08, 0xd2, 0xde, 0xba, 0x22, 0xe0, 0x58, 0x51, 0xa0, 0x47, + 0xe5, 0x53, 0xab, 0xd9, 0x77, 0x1a, 0x96, 0x6b, 0xc6, 0x6b, 0xab, 0x83, 0x3e, 0xb8, 0xf2, 0x08, + 0x0c, 0xb5, 0x43, 0xf5, 0x0e, 0xb6, 0x1a, 0x4f, 0xbc, 0x5c, 0x6b, 0x84, 0xa1, 0x8f, 0x19, 0x12, + 0x7d, 0x4c, 0x7c, 0x7d, 0xe6, 0xbe, 0x02, 0x3b, 0x6e, 0x18, 0x6b, 0x5d, 0xf0, 0x38, 0x8c, 0x6e, + 0x93, 0xbd, 0xc8, 0x0b, 0x36, 0xb3, 0xb7, 0x35, 0x97, 0x38, 0x18, 0x4b, 0xbc, 0x99, 0xb0, 0x7c, + 0xf4, 0x9e, 0xbc, 0x99, 0x32, 0xd6, 0x77, 0x57, 0xfb, 0x2d, 0x0b, 0xa6, 0x59, 0xb6, 0x52, 0x11, + 0x67, 0xef, 0x85, 0xc1, 0x31, 0xe8, 0x09, 0x8f, 0xc0, 0x70, 0x44, 0x2b, 0xcd, 0x3e, 0x84, 0xc0, + 0x5a, 0x82, 0x39, 0x0e, 0x3d, 0x08, 0x43, 0xac, 0x09, 0x74, 0xf0, 0x26, 0x78, 0xbe, 0xf2, 0x9a, + 0x93, 0x38, 0x98, 0x41, 0x59, 0xc0, 0x13, 0x26, 0x6d, 0xdf, 0xe3, 0x8d, 0x4e, 0xcd, 0xad, 0x1f, + 0x8e, 0x80, 0xa7, 0xdc, 0xa6, 0xbd, 0xbf, 0x80, 0xa7, 0x7c, 0x96, 0xbd, 0x75, 0xf0, 0xff, 0x5e, + 0x82, 0x33, 0xb9, 0xe5, 0xd2, 0x9b, 0xdd, 0x35, 0xe3, 0x66, 0xf7, 0x7c, 0xe6, 0x66, 0xd7, 0xee, + 0x5d, 0xfa, 0xee, 0xdc, 0xf5, 0xe6, 0x5f, 0xc1, 0x96, 0x8f, 0xf1, 0x0a, 0x76, 0x68, 0x50, 0x35, + 0x65, 0xb8, 0x8f, 0x9a, 0xf2, 0x6d, 0x0b, 0x4e, 0xe7, 0x76, 0xd9, 0x87, 0x24, 0xc2, 0x2c, 0xb7, + 0x6d, 0x05, 0x67, 0x88, 0x1f, 0x96, 0x0a, 0xbe, 0x85, 0x9d, 0x26, 0xce, 0x51, 0x39, 0xc3, 0x90, + 0xb1, 0x50, 0xbb, 0x26, 0xb8, 0x8c, 0xe1, 0x30, 0xac, 0xb0, 0xc8, 0xd3, 0x62, 0xb5, 0x78, 0xd3, + 0x5e, 0x3a, 0xd2, 0x92, 0x59, 0x30, 0xad, 0xe3, 0x7a, 0x52, 0x80, 0x6c, 0xdc, 0xd6, 0x15, 0xed, + 0x04, 0x58, 0x1e, 0xfc, 0x04, 0x38, 0x91, 0x7f, 0xfa, 0x43, 0x4b, 0x30, 0xbd, 0xe3, 0x05, 0xec, + 0x35, 0x53, 0x53, 0xef, 0x51, 0x01, 0xae, 0x57, 0x4c, 0x34, 0xce, 0xd2, 0xcf, 0xbd, 0x04, 0x93, + 0x77, 0x6e, 0xb2, 0xfa, 0x6e, 0x19, 0x1e, 0xe8, 0xb1, 0xec, 0xb9, 0xac, 0x37, 0xc6, 0x40, 0x93, + 0xf5, 0x5d, 0xe3, 0xd0, 0x80, 0x93, 0x1b, 0x1d, 0xdf, 0xdf, 0x63, 0x5e, 0x4e, 0xc4, 0x95, 0x14, + 0x42, 0x31, 0x51, 0xa9, 0x88, 0xd7, 0x72, 0x68, 0x70, 0x6e, 0x49, 0xf4, 0x2a, 0xa0, 0xf0, 0x26, + 0x4b, 0x8f, 0xeb, 0xa6, 0xa9, 0x0e, 0x58, 0xc7, 0x97, 0xd3, 0xc5, 0x78, 0xad, 0x8b, 0x02, 0xe7, + 0x94, 0xa2, 0x1a, 0x26, 0x7b, 0x83, 0x5d, 0x35, 0x2b, 0xa3, 0x61, 0x62, 0x1d, 0x89, 0x4d, 0x5a, + 0x74, 0x01, 0x66, 0x9d, 0x5d, 0xc7, 0xe3, 0xa9, 0xaf, 0x24, 0x03, 0xae, 0x62, 0x2a, 0x43, 0xd1, + 0x52, 0x96, 0x00, 0x77, 0x97, 0x41, 0x1b, 0x86, 0x95, 0x8f, 0x67, 0xde, 0x3f, 0x3f, 0xf0, 0x6c, + 0x1d, 0xd8, 0xee, 0x67, 0xff, 0x27, 0x8b, 0x6e, 0x5f, 0x39, 0xcf, 0x67, 0xd2, 0x7e, 0x50, 0xf6, + 0x2b, 0x2d, 0xce, 0x4c, 0xf5, 0xc3, 0x8a, 0x8e, 0xc4, 0x26, 0x2d, 0x9f, 0x10, 0x71, 0xea, 0x64, + 0x6d, 0xe8, 0x89, 0x22, 0x30, 0x53, 0x51, 0xa0, 0x37, 0x60, 0xd4, 0xf5, 0x76, 0xbd, 0x38, 0x8c, + 0xc4, 0x62, 0x39, 0xea, 0xb3, 0xd1, 0x4a, 0x0e, 0xd6, 0x38, 0x1b, 0x2c, 0xf9, 0xd9, 0x5f, 0x2e, + 0xc1, 0xa4, 0xac, 0xf1, 0xb5, 0x4e, 0x98, 0x38, 0xc7, 0xb0, 0x2d, 0x5f, 0x30, 0xb6, 0xe5, 0x8f, + 0xf5, 0x8a, 0x4e, 0x65, 0x4d, 0x2a, 0xdc, 0x8e, 0xaf, 0x65, 0xb6, 0xe3, 0xc7, 0xfa, 0xb3, 0xea, + 0xbd, 0x0d, 0xff, 0xae, 0x05, 0xb3, 0x06, 0xfd, 0x31, 0xec, 0x06, 0x6b, 0xe6, 0x6e, 0xf0, 0x70, + 0xdf, 0x6f, 0x28, 0xd8, 0x05, 0xbe, 0x56, 0xca, 0xb4, 0x9d, 0x49, 0xff, 0x77, 0x61, 0x68, 0xcb, + 0x89, 0xdc, 0x5e, 0x09, 0x1c, 0xbb, 0x0a, 0x2d, 0x5c, 0x74, 0x22, 0x97, 0xcb, 0xf0, 0xa7, 0xd4, + 0xcb, 0x5e, 0x4e, 0xe4, 0xf6, 0x8d, 0x29, 0x60, 0x55, 0xa1, 0x17, 0x61, 0x24, 0x6e, 0x85, 0x6d, + 0xe5, 0x7b, 0x79, 0x96, 0xbf, 0xfa, 0x45, 0x21, 0x87, 0xfb, 0xf3, 0xc8, 0xac, 0x8e, 0x82, 0xb1, + 0xa0, 0x9f, 0xdb, 0x84, 0x8a, 0xaa, 0xfa, 0x9e, 0x7a, 0x95, 0xff, 0x51, 0x19, 0x4e, 0xe4, 0xcc, + 0x0b, 0x14, 0x1b, 0xbd, 0xf5, 0xcc, 0x80, 0xd3, 0xe9, 0x7d, 0xf6, 0x57, 0xcc, 0x4e, 0x2c, 0xae, + 0x18, 0xff, 0x81, 0x2b, 0xbd, 0x1e, 0x93, 0x6c, 0xa5, 0x14, 0xd4, 0xbf, 0x52, 0x5a, 0xd9, 0xb1, + 0x75, 0x35, 0xad, 0x48, 0xb5, 0xf4, 0x9e, 0x8e, 0xe9, 0x9f, 0x95, 0xe1, 0x64, 0x5e, 0x50, 0x3b, + 0xfa, 0x99, 0xcc, 0x73, 0x10, 0xcf, 0x0d, 0x1a, 0x0e, 0xcf, 0xdf, 0x88, 0x10, 0xb9, 0x62, 0x16, + 0xcc, 0x07, 0x22, 0xfa, 0x76, 0xb3, 0xa8, 0x93, 0x05, 0xf9, 0x44, 0xfc, 0x19, 0x0f, 0xb9, 0xc4, + 0x3f, 0x31, 0x70, 0x03, 0xc4, 0xfb, 0x1f, 0x71, 0x26, 0xc8, 0x47, 0x82, 0xfb, 0x07, 0xf9, 0xc8, + 0x9a, 0xe7, 0x3c, 0x18, 0xd7, 0xbe, 0xe6, 0x9e, 0x8e, 0xf8, 0x36, 0xdd, 0x51, 0xb4, 0x76, 0xdf, + 0xd3, 0x51, 0xff, 0x8a, 0x05, 0x19, 0x3f, 0x29, 0x65, 0xff, 0xb0, 0x0a, 0xed, 0x1f, 0x67, 0x61, + 0x28, 0x0a, 0x7d, 0x92, 0x7d, 0x21, 0x00, 0x87, 0x3e, 0xc1, 0x0c, 0xa3, 0x9e, 0xf1, 0x2d, 0x17, + 0x3d, 0xe3, 0x4b, 0x8f, 0xc6, 0x3e, 0xd9, 0x25, 0xd2, 0x1a, 0xa1, 0x64, 0xf2, 0x65, 0x0a, 0xc4, + 0x1c, 0x67, 0xff, 0xc6, 0x10, 0x9c, 0xc8, 0x09, 0x69, 0xa3, 0x07, 0x95, 0x4d, 0x27, 0x21, 0xb7, + 0x9c, 0xbd, 0x6c, 0xd6, 0xd2, 0x0b, 0x1c, 0x8c, 0x25, 0x9e, 0xf9, 0x72, 0xf2, 0xc4, 0x67, 0x19, + 0x1b, 0x91, 0xc8, 0x77, 0x26, 0xb0, 0xf7, 0xea, 0x65, 0xd7, 0xf3, 0x00, 0x71, 0xec, 0xaf, 0x06, + 0x54, 0xf9, 0x72, 0x85, 0xa7, 0x68, 0x9a, 0x25, 0xaf, 0x79, 0x59, 0x60, 0xb0, 0x46, 0x85, 0x6a, + 0x30, 0xd3, 0x8e, 0xc2, 0x84, 0xdb, 0xdd, 0x6a, 0xdc, 0x47, 0x61, 0xd8, 0x0c, 0x4e, 0x6a, 0x64, + 0xf0, 0xb8, 0xab, 0x04, 0x7a, 0x1e, 0xc6, 0x45, 0xc0, 0x52, 0x23, 0x0c, 0x7d, 0x61, 0xa5, 0x51, + 0x37, 0xde, 0xcd, 0x14, 0x85, 0x75, 0x3a, 0xad, 0x18, 0x33, 0xe6, 0x8d, 0xe6, 0x16, 0xe3, 0x06, + 0x3d, 0x8d, 0x2e, 0x93, 0xdb, 0x62, 0x6c, 0xa0, 0xdc, 0x16, 0xa9, 0xdd, 0xaa, 0x32, 0xf0, 0xfd, + 0x05, 0xf4, 0xb5, 0xf4, 0x7c, 0xa3, 0x0c, 0x23, 0x7c, 0x28, 0x8e, 0x41, 0x15, 0x5b, 0x13, 0xb6, + 0x9b, 0x1e, 0x19, 0x05, 0x78, 0x5b, 0x16, 0x6a, 0x4e, 0xe2, 0x70, 0x31, 0xa4, 0x56, 0x43, 0x6a, + 0xe5, 0x41, 0x0b, 0xc6, 0x7a, 0x99, 0xcb, 0x18, 0x27, 0x80, 0xf3, 0xd0, 0x56, 0xcf, 0xdb, 0x00, + 0x31, 0x7b, 0x5d, 0x94, 0xf2, 0x10, 0x19, 0x50, 0x9f, 0xe8, 0x51, 0x7b, 0x53, 0x11, 0xf3, 0x36, + 0xa4, 0x53, 0x50, 0x21, 0xb0, 0xc6, 0x71, 0xee, 0x05, 0xa8, 0x28, 0xe2, 0x7e, 0x27, 0xb9, 0x09, + 0x5d, 0x78, 0x7d, 0x0a, 0xa6, 0x33, 0x75, 0x1d, 0xe9, 0x20, 0xf8, 0xdb, 0x16, 0x4c, 0xf3, 0x26, + 0xaf, 0x06, 0xbb, 0x62, 0xb1, 0xbf, 0x07, 0x27, 0xfd, 0x9c, 0x45, 0x27, 0x46, 0x74, 0xf0, 0x45, + 0xaa, 0x0e, 0x7e, 0x79, 0x58, 0x9c, 0x5b, 0x07, 0x3d, 0xfc, 0xf3, 0x77, 0x91, 0x1d, 0x5f, 0x78, + 0x20, 0x4f, 0xf0, 0xcc, 0xd0, 0x1c, 0x86, 0x15, 0xd6, 0xfe, 0x9e, 0x05, 0xb3, 0x5d, 0xaf, 0xea, + 0x7f, 0xa0, 0x6d, 0x17, 0x89, 0xaf, 0x4b, 0x05, 0x89, 0xaf, 0xf5, 0x4f, 0x2b, 0xf7, 0xfc, 0xb4, + 0x5f, 0xb3, 0x40, 0xcc, 0xc0, 0x63, 0x50, 0xe7, 0x3f, 0x6d, 0xaa, 0xf3, 0x73, 0xc5, 0x93, 0xba, + 0x40, 0x8f, 0xff, 0x0b, 0x0b, 0x66, 0x38, 0x41, 0x7a, 0x79, 0xf1, 0x81, 0x8e, 0xc3, 0x20, 0xaf, + 0xb1, 0xa8, 0xe7, 0x2f, 0xf3, 0x3f, 0xca, 0x18, 0xac, 0xa1, 0x9e, 0x83, 0xe5, 0xca, 0x05, 0x74, + 0x84, 0x57, 0x86, 0x8e, 0x9c, 0xab, 0xcd, 0xfe, 0x53, 0x0b, 0x10, 0xaf, 0x26, 0xfb, 0x20, 0x35, + 0xdf, 0xfa, 0xb4, 0x03, 0x7d, 0x2a, 0x6a, 0x14, 0x06, 0x6b, 0x54, 0x77, 0xa5, 0x7b, 0x32, 0x37, + 0x50, 0xe5, 0xfe, 0x37, 0x50, 0x47, 0xe8, 0xd1, 0xbf, 0x31, 0x04, 0x59, 0x77, 0x47, 0x74, 0x03, + 0x26, 0x5a, 0x4e, 0xdb, 0xb9, 0xe9, 0xf9, 0x5e, 0xe2, 0x91, 0xb8, 0xd7, 0xd5, 0xf5, 0x8a, 0x46, + 0x27, 0xae, 0x7b, 0x34, 0x08, 0x36, 0xf8, 0xa0, 0x05, 0x80, 0x76, 0xe4, 0xed, 0x7a, 0x3e, 0xd9, + 0x64, 0x27, 0x1a, 0x16, 0xf3, 0xc0, 0xef, 0x63, 0x25, 0x14, 0x6b, 0x14, 0x39, 0x3e, 0xf2, 0xe5, + 0x7b, 0xe7, 0x23, 0x3f, 0x74, 0x44, 0x1f, 0xf9, 0xe1, 0x81, 0x7c, 0xe4, 0x31, 0xdc, 0x27, 0xf7, + 0x6e, 0xfa, 0x7f, 0xcd, 0xf3, 0x89, 0x50, 0xd8, 0x78, 0x24, 0xc4, 0xdc, 0xc1, 0xfe, 0xfc, 0x7d, + 0x38, 0x97, 0x02, 0x17, 0x94, 0x44, 0x3f, 0x09, 0x55, 0xc7, 0xf7, 0xc3, 0x5b, 0xaa, 0xd7, 0x56, + 0xe3, 0x96, 0xe3, 0xa7, 0xa9, 0x4b, 0xc7, 0x96, 0x1f, 0x3c, 0xd8, 0x9f, 0xaf, 0x2e, 0x15, 0xd0, + 0xe0, 0xc2, 0xd2, 0xf6, 0x36, 0x9c, 0x68, 0x92, 0x48, 0x3e, 0x5c, 0xa6, 0x96, 0xd8, 0x3a, 0x54, + 0xa2, 0x8c, 0x50, 0x19, 0x28, 0x5c, 0x5e, 0x4b, 0x18, 0x26, 0x85, 0x48, 0xca, 0xc8, 0xfe, 0x73, + 0x0b, 0x46, 0x85, 0x0b, 0xe5, 0x31, 0xe8, 0x32, 0x4b, 0x86, 0x59, 0x69, 0x3e, 0x5f, 0xf0, 0xb2, + 0xc6, 0x14, 0x1a, 0x94, 0xea, 0x19, 0x83, 0xd2, 0xc3, 0xbd, 0x98, 0xf4, 0x36, 0x25, 0xfd, 0x62, + 0x19, 0xa6, 0x4c, 0xf7, 0xd1, 0x63, 0xe8, 0x82, 0xab, 0x30, 0x1a, 0x0b, 0x5f, 0xe5, 0x52, 0xb1, + 0xcf, 0x5b, 0x76, 0x10, 0xd3, 0x9b, 0x71, 0xe1, 0x9d, 0x2c, 0x99, 0xe4, 0x3a, 0x41, 0x97, 0xef, + 0xa1, 0x13, 0x74, 0x3f, 0x0f, 0xde, 0xa1, 0xbb, 0xe1, 0xc1, 0x6b, 0x7f, 0x93, 0x09, 0x7f, 0x1d, + 0x7e, 0x0c, 0x7a, 0xc1, 0x05, 0x73, 0x9b, 0xb0, 0x7b, 0xcc, 0x2c, 0xd1, 0xa8, 0x02, 0xfd, 0xe0, + 0x9f, 0x58, 0x30, 0x2e, 0x08, 0x8f, 0xa1, 0xd9, 0x9f, 0x31, 0x9b, 0xfd, 0x40, 0x8f, 0x66, 0x17, + 0xb4, 0xf7, 0xef, 0x96, 0x54, 0x7b, 0x1b, 0x61, 0x94, 0x0c, 0x94, 0xca, 0x7a, 0x8c, 0x9e, 0x06, + 0xc3, 0x56, 0xe8, 0x8b, 0xcd, 0xfc, 0xc1, 0x34, 0x18, 0x8e, 0xc3, 0x0f, 0xb5, 0xdf, 0x58, 0x51, + 0xb3, 0x58, 0xad, 0x30, 0x4a, 0xc4, 0x06, 0x9a, 0xc6, 0x6a, 0x85, 0x51, 0x82, 0x19, 0x06, 0xb9, + 0x00, 0xe9, 0x8b, 0xee, 0x22, 0x7a, 0xb4, 0x78, 0x15, 0x76, 0x12, 0xcf, 0x5f, 0xf0, 0x82, 0x24, + 0x4e, 0xa2, 0x85, 0x7a, 0x90, 0x5c, 0x8b, 0xf8, 0xd9, 0x40, 0x8b, 0x6e, 0x53, 0xbc, 0xb0, 0xc6, + 0x57, 0x86, 0x57, 0xb0, 0x3a, 0x86, 0xcd, 0xfb, 0x9e, 0xab, 0x02, 0x8e, 0x15, 0x85, 0xfd, 0x02, + 0x93, 0xc9, 0xac, 0x83, 0x8e, 0x16, 0x78, 0xf6, 0x9d, 0x31, 0xd5, 0xb5, 0xcc, 0xd8, 0x5b, 0xd3, + 0xc3, 0xdb, 0x7a, 0x8b, 0x40, 0x5a, 0xb1, 0xee, 0x4a, 0x9c, 0xc6, 0xc0, 0xa1, 0xcf, 0x76, 0x5d, + 0x03, 0x3e, 0xdd, 0x47, 0x96, 0x1e, 0xe1, 0xe2, 0x8f, 0x65, 0xe6, 0x63, 0x19, 0xcc, 0xea, 0x8d, + 0x6c, 0xb2, 0xf1, 0x15, 0x89, 0xc0, 0x29, 0x0d, 0x5a, 0x14, 0x27, 0x4b, 0x6e, 0x66, 0x79, 0x20, + 0x73, 0xb2, 0x94, 0x9f, 0xaf, 0x1d, 0x2d, 0x9f, 0x81, 0x71, 0xf5, 0x80, 0x4b, 0x83, 0xbf, 0x83, + 0x51, 0xe1, 0xba, 0xd4, 0x6a, 0x0a, 0xc6, 0x3a, 0x0d, 0x5a, 0x87, 0xe9, 0x98, 0xbf, 0x2e, 0x23, + 0x23, 0x1e, 0x84, 0xdd, 0xe0, 0x89, 0xcc, 0xdb, 0xf1, 0x12, 0x7d, 0xc8, 0x40, 0x7c, 0xb1, 0xca, + 0x18, 0x89, 0x2c, 0x0b, 0xf4, 0x0a, 0x4c, 0xf9, 0xfa, 0x2b, 0x9b, 0x0d, 0x61, 0x56, 0x50, 0xae, + 0x5c, 0xc6, 0x1b, 0x9c, 0x0d, 0x9c, 0xa1, 0xa6, 0x4a, 0x80, 0x0e, 0x11, 0x09, 0x74, 0x9c, 0x60, + 0x93, 0xc4, 0xe2, 0xf9, 0x09, 0xa6, 0x04, 0x5c, 0x2e, 0xa0, 0xc1, 0x85, 0xa5, 0xd1, 0x8b, 0x30, + 0x21, 0x3f, 0x5f, 0x8b, 0x00, 0x4a, 0x1d, 0x06, 0x35, 0x1c, 0x36, 0x28, 0xd1, 0x2d, 0x38, 0x25, + 0xff, 0xaf, 0x47, 0xce, 0xc6, 0x86, 0xd7, 0x12, 0x01, 0x58, 0xe3, 0x8c, 0xc5, 0x92, 0xf4, 0x9e, + 0x5e, 0xcd, 0x23, 0x3a, 0xdc, 0x9f, 0x3f, 0x2b, 0x7a, 0x2d, 0x17, 0xcf, 0x06, 0x31, 0x9f, 0x3f, + 0xba, 0x02, 0x27, 0xb6, 0x88, 0xe3, 0x27, 0x5b, 0x2b, 0x5b, 0xa4, 0xb5, 0x2d, 0x17, 0x11, 0x8b, + 0x2b, 0xd2, 0xdc, 0xec, 0x2e, 0x76, 0x93, 0xe0, 0xbc, 0x72, 0xe8, 0x2d, 0xa8, 0xb6, 0x3b, 0x37, + 0x7d, 0x2f, 0xde, 0xba, 0x1a, 0x26, 0xec, 0xc6, 0x52, 0xbd, 0x7f, 0x22, 0x02, 0x90, 0x54, 0x4c, + 0x55, 0xa3, 0x80, 0x0e, 0x17, 0x72, 0x40, 0xef, 0xc1, 0xa9, 0xcc, 0x64, 0xe0, 0x4f, 0xea, 0x88, + 0x40, 0xa5, 0xc7, 0xf3, 0x97, 0x53, 0x4e, 0x01, 0x1e, 0x16, 0x97, 0x8b, 0xc2, 0xf9, 0x55, 0xbc, + 0xbf, 0x7b, 0xec, 0x77, 0x69, 0x61, 0x4d, 0xbb, 0x41, 0x9f, 0x83, 0x09, 0x7d, 0x16, 0x89, 0x0d, + 0xe6, 0xd1, 0x7e, 0x2f, 0xca, 0x0a, 0xdd, 0x48, 0xcd, 0x28, 0x1d, 0x87, 0x0d, 0x8e, 0x36, 0x81, + 0xfc, 0xef, 0x43, 0x97, 0x61, 0xac, 0xe5, 0x7b, 0x24, 0x48, 0xea, 0x8d, 0x5e, 0x81, 0xb3, 0x2b, + 0x82, 0x46, 0x74, 0x98, 0xc8, 0xdf, 0xc4, 0x61, 0x58, 0x71, 0xb0, 0x7f, 0xbf, 0x04, 0xf3, 0x7d, + 0x52, 0x78, 0x65, 0x6c, 0x80, 0xd6, 0x40, 0x36, 0xc0, 0x25, 0xf9, 0x9a, 0xcb, 0xd5, 0xcc, 0xf9, + 0x33, 0xf3, 0x52, 0x4b, 0x7a, 0x0a, 0xcd, 0xd2, 0x0f, 0xec, 0xfe, 0xa6, 0x9b, 0x11, 0x87, 0xfa, + 0x7a, 0x01, 0x36, 0x74, 0x7b, 0xf0, 0xf0, 0xe0, 0x1a, 0x7d, 0xa1, 0x29, 0xd8, 0xfe, 0x66, 0x09, + 0x4e, 0xa9, 0x2e, 0xfc, 0xf1, 0xed, 0xb8, 0xeb, 0xdd, 0x1d, 0x77, 0x17, 0x0c, 0xe9, 0xf6, 0x35, + 0x18, 0x69, 0xee, 0xc5, 0xad, 0xc4, 0x1f, 0x40, 0x01, 0x7a, 0xc4, 0x58, 0xa0, 0xe9, 0x36, 0xcd, + 0x1e, 0x64, 0x13, 0xeb, 0xd5, 0xfe, 0xab, 0x16, 0x4c, 0xaf, 0xaf, 0x34, 0x9a, 0x61, 0x6b, 0x9b, + 0x24, 0x4b, 0xdc, 0x4c, 0x84, 0x85, 0xfe, 0x63, 0xdd, 0xa1, 0x5e, 0x93, 0xa7, 0x31, 0x9d, 0x85, + 0xa1, 0xad, 0x30, 0x4e, 0xb2, 0x97, 0x25, 0x17, 0xc3, 0x38, 0xc1, 0x0c, 0x63, 0xff, 0xb1, 0x05, + 0xc3, 0xec, 0x0d, 0xb2, 0x7e, 0x6f, 0xd5, 0x0d, 0xf2, 0x5d, 0xe8, 0x79, 0x18, 0x21, 0x1b, 0x1b, + 0xa4, 0x95, 0x88, 0x51, 0x95, 0x11, 0x39, 0x23, 0xab, 0x0c, 0x4a, 0x37, 0x7d, 0x56, 0x19, 0xff, + 0x8b, 0x05, 0x31, 0xfa, 0x2c, 0x54, 0x12, 0x6f, 0x87, 0x2c, 0xb9, 0xae, 0xb8, 0xa7, 0x38, 0x9a, + 0x4b, 0x9a, 0x52, 0x42, 0xd6, 0x25, 0x13, 0x9c, 0xf2, 0xb3, 0x7f, 0xbe, 0x04, 0x90, 0x46, 0xee, + 0xf5, 0xfb, 0xcc, 0xe5, 0xae, 0x27, 0xf9, 0x1e, 0xcd, 0x79, 0x92, 0x0f, 0xa5, 0x0c, 0x73, 0x1e, + 0xe4, 0x53, 0x5d, 0x55, 0x1e, 0xa8, 0xab, 0x86, 0x8e, 0xd2, 0x55, 0x2b, 0x30, 0x9b, 0x46, 0x1e, + 0x9a, 0x61, 0xd8, 0x2c, 0x5d, 0xef, 0x7a, 0x16, 0x89, 0xbb, 0xe9, 0xed, 0x2f, 0x59, 0x20, 0xdc, + 0x94, 0x07, 0x98, 0xd0, 0x6f, 0xca, 0xd7, 0xb3, 0x8c, 0xbc, 0x82, 0x67, 0x8b, 0xfd, 0xb6, 0x45, + 0x36, 0x41, 0xb5, 0x81, 0x18, 0x39, 0x04, 0x0d, 0x5e, 0xf6, 0xef, 0x5a, 0x30, 0xce, 0xd1, 0x2c, + 0x67, 0xdd, 0x00, 0xad, 0x39, 0x52, 0xb2, 0x67, 0xf6, 0xb0, 0x14, 0x65, 0xac, 0x72, 0x02, 0xeb, + 0x0f, 0x4b, 0x49, 0x04, 0x4e, 0x69, 0xd0, 0xe3, 0x30, 0x1a, 0x77, 0x6e, 0x32, 0xf2, 0x8c, 0xa7, + 0x72, 0x93, 0x83, 0xb1, 0xc4, 0xd3, 0x79, 0x35, 0x93, 0x75, 0x54, 0x47, 0x17, 0x61, 0x84, 0x8b, + 0x0d, 0xb1, 0x8c, 0x7b, 0xdc, 0xca, 0x68, 0xee, 0xed, 0xc0, 0x5f, 0x42, 0x67, 0xe2, 0x46, 0x94, + 0x47, 0x6f, 0xc1, 0xb8, 0x1b, 0xde, 0x0a, 0x6e, 0x39, 0x91, 0xbb, 0xd4, 0xa8, 0x8b, 0x5e, 0xcf, + 0xd5, 0x3e, 0x6a, 0x29, 0x99, 0xee, 0x32, 0xcf, 0x2c, 0x90, 0x29, 0x0a, 0xeb, 0xec, 0xd0, 0x3a, + 0x4b, 0x86, 0xc2, 0xdf, 0x67, 0xed, 0xe5, 0x80, 0xa3, 0x9e, 0x74, 0xd5, 0x38, 0x4f, 0x8a, 0x8c, + 0x29, 0xe2, 0x75, 0xd7, 0x94, 0x91, 0xfd, 0x85, 0x13, 0x60, 0x8c, 0xb6, 0x91, 0x92, 0xd9, 0xba, + 0x4b, 0x29, 0x99, 0x31, 0x8c, 0x91, 0x9d, 0x76, 0xb2, 0x57, 0xf3, 0xa2, 0x5e, 0x39, 0xf2, 0x57, + 0x05, 0x4d, 0x37, 0x4f, 0x89, 0xc1, 0x8a, 0x4f, 0x7e, 0xde, 0xec, 0xf2, 0x07, 0x98, 0x37, 0x7b, + 0xe8, 0x18, 0xf3, 0x66, 0x5f, 0x85, 0xd1, 0x4d, 0x2f, 0xc1, 0xa4, 0x1d, 0x8a, 0x2d, 0x33, 0x77, + 0x26, 0x5c, 0xe0, 0x24, 0xdd, 0xd9, 0x5d, 0x05, 0x02, 0x4b, 0x26, 0xe8, 0x55, 0xb5, 0x06, 0x46, + 0x8a, 0x35, 0xce, 0x6e, 0x03, 0x7e, 0xee, 0x2a, 0x10, 0x79, 0xb2, 0x47, 0xef, 0x34, 0x4f, 0xb6, + 0xca, 0x73, 0x3d, 0xf6, 0xfe, 0xf2, 0x5c, 0x1b, 0x79, 0xc0, 0x2b, 0x77, 0x2f, 0x0f, 0xf8, 0x97, + 0x2c, 0x38, 0xd5, 0xce, 0x4b, 0x89, 0x2f, 0x32, 0x56, 0x3f, 0x3f, 0xf0, 0xd3, 0x00, 0x46, 0x85, + 0xec, 0xe8, 0x91, 0x4b, 0x86, 0xf3, 0xab, 0x93, 0x09, 0xc5, 0xc7, 0xef, 0x34, 0xa1, 0xf8, 0xbd, + 0x49, 0x6d, 0x9d, 0xa6, 0x17, 0x9f, 0x7c, 0xdf, 0xe9, 0xc5, 0x5f, 0x55, 0xe9, 0xc5, 0x7b, 0xa4, + 0x9c, 0xe0, 0xc9, 0xc3, 0xfb, 0x26, 0x15, 0xd7, 0x12, 0x83, 0x4f, 0xdf, 0x8d, 0xc4, 0xe0, 0x6f, + 0x9b, 0xc2, 0x9e, 0x67, 0xa9, 0x7e, 0xb2, 0x8f, 0xb0, 0x37, 0xf8, 0xf6, 0x16, 0xf7, 0x3c, 0x09, + 0xfa, 0xec, 0x1d, 0x25, 0x41, 0xbf, 0xa1, 0xa7, 0x17, 0x47, 0x7d, 0xf2, 0x67, 0x53, 0xa2, 0x01, + 0x93, 0x8a, 0xdf, 0xd0, 0xb7, 0xa0, 0x13, 0xc5, 0x7c, 0xd5, 0x4e, 0xd3, 0xcd, 0x37, 0x6f, 0x13, + 0xea, 0x4e, 0x56, 0x7e, 0xf2, 0x78, 0x92, 0x95, 0x9f, 0xba, 0xeb, 0xc9, 0xca, 0xef, 0x3b, 0x86, + 0x64, 0xe5, 0xf7, 0x7f, 0xa0, 0xc9, 0xca, 0xab, 0xf7, 0x36, 0x59, 0xf9, 0xe9, 0xbb, 0x91, 0xac, + 0xfc, 0x06, 0x54, 0xda, 0x32, 0x96, 0xb1, 0x3a, 0x57, 0x3c, 0x24, 0xb9, 0x01, 0x8f, 0x7c, 0x48, + 0x14, 0x0a, 0xa7, 0xac, 0x28, 0xdf, 0x34, 0x79, 0xf9, 0x03, 0x3d, 0x8c, 0x4b, 0x79, 0xc7, 0xf6, + 0x1e, 0x29, 0xcb, 0xff, 0x5a, 0x09, 0xce, 0xf4, 0x9e, 0xd7, 0xe9, 0x99, 0xbf, 0x91, 0xda, 0xa8, + 0x33, 0x67, 0x7e, 0xa6, 0x74, 0x69, 0x54, 0x03, 0x07, 0x7c, 0x5f, 0x80, 0x59, 0xe5, 0xd2, 0xe5, + 0x7b, 0xad, 0x3d, 0xed, 0xbd, 0x21, 0x15, 0x25, 0xd0, 0xcc, 0x12, 0xe0, 0xee, 0x32, 0x68, 0x09, + 0xa6, 0x0d, 0x60, 0xbd, 0x26, 0x54, 0x72, 0x65, 0x64, 0x68, 0x9a, 0x68, 0x9c, 0xa5, 0xb7, 0xbf, + 0x66, 0xc1, 0xfd, 0x05, 0x19, 0x4c, 0x07, 0x8e, 0x67, 0xde, 0x80, 0xe9, 0xb6, 0x59, 0xb4, 0x4f, + 0xda, 0x03, 0x23, 0x4f, 0xaa, 0x6a, 0x6b, 0x06, 0x81, 0xb3, 0x4c, 0x97, 0xcf, 0x7d, 0xeb, 0xfb, + 0x67, 0x3e, 0xf2, 0x87, 0xdf, 0x3f, 0xf3, 0x91, 0xef, 0x7d, 0xff, 0xcc, 0x47, 0xfe, 0xd2, 0xc1, + 0x19, 0xeb, 0x5b, 0x07, 0x67, 0xac, 0x3f, 0x3c, 0x38, 0x63, 0x7d, 0xef, 0xe0, 0x8c, 0xf5, 0x27, + 0x07, 0x67, 0xac, 0x9f, 0xff, 0xc1, 0x99, 0x8f, 0xbc, 0x59, 0xda, 0x7d, 0xe6, 0xff, 0x05, 0x00, + 0x00, 0xff, 0xff, 0xa5, 0x3e, 0x76, 0x4b, 0x79, 0xce, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/core/v1/generated.proto b/staging/src/k8s.io/api/core/v1/generated.proto index 63d797e018a0c..fda350d46c722 100644 --- a/staging/src/k8s.io/api/core/v1/generated.proto +++ b/staging/src/k8s.io/api/core/v1/generated.proto @@ -124,6 +124,25 @@ message AzureDiskVolumeSource { optional string kind = 6; } +// AzureFile represents an Azure File Service mount on the host and bind mount to the pod. +message AzureFilePersistentVolumeSource { + // the name of secret that contains Azure Storage Account Name and Key + optional string secretName = 1; + + // Share Name + optional string shareName = 2; + + // Defaults to false (read/write). ReadOnly here will force + // the ReadOnly setting in VolumeMounts. + // +optional + optional bool readOnly = 3; + + // the namespace of the secret that contains Azure Storage Account Name and Key + // default is the same as the Pod + // +optional + optional string secretNamespace = 4; +} + // AzureFile represents an Azure File Service mount on the host and bind mount to the pod. message AzureFileVolumeSource { // the name of secret that contains Azure Storage Account Name and Key @@ -161,6 +180,39 @@ message Capabilities { repeated string drop = 2; } +// Represents a Ceph Filesystem mount that lasts the lifetime of a pod +// Cephfs volumes do not support ownership management or SELinux relabeling. +message CephFSPersistentVolumeSource { + // Required: Monitors is a collection of Ceph monitors + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + repeated string monitors = 1; + + // Optional: Used as the mounted root, rather than the full Ceph tree, default is / + // +optional + optional string path = 2; + + // Optional: User is the rados user name, default is admin + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + // +optional + optional string user = 3; + + // Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + // +optional + optional string secretFile = 4; + + // Optional: SecretRef is reference to the authentication secret for User, default is empty. + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + // +optional + optional SecretReference secretRef = 5; + + // Optional: Defaults to false (read/write). ReadOnly here will force + // the ReadOnly setting in VolumeMounts. + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + // +optional + optional bool readOnly = 6; +} + // Represents a Ceph Filesystem mount that lasts the lifetime of a pod // Cephfs volumes do not support ownership management or SELinux relabeling. message CephFSVolumeSource { @@ -217,6 +269,15 @@ message CinderVolumeSource { optional bool readOnly = 3; } +// ClientIPConfig represents the configurations of Client IP based session affinity. +message ClientIPConfig { + // timeoutSeconds specifies the seconds of ClientIP type session sticky time. + // The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + // Default value is 10800(for 3 hours). + // +optional + optional int32 timeoutSeconds = 1; +} + // Information about the condition of a component. message ComponentCondition { // Type of condition for a component. @@ -922,7 +983,7 @@ message EnvVarSource { optional ObjectFieldSelector fieldRef = 1; // Selects a resource of the container: only resources limits and requests - // (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + // (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. // +optional optional ResourceFieldSelector resourceFieldRef = 2; @@ -1223,8 +1284,15 @@ message HostAlias { // Host path volumes do not support ownership management or SELinux relabeling. message HostPathVolumeSource { // Path of the directory on the host. + // If the path is a symlink, it will follow the link to the real path. // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath optional string path = 1; + + // Type for HostPath Volume + // Defaults to "" + // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + // +optional + optional string type = 2; } // Represents an ISCSI disk. @@ -1274,6 +1342,12 @@ message ISCSIVolumeSource { // CHAP secret for iSCSI target and initiator authentication // +optional optional LocalObjectReference secretRef = 10; + + // Custom iSCSI initiator name. + // If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + // : will be created for the connection. + // +optional + optional string initiatorName = 12; } // Maps a string key to a path within a volume. @@ -1655,8 +1729,6 @@ message NodeSelector { // that relates the key and values. message NodeSelectorRequirement { // The label key that the selector applies to. - // +patchMergeKey=key - // +patchStrategy=merge optional string key = 1; // Represents a key's relationship to a set of values. @@ -2193,7 +2265,7 @@ message PersistentVolumeSource { // CephFS represents a Ceph FS mount on the host that shares a pod's lifetime // +optional - optional CephFSVolumeSource cephfs = 9; + optional CephFSPersistentVolumeSource cephfs = 9; // FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. // +optional @@ -2211,7 +2283,7 @@ message PersistentVolumeSource { // AzureFile represents an Azure File Service mount on the host and bind mount to the pod. // +optional - optional AzureFileVolumeSource azureFile = 13; + optional AzureFilePersistentVolumeSource azureFile = 13; // VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine // +optional @@ -2279,6 +2351,12 @@ message PersistentVolumeSpec { // means that this volume does not belong to any StorageClass. // +optional optional string storageClassName = 6; + + // A list of mount options, e.g. ["ro", "soft"]. Not validated - mount will + // simply fail if one is invalid. + // More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options + // +optional + repeated string mountOptions = 7; } // PersistentVolumeStatus is the current status of a persistent volume. @@ -2642,7 +2720,7 @@ message PodSpec { // More info: https://kubernetes.io/docs/concepts/storage/volumes // +optional // +patchMergeKey=name - // +patchStrategy=merge + // +patchStrategy=merge,retainKeys repeated Volume volumes = 1; // List of initialization containers belonging to the pod. @@ -3461,6 +3539,18 @@ message SecretProjection { optional bool optional = 4; } +// SecretReference represents a Secret Reference. It has enough information to retrieve secret +// in any namespace +message SecretReference { + // Name is unique within a namespace to reference a secret resource. + // +optional + optional string name = 1; + + // Namespace defines the space within which the secret name must be unique. + // +optional + optional string namespace = 2; +} + // Adapts a Secret into a volume. // // The contents of the target Secret's Data field will be presented in a volume @@ -3789,6 +3879,10 @@ message ServiceSpec { // field. // +optional optional bool publishNotReadyAddresses = 13; + + // sessionAffinityConfig contains the configurations of session affinity. + // +optional + optional SessionAffinityConfig sessionAffinityConfig = 14; } // ServiceStatus represents the current status of a service. @@ -3799,6 +3893,13 @@ message ServiceStatus { optional LoadBalancerStatus loadBalancer = 1; } +// SessionAffinityConfig represents the configurations of session affinity. +message SessionAffinityConfig { + // clientIP contains the configurations of Client IP based session affinity. + // +optional + optional ClientIPConfig clientIP = 1; +} + // Represents a StorageOS persistent volume resource. message StorageOSPersistentVolumeSource { // VolumeName is the human-readable name of the StorageOS volume. Volume @@ -3888,8 +3989,6 @@ message TCPSocketAction { // any pod that does not tolerate the Taint. message Taint { // Required. The taint key to be applied to a node. - // +patchMergeKey=key - // +patchStrategy=merge optional string key = 1; // Required. The taint value corresponding to the taint key. @@ -3913,8 +4012,6 @@ message Toleration { // Key is the taint key that the toleration applies to. Empty means match all taint keys. // If the key is empty, operator must be Exists; this combination means to match all values and all keys. // +optional - // +patchMergeKey=key - // +patchStrategy=merge optional string key = 1; // Operator represents a key's relationship to the value. diff --git a/staging/src/k8s.io/api/core/v1/resource.go b/staging/src/k8s.io/api/core/v1/resource.go index 0d1c1dccd29dc..3bd6fec62fa60 100644 --- a/staging/src/k8s.io/api/core/v1/resource.go +++ b/staging/src/k8s.io/api/core/v1/resource.go @@ -55,8 +55,8 @@ func (self *ResourceList) NvidiaGPU() *resource.Quantity { return &resource.Quantity{} } -func (self *ResourceList) StorageOverlay() *resource.Quantity { - if val, ok := (*self)[ResourceStorageOverlay]; ok { +func (self *ResourceList) StorageEphemeral() *resource.Quantity { + if val, ok := (*self)[ResourceEphemeralStorage]; ok { return &val } return &resource.Quantity{} diff --git a/staging/src/k8s.io/api/core/v1/types.generated.go b/staging/src/k8s.io/api/core/v1/types.generated.go index 1c69913dbc12f..fd449cd3d903a 100644 --- a/staging/src/k8s.io/api/core/v1/types.generated.go +++ b/staging/src/k8s.io/api/core/v1/types.generated.go @@ -6069,7 +6069,7 @@ func (x *PersistentVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Deco } } else { if x.CephFS == nil { - x.CephFS = new(CephFSVolumeSource) + x.CephFS = new(CephFSPersistentVolumeSource) } x.CephFS.CodecDecodeSelf(d) } @@ -6113,7 +6113,7 @@ func (x *PersistentVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Deco } } else { if x.AzureFile == nil { - x.AzureFile = new(AzureFileVolumeSource) + x.AzureFile = new(AzureFilePersistentVolumeSource) } x.AzureFile.CodecDecodeSelf(d) } @@ -6404,7 +6404,7 @@ func (x *PersistentVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.De } } else { if x.CephFS == nil { - x.CephFS = new(CephFSVolumeSource) + x.CephFS = new(CephFSPersistentVolumeSource) } x.CephFS.CodecDecodeSelf(d) } @@ -6488,7 +6488,7 @@ func (x *PersistentVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.De } } else { if x.AzureFile == nil { - x.AzureFile = new(AzureFileVolumeSource) + x.AzureFile = new(AzureFilePersistentVolumeSource) } x.AzureFile.CodecDecodeSelf(d) } @@ -7081,7 +7081,7 @@ func (x *PersistentVolumeSpec) CodecEncodeSelf(e *codec1978.Encoder) { } else { yysep2 := !z.EncBinary() yy2arr2 := z.EncBasicHandle().StructToArray - var yyq2 [26]bool + var yyq2 [27]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false yyq2[0] = len(x.Capacity) != 0 @@ -7110,9 +7110,10 @@ func (x *PersistentVolumeSpec) CodecEncodeSelf(e *codec1978.Encoder) { yyq2[23] = x.ClaimRef != nil yyq2[24] = x.PersistentVolumeReclaimPolicy != "" yyq2[25] = x.StorageClassName != "" + yyq2[26] = len(x.MountOptions) != 0 var yynn2 int if yyr2 || yy2arr2 { - r.EncodeArrayStart(26) + r.EncodeArrayStart(27) } else { yynn2 = 0 for _, b := range yyq2 { @@ -8019,6 +8020,39 @@ func (x *PersistentVolumeSpec) CodecEncodeSelf(e *codec1978.Encoder) { } } } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[26] { + if x.MountOptions == nil { + r.EncodeNil() + } else { + yym82 := z.EncBinary() + _ = yym82 + if false { + } else { + z.F.EncSliceStringV(x.MountOptions, false, e) + } + } + } else { + r.EncodeNil() + } + } else { + if yyq2[26] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("mountOptions")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.MountOptions == nil { + r.EncodeNil() + } else { + yym83 := z.EncBinary() + _ = yym83 + if false { + } else { + z.F.EncSliceStringV(x.MountOptions, false, e) + } + } + } + } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayEnd1234) } else { @@ -8201,7 +8235,7 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromMap(l int, d *codec1978.Decode } case "cephfs": if x.PersistentVolumeSource.CephFS == nil { - x.PersistentVolumeSource.CephFS = new(CephFSVolumeSource) + x.PersistentVolumeSource.CephFS = new(CephFSPersistentVolumeSource) } if r.TryDecodeAsNil() { if x.CephFS != nil { @@ -8209,7 +8243,7 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromMap(l int, d *codec1978.Decode } } else { if x.CephFS == nil { - x.CephFS = new(CephFSVolumeSource) + x.CephFS = new(CephFSPersistentVolumeSource) } x.CephFS.CodecDecodeSelf(d) } @@ -8257,7 +8291,7 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromMap(l int, d *codec1978.Decode } case "azureFile": if x.PersistentVolumeSource.AzureFile == nil { - x.PersistentVolumeSource.AzureFile = new(AzureFileVolumeSource) + x.PersistentVolumeSource.AzureFile = new(AzureFilePersistentVolumeSource) } if r.TryDecodeAsNil() { if x.AzureFile != nil { @@ -8265,7 +8299,7 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromMap(l int, d *codec1978.Decode } } else { if x.AzureFile == nil { - x.AzureFile = new(AzureFileVolumeSource) + x.AzureFile = new(AzureFilePersistentVolumeSource) } x.AzureFile.CodecDecodeSelf(d) } @@ -8423,6 +8457,18 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromMap(l int, d *codec1978.Decode *((*string)(yyv30)) = r.DecodeString() } } + case "mountOptions": + if r.TryDecodeAsNil() { + x.MountOptions = nil + } else { + yyv32 := &x.MountOptions + yym33 := z.DecBinary() + _ = yym33 + if false { + } else { + z.F.DecSliceStringX(yyv32, false, d) + } + } default: z.DecStructFieldNotFound(-1, yys3) } // end switch yys3 @@ -8434,16 +8480,16 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj32 int - var yyb32 bool - var yyhl32 bool = l >= 0 - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + var yyj34 int + var yyb34 bool + var yyhl34 bool = l >= 0 + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8451,19 +8497,19 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.Capacity = nil } else { - yyv33 := &x.Capacity - yyv33.CodecDecodeSelf(d) + yyv35 := &x.Capacity + yyv35.CodecDecodeSelf(d) } if x.PersistentVolumeSource.GCEPersistentDisk == nil { x.PersistentVolumeSource.GCEPersistentDisk = new(GCEPersistentDiskVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8481,13 +8527,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.AWSElasticBlockStore == nil { x.PersistentVolumeSource.AWSElasticBlockStore = new(AWSElasticBlockStoreVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8505,13 +8551,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.HostPath == nil { x.PersistentVolumeSource.HostPath = new(HostPathVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8529,13 +8575,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.Glusterfs == nil { x.PersistentVolumeSource.Glusterfs = new(GlusterfsVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8553,13 +8599,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.NFS == nil { x.PersistentVolumeSource.NFS = new(NFSVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8577,13 +8623,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.RBD == nil { x.PersistentVolumeSource.RBD = new(RBDVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8601,13 +8647,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.ISCSI == nil { x.PersistentVolumeSource.ISCSI = new(ISCSIVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8625,13 +8671,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.Cinder == nil { x.PersistentVolumeSource.Cinder = new(CinderVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8647,15 +8693,15 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco x.Cinder.CodecDecodeSelf(d) } if x.PersistentVolumeSource.CephFS == nil { - x.PersistentVolumeSource.CephFS = new(CephFSVolumeSource) + x.PersistentVolumeSource.CephFS = new(CephFSPersistentVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8666,20 +8712,20 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco } } else { if x.CephFS == nil { - x.CephFS = new(CephFSVolumeSource) + x.CephFS = new(CephFSPersistentVolumeSource) } x.CephFS.CodecDecodeSelf(d) } if x.PersistentVolumeSource.FC == nil { x.PersistentVolumeSource.FC = new(FCVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8697,13 +8743,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.Flocker == nil { x.PersistentVolumeSource.Flocker = new(FlockerVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8721,13 +8767,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.FlexVolume == nil { x.PersistentVolumeSource.FlexVolume = new(FlexVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8743,15 +8789,15 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco x.FlexVolume.CodecDecodeSelf(d) } if x.PersistentVolumeSource.AzureFile == nil { - x.PersistentVolumeSource.AzureFile = new(AzureFileVolumeSource) + x.PersistentVolumeSource.AzureFile = new(AzureFilePersistentVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8762,20 +8808,20 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco } } else { if x.AzureFile == nil { - x.AzureFile = new(AzureFileVolumeSource) + x.AzureFile = new(AzureFilePersistentVolumeSource) } x.AzureFile.CodecDecodeSelf(d) } if x.PersistentVolumeSource.VsphereVolume == nil { x.PersistentVolumeSource.VsphereVolume = new(VsphereVirtualDiskVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8793,13 +8839,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.Quobyte == nil { x.PersistentVolumeSource.Quobyte = new(QuobyteVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8817,13 +8863,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.AzureDisk == nil { x.PersistentVolumeSource.AzureDisk = new(AzureDiskVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8841,13 +8887,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.PhotonPersistentDisk == nil { x.PersistentVolumeSource.PhotonPersistentDisk = new(PhotonPersistentDiskVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8865,13 +8911,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.PortworxVolume == nil { x.PersistentVolumeSource.PortworxVolume = new(PortworxVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8889,13 +8935,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.ScaleIO == nil { x.PersistentVolumeSource.ScaleIO = new(ScaleIOVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8913,13 +8959,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.Local == nil { x.PersistentVolumeSource.Local = new(LocalVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8937,13 +8983,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if x.PersistentVolumeSource.StorageOS == nil { x.PersistentVolumeSource.StorageOS = new(StorageOSPersistentVolumeSource) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8958,13 +9004,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco } x.StorageOS.CodecDecodeSelf(d) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -8972,21 +9018,21 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.AccessModes = nil } else { - yyv55 := &x.AccessModes - yym56 := z.DecBinary() - _ = yym56 + yyv57 := &x.AccessModes + yym58 := z.DecBinary() + _ = yym58 if false { } else { - h.decSlicePersistentVolumeAccessMode((*[]PersistentVolumeAccessMode)(yyv55), d) + h.decSlicePersistentVolumeAccessMode((*[]PersistentVolumeAccessMode)(yyv57), d) } } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -9001,13 +9047,13 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco } x.ClaimRef.CodecDecodeSelf(d) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -9015,16 +9061,16 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.PersistentVolumeReclaimPolicy = "" } else { - yyv58 := &x.PersistentVolumeReclaimPolicy - yyv58.CodecDecodeSelf(d) + yyv60 := &x.PersistentVolumeReclaimPolicy + yyv60.CodecDecodeSelf(d) } - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -9032,26 +9078,48 @@ func (x *PersistentVolumeSpec) codecDecodeSelfFromArray(l int, d *codec1978.Deco if r.TryDecodeAsNil() { x.StorageClassName = "" } else { - yyv59 := &x.StorageClassName - yym60 := z.DecBinary() - _ = yym60 + yyv61 := &x.StorageClassName + yym62 := z.DecBinary() + _ = yym62 if false { } else { - *((*string)(yyv59)) = r.DecodeString() + *((*string)(yyv61)) = r.DecodeString() + } + } + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l + } else { + yyb34 = r.CheckBreak() + } + if yyb34 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.MountOptions = nil + } else { + yyv63 := &x.MountOptions + yym64 := z.DecBinary() + _ = yym64 + if false { + } else { + z.F.DecSliceStringX(yyv63, false, d) } } for { - yyj32++ - if yyhl32 { - yyb32 = yyj32 > l + yyj34++ + if yyhl34 { + yyb34 = yyj34 > l } else { - yyb32 = r.CheckBreak() + yyb34 = r.CheckBreak() } - if yyb32 { + if yyb34 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj32-1, "") + z.DecStructFieldNotFound(yyj34-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } @@ -11294,65 +11362,20 @@ func (x *PersistentVolumeClaimPhase) CodecDecodeSelf(d *codec1978.Decoder) { } } -func (x *HostPathVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { +func (x HostPathType) CodecEncodeSelf(e *codec1978.Encoder) { var h codecSelfer1234 z, r := codec1978.GenHelperEncoder(e) _, _, _ = h, z, r - if x == nil { - r.EncodeNil() + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { } else { - yym1 := z.EncBinary() - _ = yym1 - if false { - } else if z.HasExtensions() && z.EncExt(x) { - } else { - yysep2 := !z.EncBinary() - yy2arr2 := z.EncBasicHandle().StructToArray - var yyq2 [1]bool - _, _, _ = yysep2, yyq2, yy2arr2 - const yyr2 bool = false - var yynn2 int - if yyr2 || yy2arr2 { - r.EncodeArrayStart(1) - } else { - yynn2 = 1 - for _, b := range yyq2 { - if b { - yynn2++ - } - } - r.EncodeMapStart(yynn2) - yynn2 = 0 - } - if yyr2 || yy2arr2 { - z.EncSendContainerState(codecSelfer_containerArrayElem1234) - yym4 := z.EncBinary() - _ = yym4 - if false { - } else { - r.EncodeString(codecSelferC_UTF81234, string(x.Path)) - } - } else { - z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("path")) - z.EncSendContainerState(codecSelfer_containerMapValue1234) - yym5 := z.EncBinary() - _ = yym5 - if false { - } else { - r.EncodeString(codecSelferC_UTF81234, string(x.Path)) - } - } - if yyr2 || yy2arr2 { - z.EncSendContainerState(codecSelfer_containerArrayEnd1234) - } else { - z.EncSendContainerState(codecSelfer_containerMapEnd1234) - } - } + r.EncodeString(codecSelferC_UTF81234, string(x)) } } -func (x *HostPathVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { +func (x *HostPathType) CodecDecodeSelf(d *codec1978.Decoder) { var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r @@ -11361,114 +11384,11 @@ func (x *HostPathVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { if false { } else if z.HasExtensions() && z.DecExt(x) { } else { - yyct2 := r.ContainerType() - if yyct2 == codecSelferValueTypeMap1234 { - yyl2 := r.ReadMapStart() - if yyl2 == 0 { - z.DecSendContainerState(codecSelfer_containerMapEnd1234) - } else { - x.codecDecodeSelfFromMap(yyl2, d) - } - } else if yyct2 == codecSelferValueTypeArray1234 { - yyl2 := r.ReadArrayStart() - if yyl2 == 0 { - z.DecSendContainerState(codecSelfer_containerArrayEnd1234) - } else { - x.codecDecodeSelfFromArray(yyl2, d) - } - } else { - panic(codecSelferOnlyMapOrArrayEncodeToStructErr1234) - } - } -} - -func (x *HostPathVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { - var h codecSelfer1234 - z, r := codec1978.GenHelperDecoder(d) - _, _, _ = h, z, r - var yys3Slc = z.DecScratchBuffer() // default slice to decode into - _ = yys3Slc - var yyhl3 bool = l >= 0 - for yyj3 := 0; ; yyj3++ { - if yyhl3 { - if yyj3 >= l { - break - } - } else { - if r.CheckBreak() { - break - } - } - z.DecSendContainerState(codecSelfer_containerMapKey1234) - yys3Slc = r.DecodeBytes(yys3Slc, true, true) - yys3 := string(yys3Slc) - z.DecSendContainerState(codecSelfer_containerMapValue1234) - switch yys3 { - case "path": - if r.TryDecodeAsNil() { - x.Path = "" - } else { - yyv4 := &x.Path - yym5 := z.DecBinary() - _ = yym5 - if false { - } else { - *((*string)(yyv4)) = r.DecodeString() - } - } - default: - z.DecStructFieldNotFound(-1, yys3) - } // end switch yys3 - } // end for yyj3 - z.DecSendContainerState(codecSelfer_containerMapEnd1234) -} - -func (x *HostPathVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { - var h codecSelfer1234 - z, r := codec1978.GenHelperDecoder(d) - _, _, _ = h, z, r - var yyj6 int - var yyb6 bool - var yyhl6 bool = l >= 0 - yyj6++ - if yyhl6 { - yyb6 = yyj6 > l - } else { - yyb6 = r.CheckBreak() - } - if yyb6 { - z.DecSendContainerState(codecSelfer_containerArrayEnd1234) - return - } - z.DecSendContainerState(codecSelfer_containerArrayElem1234) - if r.TryDecodeAsNil() { - x.Path = "" - } else { - yyv7 := &x.Path - yym8 := z.DecBinary() - _ = yym8 - if false { - } else { - *((*string)(yyv7)) = r.DecodeString() - } - } - for { - yyj6++ - if yyhl6 { - yyb6 = yyj6 > l - } else { - yyb6 = r.CheckBreak() - } - if yyb6 { - break - } - z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj6-1, "") + *((*string)(x)) = r.DecodeString() } - z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } -func (x *EmptyDirVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { +func (x *HostPathVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { var h codecSelfer1234 z, r := codec1978.GenHelperEncoder(e) _, _, _ = h, z, r @@ -11485,13 +11405,12 @@ func (x *EmptyDirVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { var yyq2 [2]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false - yyq2[0] = x.Medium != "" - yyq2[1] = true + yyq2[1] = x.Type != nil var yynn2 int if yyr2 || yy2arr2 { r.EncodeArrayStart(2) } else { - yynn2 = 0 + yynn2 = 1 for _, b := range yyq2 { if b { yynn2++ @@ -11502,31 +11421,31 @@ func (x *EmptyDirVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayElem1234) - if yyq2[0] { - x.Medium.CodecEncodeSelf(e) + yym4 := z.EncBinary() + _ = yym4 + if false { } else { - r.EncodeString(codecSelferC_UTF81234, "") + r.EncodeString(codecSelferC_UTF81234, string(x.Path)) } } else { - if yyq2[0] { - z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("medium")) - z.EncSendContainerState(codecSelfer_containerMapValue1234) - x.Medium.CodecEncodeSelf(e) + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("path")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym5 := z.EncBinary() + _ = yym5 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.Path)) } } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayElem1234) if yyq2[1] { - yy7 := &x.SizeLimit - yym8 := z.EncBinary() - _ = yym8 - if false { - } else if z.HasExtensions() && z.EncExt(yy7) { - } else if !yym8 && z.IsJSONHandle() { - z.EncJSONMarshal(yy7) + if x.Type == nil { + r.EncodeNil() } else { - z.EncFallback(yy7) + yy7 := *x.Type + yy7.CodecEncodeSelf(e) } } else { r.EncodeNil() @@ -11534,17 +11453,13 @@ func (x *EmptyDirVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { } else { if yyq2[1] { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("sizeLimit")) + r.EncodeString(codecSelferC_UTF81234, string("type")) z.EncSendContainerState(codecSelfer_containerMapValue1234) - yy9 := &x.SizeLimit - yym10 := z.EncBinary() - _ = yym10 - if false { - } else if z.HasExtensions() && z.EncExt(yy9) { - } else if !yym10 && z.IsJSONHandle() { - z.EncJSONMarshal(yy9) + if x.Type == nil { + r.EncodeNil() } else { - z.EncFallback(yy9) + yy9 := *x.Type + yy9.CodecEncodeSelf(e) } } } @@ -11557,7 +11472,244 @@ func (x *EmptyDirVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { } } -func (x *EmptyDirVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { +func (x *HostPathVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerMapEnd1234) + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(codecSelferOnlyMapOrArrayEncodeToStructErr1234) + } + } +} + +func (x *HostPathVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yys3Slc = z.DecScratchBuffer() // default slice to decode into + _ = yys3Slc + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + z.DecSendContainerState(codecSelfer_containerMapKey1234) + yys3Slc = r.DecodeBytes(yys3Slc, true, true) + yys3 := string(yys3Slc) + z.DecSendContainerState(codecSelfer_containerMapValue1234) + switch yys3 { + case "path": + if r.TryDecodeAsNil() { + x.Path = "" + } else { + yyv4 := &x.Path + yym5 := z.DecBinary() + _ = yym5 + if false { + } else { + *((*string)(yyv4)) = r.DecodeString() + } + } + case "type": + if r.TryDecodeAsNil() { + if x.Type != nil { + x.Type = nil + } + } else { + if x.Type == nil { + x.Type = new(HostPathType) + } + x.Type.CodecDecodeSelf(d) + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + z.DecSendContainerState(codecSelfer_containerMapEnd1234) +} + +func (x *HostPathVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj7 int + var yyb7 bool + var yyhl7 bool = l >= 0 + yyj7++ + if yyhl7 { + yyb7 = yyj7 > l + } else { + yyb7 = r.CheckBreak() + } + if yyb7 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.Path = "" + } else { + yyv8 := &x.Path + yym9 := z.DecBinary() + _ = yym9 + if false { + } else { + *((*string)(yyv8)) = r.DecodeString() + } + } + yyj7++ + if yyhl7 { + yyb7 = yyj7 > l + } else { + yyb7 = r.CheckBreak() + } + if yyb7 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + if x.Type != nil { + x.Type = nil + } + } else { + if x.Type == nil { + x.Type = new(HostPathType) + } + x.Type.CodecDecodeSelf(d) + } + for { + yyj7++ + if yyhl7 { + yyb7 = yyj7 > l + } else { + yyb7 = r.CheckBreak() + } + if yyb7 { + break + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + z.DecStructFieldNotFound(yyj7-1, "") + } + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) +} + +func (x *EmptyDirVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + var yyq2 [2]bool + _, _, _ = yysep2, yyq2, yy2arr2 + const yyr2 bool = false + yyq2[0] = x.Medium != "" + yyq2[1] = true + var yynn2 int + if yyr2 || yy2arr2 { + r.EncodeArrayStart(2) + } else { + yynn2 = 0 + for _, b := range yyq2 { + if b { + yynn2++ + } + } + r.EncodeMapStart(yynn2) + yynn2 = 0 + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[0] { + x.Medium.CodecEncodeSelf(e) + } else { + r.EncodeString(codecSelferC_UTF81234, "") + } + } else { + if yyq2[0] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("medium")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + x.Medium.CodecEncodeSelf(e) + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[1] { + yy7 := &x.SizeLimit + yym8 := z.EncBinary() + _ = yym8 + if false { + } else if z.HasExtensions() && z.EncExt(yy7) { + } else if !yym8 && z.IsJSONHandle() { + z.EncJSONMarshal(yy7) + } else { + z.EncFallback(yy7) + } + } else { + r.EncodeNil() + } + } else { + if yyq2[1] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("sizeLimit")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yy9 := &x.SizeLimit + yym10 := z.EncBinary() + _ = yym10 + if false { + } else if z.HasExtensions() && z.EncExt(yy9) { + } else if !yym10 && z.IsJSONHandle() { + z.EncJSONMarshal(yy9) + } else { + z.EncFallback(yy9) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + z.EncSendContainerState(codecSelfer_containerMapEnd1234) + } + } + } +} + +func (x *EmptyDirVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r @@ -13353,7 +13505,7 @@ func (x *CephFSVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decode z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } -func (x *FlockerVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { +func (x *SecretReference) CodecEncodeSelf(e *codec1978.Encoder) { var h codecSelfer1234 z, r := codec1978.GenHelperEncoder(e) _, _, _ = h, z, r @@ -13370,8 +13522,8 @@ func (x *FlockerVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { var yyq2 [2]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false - yyq2[0] = x.DatasetName != "" - yyq2[1] = x.DatasetUUID != "" + yyq2[0] = x.Name != "" + yyq2[1] = x.Namespace != "" var yynn2 int if yyr2 || yy2arr2 { r.EncodeArrayStart(2) @@ -13392,7 +13544,7 @@ func (x *FlockerVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym4 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.DatasetName)) + r.EncodeString(codecSelferC_UTF81234, string(x.Name)) } } else { r.EncodeString(codecSelferC_UTF81234, "") @@ -13400,13 +13552,13 @@ func (x *FlockerVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { } else { if yyq2[0] { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("datasetName")) + r.EncodeString(codecSelferC_UTF81234, string("name")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym5 := z.EncBinary() _ = yym5 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.DatasetName)) + r.EncodeString(codecSelferC_UTF81234, string(x.Name)) } } } @@ -13417,7 +13569,7 @@ func (x *FlockerVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { _ = yym7 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.DatasetUUID)) + r.EncodeString(codecSelferC_UTF81234, string(x.Namespace)) } } else { r.EncodeString(codecSelferC_UTF81234, "") @@ -13425,13 +13577,13 @@ func (x *FlockerVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { } else { if yyq2[1] { z.EncSendContainerState(codecSelfer_containerMapKey1234) - r.EncodeString(codecSelferC_UTF81234, string("datasetUUID")) + r.EncodeString(codecSelferC_UTF81234, string("namespace")) z.EncSendContainerState(codecSelfer_containerMapValue1234) yym8 := z.EncBinary() _ = yym8 if false { } else { - r.EncodeString(codecSelferC_UTF81234, string(x.DatasetUUID)) + r.EncodeString(codecSelferC_UTF81234, string(x.Namespace)) } } } @@ -13444,7 +13596,7 @@ func (x *FlockerVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { } } -func (x *FlockerVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { +func (x *SecretReference) CodecDecodeSelf(d *codec1978.Decoder) { var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r @@ -13474,7 +13626,7 @@ func (x *FlockerVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { } } -func (x *FlockerVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { +func (x *SecretReference) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r @@ -13496,11 +13648,11 @@ func (x *FlockerVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder yys3 := string(yys3Slc) z.DecSendContainerState(codecSelfer_containerMapValue1234) switch yys3 { - case "datasetName": + case "name": if r.TryDecodeAsNil() { - x.DatasetName = "" + x.Name = "" } else { - yyv4 := &x.DatasetName + yyv4 := &x.Name yym5 := z.DecBinary() _ = yym5 if false { @@ -13508,11 +13660,11 @@ func (x *FlockerVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder *((*string)(yyv4)) = r.DecodeString() } } - case "datasetUUID": + case "namespace": if r.TryDecodeAsNil() { - x.DatasetUUID = "" + x.Namespace = "" } else { - yyv6 := &x.DatasetUUID + yyv6 := &x.Namespace yym7 := z.DecBinary() _ = yym7 if false { @@ -13527,7 +13679,7 @@ func (x *FlockerVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder z.DecSendContainerState(codecSelfer_containerMapEnd1234) } -func (x *FlockerVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { +func (x *SecretReference) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r @@ -13546,9 +13698,9 @@ func (x *FlockerVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decod } z.DecSendContainerState(codecSelfer_containerArrayElem1234) if r.TryDecodeAsNil() { - x.DatasetName = "" + x.Name = "" } else { - yyv9 := &x.DatasetName + yyv9 := &x.Name yym10 := z.DecBinary() _ = yym10 if false { @@ -13568,9 +13720,728 @@ func (x *FlockerVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decod } z.DecSendContainerState(codecSelfer_containerArrayElem1234) if r.TryDecodeAsNil() { - x.DatasetUUID = "" + x.Namespace = "" } else { - yyv11 := &x.DatasetUUID + yyv11 := &x.Namespace + yym12 := z.DecBinary() + _ = yym12 + if false { + } else { + *((*string)(yyv11)) = r.DecodeString() + } + } + for { + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + break + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + z.DecStructFieldNotFound(yyj8-1, "") + } + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) +} + +func (x *CephFSPersistentVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + var yyq2 [6]bool + _, _, _ = yysep2, yyq2, yy2arr2 + const yyr2 bool = false + yyq2[1] = x.Path != "" + yyq2[2] = x.User != "" + yyq2[3] = x.SecretFile != "" + yyq2[4] = x.SecretRef != nil + yyq2[5] = x.ReadOnly != false + var yynn2 int + if yyr2 || yy2arr2 { + r.EncodeArrayStart(6) + } else { + yynn2 = 1 + for _, b := range yyq2 { + if b { + yynn2++ + } + } + r.EncodeMapStart(yynn2) + yynn2 = 0 + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if x.Monitors == nil { + r.EncodeNil() + } else { + yym4 := z.EncBinary() + _ = yym4 + if false { + } else { + z.F.EncSliceStringV(x.Monitors, false, e) + } + } + } else { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("monitors")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.Monitors == nil { + r.EncodeNil() + } else { + yym5 := z.EncBinary() + _ = yym5 + if false { + } else { + z.F.EncSliceStringV(x.Monitors, false, e) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[1] { + yym7 := z.EncBinary() + _ = yym7 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.Path)) + } + } else { + r.EncodeString(codecSelferC_UTF81234, "") + } + } else { + if yyq2[1] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("path")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym8 := z.EncBinary() + _ = yym8 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.Path)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[2] { + yym10 := z.EncBinary() + _ = yym10 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.User)) + } + } else { + r.EncodeString(codecSelferC_UTF81234, "") + } + } else { + if yyq2[2] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("user")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym11 := z.EncBinary() + _ = yym11 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.User)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[3] { + yym13 := z.EncBinary() + _ = yym13 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.SecretFile)) + } + } else { + r.EncodeString(codecSelferC_UTF81234, "") + } + } else { + if yyq2[3] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("secretFile")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym14 := z.EncBinary() + _ = yym14 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.SecretFile)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[4] { + if x.SecretRef == nil { + r.EncodeNil() + } else { + x.SecretRef.CodecEncodeSelf(e) + } + } else { + r.EncodeNil() + } + } else { + if yyq2[4] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("secretRef")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.SecretRef == nil { + r.EncodeNil() + } else { + x.SecretRef.CodecEncodeSelf(e) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[5] { + yym19 := z.EncBinary() + _ = yym19 + if false { + } else { + r.EncodeBool(bool(x.ReadOnly)) + } + } else { + r.EncodeBool(false) + } + } else { + if yyq2[5] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("readOnly")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym20 := z.EncBinary() + _ = yym20 + if false { + } else { + r.EncodeBool(bool(x.ReadOnly)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + z.EncSendContainerState(codecSelfer_containerMapEnd1234) + } + } + } +} + +func (x *CephFSPersistentVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerMapEnd1234) + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(codecSelferOnlyMapOrArrayEncodeToStructErr1234) + } + } +} + +func (x *CephFSPersistentVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yys3Slc = z.DecScratchBuffer() // default slice to decode into + _ = yys3Slc + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + z.DecSendContainerState(codecSelfer_containerMapKey1234) + yys3Slc = r.DecodeBytes(yys3Slc, true, true) + yys3 := string(yys3Slc) + z.DecSendContainerState(codecSelfer_containerMapValue1234) + switch yys3 { + case "monitors": + if r.TryDecodeAsNil() { + x.Monitors = nil + } else { + yyv4 := &x.Monitors + yym5 := z.DecBinary() + _ = yym5 + if false { + } else { + z.F.DecSliceStringX(yyv4, false, d) + } + } + case "path": + if r.TryDecodeAsNil() { + x.Path = "" + } else { + yyv6 := &x.Path + yym7 := z.DecBinary() + _ = yym7 + if false { + } else { + *((*string)(yyv6)) = r.DecodeString() + } + } + case "user": + if r.TryDecodeAsNil() { + x.User = "" + } else { + yyv8 := &x.User + yym9 := z.DecBinary() + _ = yym9 + if false { + } else { + *((*string)(yyv8)) = r.DecodeString() + } + } + case "secretFile": + if r.TryDecodeAsNil() { + x.SecretFile = "" + } else { + yyv10 := &x.SecretFile + yym11 := z.DecBinary() + _ = yym11 + if false { + } else { + *((*string)(yyv10)) = r.DecodeString() + } + } + case "secretRef": + if r.TryDecodeAsNil() { + if x.SecretRef != nil { + x.SecretRef = nil + } + } else { + if x.SecretRef == nil { + x.SecretRef = new(SecretReference) + } + x.SecretRef.CodecDecodeSelf(d) + } + case "readOnly": + if r.TryDecodeAsNil() { + x.ReadOnly = false + } else { + yyv13 := &x.ReadOnly + yym14 := z.DecBinary() + _ = yym14 + if false { + } else { + *((*bool)(yyv13)) = r.DecodeBool() + } + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + z.DecSendContainerState(codecSelfer_containerMapEnd1234) +} + +func (x *CephFSPersistentVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj15 int + var yyb15 bool + var yyhl15 bool = l >= 0 + yyj15++ + if yyhl15 { + yyb15 = yyj15 > l + } else { + yyb15 = r.CheckBreak() + } + if yyb15 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.Monitors = nil + } else { + yyv16 := &x.Monitors + yym17 := z.DecBinary() + _ = yym17 + if false { + } else { + z.F.DecSliceStringX(yyv16, false, d) + } + } + yyj15++ + if yyhl15 { + yyb15 = yyj15 > l + } else { + yyb15 = r.CheckBreak() + } + if yyb15 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.Path = "" + } else { + yyv18 := &x.Path + yym19 := z.DecBinary() + _ = yym19 + if false { + } else { + *((*string)(yyv18)) = r.DecodeString() + } + } + yyj15++ + if yyhl15 { + yyb15 = yyj15 > l + } else { + yyb15 = r.CheckBreak() + } + if yyb15 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.User = "" + } else { + yyv20 := &x.User + yym21 := z.DecBinary() + _ = yym21 + if false { + } else { + *((*string)(yyv20)) = r.DecodeString() + } + } + yyj15++ + if yyhl15 { + yyb15 = yyj15 > l + } else { + yyb15 = r.CheckBreak() + } + if yyb15 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.SecretFile = "" + } else { + yyv22 := &x.SecretFile + yym23 := z.DecBinary() + _ = yym23 + if false { + } else { + *((*string)(yyv22)) = r.DecodeString() + } + } + yyj15++ + if yyhl15 { + yyb15 = yyj15 > l + } else { + yyb15 = r.CheckBreak() + } + if yyb15 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + if x.SecretRef != nil { + x.SecretRef = nil + } + } else { + if x.SecretRef == nil { + x.SecretRef = new(SecretReference) + } + x.SecretRef.CodecDecodeSelf(d) + } + yyj15++ + if yyhl15 { + yyb15 = yyj15 > l + } else { + yyb15 = r.CheckBreak() + } + if yyb15 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.ReadOnly = false + } else { + yyv25 := &x.ReadOnly + yym26 := z.DecBinary() + _ = yym26 + if false { + } else { + *((*bool)(yyv25)) = r.DecodeBool() + } + } + for { + yyj15++ + if yyhl15 { + yyb15 = yyj15 > l + } else { + yyb15 = r.CheckBreak() + } + if yyb15 { + break + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + z.DecStructFieldNotFound(yyj15-1, "") + } + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) +} + +func (x *FlockerVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + var yyq2 [2]bool + _, _, _ = yysep2, yyq2, yy2arr2 + const yyr2 bool = false + yyq2[0] = x.DatasetName != "" + yyq2[1] = x.DatasetUUID != "" + var yynn2 int + if yyr2 || yy2arr2 { + r.EncodeArrayStart(2) + } else { + yynn2 = 0 + for _, b := range yyq2 { + if b { + yynn2++ + } + } + r.EncodeMapStart(yynn2) + yynn2 = 0 + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[0] { + yym4 := z.EncBinary() + _ = yym4 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.DatasetName)) + } + } else { + r.EncodeString(codecSelferC_UTF81234, "") + } + } else { + if yyq2[0] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("datasetName")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym5 := z.EncBinary() + _ = yym5 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.DatasetName)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[1] { + yym7 := z.EncBinary() + _ = yym7 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.DatasetUUID)) + } + } else { + r.EncodeString(codecSelferC_UTF81234, "") + } + } else { + if yyq2[1] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("datasetUUID")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym8 := z.EncBinary() + _ = yym8 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.DatasetUUID)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + z.EncSendContainerState(codecSelfer_containerMapEnd1234) + } + } + } +} + +func (x *FlockerVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerMapEnd1234) + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(codecSelferOnlyMapOrArrayEncodeToStructErr1234) + } + } +} + +func (x *FlockerVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yys3Slc = z.DecScratchBuffer() // default slice to decode into + _ = yys3Slc + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + z.DecSendContainerState(codecSelfer_containerMapKey1234) + yys3Slc = r.DecodeBytes(yys3Slc, true, true) + yys3 := string(yys3Slc) + z.DecSendContainerState(codecSelfer_containerMapValue1234) + switch yys3 { + case "datasetName": + if r.TryDecodeAsNil() { + x.DatasetName = "" + } else { + yyv4 := &x.DatasetName + yym5 := z.DecBinary() + _ = yym5 + if false { + } else { + *((*string)(yyv4)) = r.DecodeString() + } + } + case "datasetUUID": + if r.TryDecodeAsNil() { + x.DatasetUUID = "" + } else { + yyv6 := &x.DatasetUUID + yym7 := z.DecBinary() + _ = yym7 + if false { + } else { + *((*string)(yyv6)) = r.DecodeString() + } + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + z.DecSendContainerState(codecSelfer_containerMapEnd1234) +} + +func (x *FlockerVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj8 int + var yyb8 bool + var yyhl8 bool = l >= 0 + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.DatasetName = "" + } else { + yyv9 := &x.DatasetName + yym10 := z.DecBinary() + _ = yym10 + if false { + } else { + *((*string)(yyv9)) = r.DecodeString() + } + } + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.DatasetUUID = "" + } else { + yyv11 := &x.DatasetUUID yym12 := z.DecBinary() _ = yym12 if false { @@ -16506,7 +17377,7 @@ func (x *ISCSIVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { } else { yysep2 := !z.EncBinary() yy2arr2 := z.EncBasicHandle().StructToArray - var yyq2 [10]bool + var yyq2 [11]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false yyq2[3] = x.ISCSIInterface != "" @@ -16516,9 +17387,10 @@ func (x *ISCSIVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { yyq2[7] = x.DiscoveryCHAPAuth != false yyq2[8] = x.SessionCHAPAuth != false yyq2[9] = x.SecretRef != nil + yyq2[10] = x.InitiatorName != nil var yynn2 int if yyr2 || yy2arr2 { - r.EncodeArrayStart(10) + r.EncodeArrayStart(11) } else { yynn2 = 3 for _, b := range yyq2 { @@ -16767,6 +17639,41 @@ func (x *ISCSIVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { } } } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[10] { + if x.InitiatorName == nil { + r.EncodeNil() + } else { + yy34 := *x.InitiatorName + yym35 := z.EncBinary() + _ = yym35 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(yy34)) + } + } + } else { + r.EncodeNil() + } + } else { + if yyq2[10] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("initiatorName")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.InitiatorName == nil { + r.EncodeNil() + } else { + yy36 := *x.InitiatorName + yym37 := z.EncBinary() + _ = yym37 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(yy36)) + } + } + } + } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayEnd1234) } else { @@ -16947,6 +17854,22 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) } x.SecretRef.CodecDecodeSelf(d) } + case "initiatorName": + if r.TryDecodeAsNil() { + if x.InitiatorName != nil { + x.InitiatorName = nil + } + } else { + if x.InitiatorName == nil { + x.InitiatorName = new(string) + } + yym24 := z.DecBinary() + _ = yym24 + if false { + } else { + *((*string)(x.InitiatorName)) = r.DecodeString() + } + } default: z.DecStructFieldNotFound(-1, yys3) } // end switch yys3 @@ -16958,16 +17881,16 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj23 int - var yyb23 bool - var yyhl23 bool = l >= 0 - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + var yyj25 int + var yyb25 bool + var yyhl25 bool = l >= 0 + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -16975,21 +17898,21 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if r.TryDecodeAsNil() { x.TargetPortal = "" } else { - yyv24 := &x.TargetPortal - yym25 := z.DecBinary() - _ = yym25 + yyv26 := &x.TargetPortal + yym27 := z.DecBinary() + _ = yym27 if false { } else { - *((*string)(yyv24)) = r.DecodeString() + *((*string)(yyv26)) = r.DecodeString() } } - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -16997,21 +17920,21 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if r.TryDecodeAsNil() { x.IQN = "" } else { - yyv26 := &x.IQN - yym27 := z.DecBinary() - _ = yym27 + yyv28 := &x.IQN + yym29 := z.DecBinary() + _ = yym29 if false { } else { - *((*string)(yyv26)) = r.DecodeString() + *((*string)(yyv28)) = r.DecodeString() } } - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -17019,21 +17942,21 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if r.TryDecodeAsNil() { x.Lun = 0 } else { - yyv28 := &x.Lun - yym29 := z.DecBinary() - _ = yym29 + yyv30 := &x.Lun + yym31 := z.DecBinary() + _ = yym31 if false { } else { - *((*int32)(yyv28)) = int32(r.DecodeInt(32)) + *((*int32)(yyv30)) = int32(r.DecodeInt(32)) } } - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -17041,21 +17964,21 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if r.TryDecodeAsNil() { x.ISCSIInterface = "" } else { - yyv30 := &x.ISCSIInterface - yym31 := z.DecBinary() - _ = yym31 + yyv32 := &x.ISCSIInterface + yym33 := z.DecBinary() + _ = yym33 if false { } else { - *((*string)(yyv30)) = r.DecodeString() + *((*string)(yyv32)) = r.DecodeString() } } - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -17063,21 +17986,21 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if r.TryDecodeAsNil() { x.FSType = "" } else { - yyv32 := &x.FSType - yym33 := z.DecBinary() - _ = yym33 + yyv34 := &x.FSType + yym35 := z.DecBinary() + _ = yym35 if false { } else { - *((*string)(yyv32)) = r.DecodeString() + *((*string)(yyv34)) = r.DecodeString() } } - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -17085,21 +18008,21 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if r.TryDecodeAsNil() { x.ReadOnly = false } else { - yyv34 := &x.ReadOnly - yym35 := z.DecBinary() - _ = yym35 + yyv36 := &x.ReadOnly + yym37 := z.DecBinary() + _ = yym37 if false { } else { - *((*bool)(yyv34)) = r.DecodeBool() + *((*bool)(yyv36)) = r.DecodeBool() } } - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -17107,21 +18030,21 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if r.TryDecodeAsNil() { x.Portals = nil } else { - yyv36 := &x.Portals - yym37 := z.DecBinary() - _ = yym37 + yyv38 := &x.Portals + yym39 := z.DecBinary() + _ = yym39 if false { } else { - z.F.DecSliceStringX(yyv36, false, d) + z.F.DecSliceStringX(yyv38, false, d) } } - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -17129,21 +18052,21 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if r.TryDecodeAsNil() { x.DiscoveryCHAPAuth = false } else { - yyv38 := &x.DiscoveryCHAPAuth - yym39 := z.DecBinary() - _ = yym39 + yyv40 := &x.DiscoveryCHAPAuth + yym41 := z.DecBinary() + _ = yym41 if false { } else { - *((*bool)(yyv38)) = r.DecodeBool() + *((*bool)(yyv40)) = r.DecodeBool() } } - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -17151,21 +18074,21 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if r.TryDecodeAsNil() { x.SessionCHAPAuth = false } else { - yyv40 := &x.SessionCHAPAuth - yym41 := z.DecBinary() - _ = yym41 + yyv42 := &x.SessionCHAPAuth + yym43 := z.DecBinary() + _ = yym43 if false { } else { - *((*bool)(yyv40)) = r.DecodeBool() + *((*bool)(yyv42)) = r.DecodeBool() } } - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -17180,18 +18103,44 @@ func (x *ISCSIVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder } x.SecretRef.CodecDecodeSelf(d) } + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l + } else { + yyb25 = r.CheckBreak() + } + if yyb25 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + if x.InitiatorName != nil { + x.InitiatorName = nil + } + } else { + if x.InitiatorName == nil { + x.InitiatorName = new(string) + } + yym46 := z.DecBinary() + _ = yym46 + if false { + } else { + *((*string)(x.InitiatorName)) = r.DecodeString() + } + } for { - yyj23++ - if yyhl23 { - yyb23 = yyj23 > l + yyj25++ + if yyhl25 { + yyb25 = yyj25 > l } else { - yyb23 = r.CheckBreak() + yyb25 = r.CheckBreak() } - if yyb23 { + if yyb25 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj23-1, "") + z.DecStructFieldNotFound(yyj25-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } @@ -17938,6 +18887,364 @@ func (x *AzureFileVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Dec z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } +func (x *AzureFilePersistentVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + var yyq2 [4]bool + _, _, _ = yysep2, yyq2, yy2arr2 + const yyr2 bool = false + yyq2[2] = x.ReadOnly != false + var yynn2 int + if yyr2 || yy2arr2 { + r.EncodeArrayStart(4) + } else { + yynn2 = 3 + for _, b := range yyq2 { + if b { + yynn2++ + } + } + r.EncodeMapStart(yynn2) + yynn2 = 0 + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + yym4 := z.EncBinary() + _ = yym4 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.SecretName)) + } + } else { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("secretName")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym5 := z.EncBinary() + _ = yym5 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.SecretName)) + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + yym7 := z.EncBinary() + _ = yym7 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.ShareName)) + } + } else { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("shareName")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym8 := z.EncBinary() + _ = yym8 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.ShareName)) + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[2] { + yym10 := z.EncBinary() + _ = yym10 + if false { + } else { + r.EncodeBool(bool(x.ReadOnly)) + } + } else { + r.EncodeBool(false) + } + } else { + if yyq2[2] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("readOnly")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym11 := z.EncBinary() + _ = yym11 + if false { + } else { + r.EncodeBool(bool(x.ReadOnly)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if x.SecretNamespace == nil { + r.EncodeNil() + } else { + yy13 := *x.SecretNamespace + yym14 := z.EncBinary() + _ = yym14 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(yy13)) + } + } + } else { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("secretNamespace")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.SecretNamespace == nil { + r.EncodeNil() + } else { + yy15 := *x.SecretNamespace + yym16 := z.EncBinary() + _ = yym16 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(yy15)) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + z.EncSendContainerState(codecSelfer_containerMapEnd1234) + } + } + } +} + +func (x *AzureFilePersistentVolumeSource) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerMapEnd1234) + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(codecSelferOnlyMapOrArrayEncodeToStructErr1234) + } + } +} + +func (x *AzureFilePersistentVolumeSource) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yys3Slc = z.DecScratchBuffer() // default slice to decode into + _ = yys3Slc + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + z.DecSendContainerState(codecSelfer_containerMapKey1234) + yys3Slc = r.DecodeBytes(yys3Slc, true, true) + yys3 := string(yys3Slc) + z.DecSendContainerState(codecSelfer_containerMapValue1234) + switch yys3 { + case "secretName": + if r.TryDecodeAsNil() { + x.SecretName = "" + } else { + yyv4 := &x.SecretName + yym5 := z.DecBinary() + _ = yym5 + if false { + } else { + *((*string)(yyv4)) = r.DecodeString() + } + } + case "shareName": + if r.TryDecodeAsNil() { + x.ShareName = "" + } else { + yyv6 := &x.ShareName + yym7 := z.DecBinary() + _ = yym7 + if false { + } else { + *((*string)(yyv6)) = r.DecodeString() + } + } + case "readOnly": + if r.TryDecodeAsNil() { + x.ReadOnly = false + } else { + yyv8 := &x.ReadOnly + yym9 := z.DecBinary() + _ = yym9 + if false { + } else { + *((*bool)(yyv8)) = r.DecodeBool() + } + } + case "secretNamespace": + if r.TryDecodeAsNil() { + if x.SecretNamespace != nil { + x.SecretNamespace = nil + } + } else { + if x.SecretNamespace == nil { + x.SecretNamespace = new(string) + } + yym11 := z.DecBinary() + _ = yym11 + if false { + } else { + *((*string)(x.SecretNamespace)) = r.DecodeString() + } + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + z.DecSendContainerState(codecSelfer_containerMapEnd1234) +} + +func (x *AzureFilePersistentVolumeSource) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj12 int + var yyb12 bool + var yyhl12 bool = l >= 0 + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.SecretName = "" + } else { + yyv13 := &x.SecretName + yym14 := z.DecBinary() + _ = yym14 + if false { + } else { + *((*string)(yyv13)) = r.DecodeString() + } + } + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.ShareName = "" + } else { + yyv15 := &x.ShareName + yym16 := z.DecBinary() + _ = yym16 + if false { + } else { + *((*string)(yyv15)) = r.DecodeString() + } + } + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.ReadOnly = false + } else { + yyv17 := &x.ReadOnly + yym18 := z.DecBinary() + _ = yym18 + if false { + } else { + *((*bool)(yyv17)) = r.DecodeBool() + } + } + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + if x.SecretNamespace != nil { + x.SecretNamespace = nil + } + } else { + if x.SecretNamespace == nil { + x.SecretNamespace = new(string) + } + yym20 := z.DecBinary() + _ = yym20 + if false { + } else { + *((*string)(x.SecretNamespace)) = r.DecodeString() + } + } + for { + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + break + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + z.DecStructFieldNotFound(yyj12-1, "") + } + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) +} + func (x *VsphereVirtualDiskVolumeSource) CodecEncodeSelf(e *codec1978.Encoder) { var h codecSelfer1234 z, r := codec1978.GenHelperEncoder(e) @@ -42940,53 +44247,518 @@ func (x *ReplicationControllerList) codecDecodeSelfFromMap(l int, d *codec1978.D yys3 := string(yys3Slc) z.DecSendContainerState(codecSelfer_containerMapValue1234) switch yys3 { - case "kind": - if r.TryDecodeAsNil() { - x.Kind = "" - } else { - yyv4 := &x.Kind - yym5 := z.DecBinary() - _ = yym5 - if false { - } else { - *((*string)(yyv4)) = r.DecodeString() - } - } - case "apiVersion": + case "kind": + if r.TryDecodeAsNil() { + x.Kind = "" + } else { + yyv4 := &x.Kind + yym5 := z.DecBinary() + _ = yym5 + if false { + } else { + *((*string)(yyv4)) = r.DecodeString() + } + } + case "apiVersion": + if r.TryDecodeAsNil() { + x.APIVersion = "" + } else { + yyv6 := &x.APIVersion + yym7 := z.DecBinary() + _ = yym7 + if false { + } else { + *((*string)(yyv6)) = r.DecodeString() + } + } + case "metadata": + if r.TryDecodeAsNil() { + x.ListMeta = pkg2_v1.ListMeta{} + } else { + yyv8 := &x.ListMeta + yym9 := z.DecBinary() + _ = yym9 + if false { + } else if z.HasExtensions() && z.DecExt(yyv8) { + } else { + z.DecFallback(yyv8, false) + } + } + case "items": + if r.TryDecodeAsNil() { + x.Items = nil + } else { + yyv10 := &x.Items + yym11 := z.DecBinary() + _ = yym11 + if false { + } else { + h.decSliceReplicationController((*[]ReplicationController)(yyv10), d) + } + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + z.DecSendContainerState(codecSelfer_containerMapEnd1234) +} + +func (x *ReplicationControllerList) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj12 int + var yyb12 bool + var yyhl12 bool = l >= 0 + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.Kind = "" + } else { + yyv13 := &x.Kind + yym14 := z.DecBinary() + _ = yym14 + if false { + } else { + *((*string)(yyv13)) = r.DecodeString() + } + } + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.APIVersion = "" + } else { + yyv15 := &x.APIVersion + yym16 := z.DecBinary() + _ = yym16 + if false { + } else { + *((*string)(yyv15)) = r.DecodeString() + } + } + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.ListMeta = pkg2_v1.ListMeta{} + } else { + yyv17 := &x.ListMeta + yym18 := z.DecBinary() + _ = yym18 + if false { + } else if z.HasExtensions() && z.DecExt(yyv17) { + } else { + z.DecFallback(yyv17, false) + } + } + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.Items = nil + } else { + yyv19 := &x.Items + yym20 := z.DecBinary() + _ = yym20 + if false { + } else { + h.decSliceReplicationController((*[]ReplicationController)(yyv19), d) + } + } + for { + yyj12++ + if yyhl12 { + yyb12 = yyj12 > l + } else { + yyb12 = r.CheckBreak() + } + if yyb12 { + break + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + z.DecStructFieldNotFound(yyj12-1, "") + } + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) +} + +func (x ServiceAffinity) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x)) + } +} + +func (x *ServiceAffinity) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + *((*string)(x)) = r.DecodeString() + } +} + +func (x *SessionAffinityConfig) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + var yyq2 [1]bool + _, _, _ = yysep2, yyq2, yy2arr2 + const yyr2 bool = false + yyq2[0] = x.ClientIP != nil + var yynn2 int + if yyr2 || yy2arr2 { + r.EncodeArrayStart(1) + } else { + yynn2 = 0 + for _, b := range yyq2 { + if b { + yynn2++ + } + } + r.EncodeMapStart(yynn2) + yynn2 = 0 + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[0] { + if x.ClientIP == nil { + r.EncodeNil() + } else { + x.ClientIP.CodecEncodeSelf(e) + } + } else { + r.EncodeNil() + } + } else { + if yyq2[0] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("clientIP")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.ClientIP == nil { + r.EncodeNil() + } else { + x.ClientIP.CodecEncodeSelf(e) + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + z.EncSendContainerState(codecSelfer_containerMapEnd1234) + } + } + } +} + +func (x *SessionAffinityConfig) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerMapEnd1234) + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(codecSelferOnlyMapOrArrayEncodeToStructErr1234) + } + } +} + +func (x *SessionAffinityConfig) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yys3Slc = z.DecScratchBuffer() // default slice to decode into + _ = yys3Slc + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + z.DecSendContainerState(codecSelfer_containerMapKey1234) + yys3Slc = r.DecodeBytes(yys3Slc, true, true) + yys3 := string(yys3Slc) + z.DecSendContainerState(codecSelfer_containerMapValue1234) + switch yys3 { + case "clientIP": + if r.TryDecodeAsNil() { + if x.ClientIP != nil { + x.ClientIP = nil + } + } else { + if x.ClientIP == nil { + x.ClientIP = new(ClientIPConfig) + } + x.ClientIP.CodecDecodeSelf(d) + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + z.DecSendContainerState(codecSelfer_containerMapEnd1234) +} + +func (x *SessionAffinityConfig) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj5 int + var yyb5 bool + var yyhl5 bool = l >= 0 + yyj5++ + if yyhl5 { + yyb5 = yyj5 > l + } else { + yyb5 = r.CheckBreak() + } + if yyb5 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + if x.ClientIP != nil { + x.ClientIP = nil + } + } else { + if x.ClientIP == nil { + x.ClientIP = new(ClientIPConfig) + } + x.ClientIP.CodecDecodeSelf(d) + } + for { + yyj5++ + if yyhl5 { + yyb5 = yyj5 > l + } else { + yyb5 = r.CheckBreak() + } + if yyb5 { + break + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + z.DecStructFieldNotFound(yyj5-1, "") + } + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) +} + +func (x *ClientIPConfig) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + var yyq2 [1]bool + _, _, _ = yysep2, yyq2, yy2arr2 + const yyr2 bool = false + yyq2[0] = x.TimeoutSeconds != nil + var yynn2 int + if yyr2 || yy2arr2 { + r.EncodeArrayStart(1) + } else { + yynn2 = 0 + for _, b := range yyq2 { + if b { + yynn2++ + } + } + r.EncodeMapStart(yynn2) + yynn2 = 0 + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[0] { + if x.TimeoutSeconds == nil { + r.EncodeNil() + } else { + yy4 := *x.TimeoutSeconds + yym5 := z.EncBinary() + _ = yym5 + if false { + } else { + r.EncodeInt(int64(yy4)) + } + } + } else { + r.EncodeNil() + } + } else { + if yyq2[0] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("timeoutSeconds")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.TimeoutSeconds == nil { + r.EncodeNil() + } else { + yy6 := *x.TimeoutSeconds + yym7 := z.EncBinary() + _ = yym7 + if false { + } else { + r.EncodeInt(int64(yy6)) + } + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + z.EncSendContainerState(codecSelfer_containerMapEnd1234) + } + } + } +} + +func (x *ClientIPConfig) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerMapEnd1234) + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(codecSelferOnlyMapOrArrayEncodeToStructErr1234) + } + } +} + +func (x *ClientIPConfig) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yys3Slc = z.DecScratchBuffer() // default slice to decode into + _ = yys3Slc + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + z.DecSendContainerState(codecSelfer_containerMapKey1234) + yys3Slc = r.DecodeBytes(yys3Slc, true, true) + yys3 := string(yys3Slc) + z.DecSendContainerState(codecSelfer_containerMapValue1234) + switch yys3 { + case "timeoutSeconds": if r.TryDecodeAsNil() { - x.APIVersion = "" - } else { - yyv6 := &x.APIVersion - yym7 := z.DecBinary() - _ = yym7 - if false { - } else { - *((*string)(yyv6)) = r.DecodeString() + if x.TimeoutSeconds != nil { + x.TimeoutSeconds = nil } - } - case "metadata": - if r.TryDecodeAsNil() { - x.ListMeta = pkg2_v1.ListMeta{} } else { - yyv8 := &x.ListMeta - yym9 := z.DecBinary() - _ = yym9 - if false { - } else if z.HasExtensions() && z.DecExt(yyv8) { - } else { - z.DecFallback(yyv8, false) + if x.TimeoutSeconds == nil { + x.TimeoutSeconds = new(int32) } - } - case "items": - if r.TryDecodeAsNil() { - x.Items = nil - } else { - yyv10 := &x.Items - yym11 := z.DecBinary() - _ = yym11 + yym5 := z.DecBinary() + _ = yym5 if false { } else { - h.decSliceReplicationController((*[]ReplicationController)(yyv10), d) + *((*int32)(x.TimeoutSeconds)) = int32(r.DecodeInt(32)) } } default: @@ -42996,144 +44768,55 @@ func (x *ReplicationControllerList) codecDecodeSelfFromMap(l int, d *codec1978.D z.DecSendContainerState(codecSelfer_containerMapEnd1234) } -func (x *ReplicationControllerList) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { +func (x *ClientIPConfig) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj12 int - var yyb12 bool - var yyhl12 bool = l >= 0 - yyj12++ - if yyhl12 { - yyb12 = yyj12 > l - } else { - yyb12 = r.CheckBreak() - } - if yyb12 { - z.DecSendContainerState(codecSelfer_containerArrayEnd1234) - return - } - z.DecSendContainerState(codecSelfer_containerArrayElem1234) - if r.TryDecodeAsNil() { - x.Kind = "" - } else { - yyv13 := &x.Kind - yym14 := z.DecBinary() - _ = yym14 - if false { - } else { - *((*string)(yyv13)) = r.DecodeString() - } - } - yyj12++ - if yyhl12 { - yyb12 = yyj12 > l + var yyj6 int + var yyb6 bool + var yyhl6 bool = l >= 0 + yyj6++ + if yyhl6 { + yyb6 = yyj6 > l } else { - yyb12 = r.CheckBreak() + yyb6 = r.CheckBreak() } - if yyb12 { + if yyb6 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } z.DecSendContainerState(codecSelfer_containerArrayElem1234) if r.TryDecodeAsNil() { - x.APIVersion = "" - } else { - yyv15 := &x.APIVersion - yym16 := z.DecBinary() - _ = yym16 - if false { - } else { - *((*string)(yyv15)) = r.DecodeString() + if x.TimeoutSeconds != nil { + x.TimeoutSeconds = nil } - } - yyj12++ - if yyhl12 { - yyb12 = yyj12 > l } else { - yyb12 = r.CheckBreak() - } - if yyb12 { - z.DecSendContainerState(codecSelfer_containerArrayEnd1234) - return - } - z.DecSendContainerState(codecSelfer_containerArrayElem1234) - if r.TryDecodeAsNil() { - x.ListMeta = pkg2_v1.ListMeta{} - } else { - yyv17 := &x.ListMeta - yym18 := z.DecBinary() - _ = yym18 - if false { - } else if z.HasExtensions() && z.DecExt(yyv17) { - } else { - z.DecFallback(yyv17, false) + if x.TimeoutSeconds == nil { + x.TimeoutSeconds = new(int32) } - } - yyj12++ - if yyhl12 { - yyb12 = yyj12 > l - } else { - yyb12 = r.CheckBreak() - } - if yyb12 { - z.DecSendContainerState(codecSelfer_containerArrayEnd1234) - return - } - z.DecSendContainerState(codecSelfer_containerArrayElem1234) - if r.TryDecodeAsNil() { - x.Items = nil - } else { - yyv19 := &x.Items - yym20 := z.DecBinary() - _ = yym20 + yym8 := z.DecBinary() + _ = yym8 if false { } else { - h.decSliceReplicationController((*[]ReplicationController)(yyv19), d) + *((*int32)(x.TimeoutSeconds)) = int32(r.DecodeInt(32)) } } for { - yyj12++ - if yyhl12 { - yyb12 = yyj12 > l + yyj6++ + if yyhl6 { + yyb6 = yyj6 > l } else { - yyb12 = r.CheckBreak() + yyb6 = r.CheckBreak() } - if yyb12 { + if yyb6 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj12-1, "") + z.DecStructFieldNotFound(yyj6-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } -func (x ServiceAffinity) CodecEncodeSelf(e *codec1978.Encoder) { - var h codecSelfer1234 - z, r := codec1978.GenHelperEncoder(e) - _, _, _ = h, z, r - yym1 := z.EncBinary() - _ = yym1 - if false { - } else if z.HasExtensions() && z.EncExt(x) { - } else { - r.EncodeString(codecSelferC_UTF81234, string(x)) - } -} - -func (x *ServiceAffinity) CodecDecodeSelf(d *codec1978.Decoder) { - var h codecSelfer1234 - z, r := codec1978.GenHelperDecoder(d) - _, _, _ = h, z, r - yym1 := z.DecBinary() - _ = yym1 - if false { - } else if z.HasExtensions() && z.DecExt(x) { - } else { - *((*string)(x)) = r.DecodeString() - } -} - func (x ServiceType) CodecEncodeSelf(e *codec1978.Encoder) { var h codecSelfer1234 z, r := codec1978.GenHelperEncoder(e) @@ -43793,7 +45476,7 @@ func (x *ServiceSpec) CodecEncodeSelf(e *codec1978.Encoder) { } else { yysep2 := !z.EncBinary() yy2arr2 := z.EncBasicHandle().StructToArray - var yyq2 [12]bool + var yyq2 [13]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false yyq2[0] = len(x.Ports) != 0 @@ -43808,9 +45491,10 @@ func (x *ServiceSpec) CodecEncodeSelf(e *codec1978.Encoder) { yyq2[9] = x.ExternalTrafficPolicy != "" yyq2[10] = x.HealthCheckNodePort != 0 yyq2[11] = x.PublishNotReadyAddresses != false + yyq2[12] = x.SessionAffinityConfig != nil var yynn2 int if yyr2 || yy2arr2 { - r.EncodeArrayStart(12) + r.EncodeArrayStart(13) } else { yynn2 = 0 for _, b := range yyq2 { @@ -44123,6 +45807,29 @@ func (x *ServiceSpec) CodecEncodeSelf(e *codec1978.Encoder) { } } } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[12] { + if x.SessionAffinityConfig == nil { + r.EncodeNil() + } else { + x.SessionAffinityConfig.CodecEncodeSelf(e) + } + } else { + r.EncodeNil() + } + } else { + if yyq2[12] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("sessionAffinityConfig")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.SessionAffinityConfig == nil { + r.EncodeNil() + } else { + x.SessionAffinityConfig.CodecEncodeSelf(e) + } + } + } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayEnd1234) } else { @@ -44313,6 +46020,17 @@ func (x *ServiceSpec) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { *((*bool)(yyv23)) = r.DecodeBool() } } + case "sessionAffinityConfig": + if r.TryDecodeAsNil() { + if x.SessionAffinityConfig != nil { + x.SessionAffinityConfig = nil + } + } else { + if x.SessionAffinityConfig == nil { + x.SessionAffinityConfig = new(SessionAffinityConfig) + } + x.SessionAffinityConfig.CodecDecodeSelf(d) + } default: z.DecStructFieldNotFound(-1, yys3) } // end switch yys3 @@ -44324,16 +46042,16 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj25 int - var yyb25 bool - var yyhl25 bool = l >= 0 - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + var yyj26 int + var yyb26 bool + var yyhl26 bool = l >= 0 + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44341,21 +46059,21 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.Ports = nil } else { - yyv26 := &x.Ports - yym27 := z.DecBinary() - _ = yym27 + yyv27 := &x.Ports + yym28 := z.DecBinary() + _ = yym28 if false { } else { - h.decSliceServicePort((*[]ServicePort)(yyv26), d) + h.decSliceServicePort((*[]ServicePort)(yyv27), d) } } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44363,21 +46081,21 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.Selector = nil } else { - yyv28 := &x.Selector - yym29 := z.DecBinary() - _ = yym29 + yyv29 := &x.Selector + yym30 := z.DecBinary() + _ = yym30 if false { } else { - z.F.DecMapStringStringX(yyv28, false, d) + z.F.DecMapStringStringX(yyv29, false, d) } } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44385,21 +46103,21 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.ClusterIP = "" } else { - yyv30 := &x.ClusterIP - yym31 := z.DecBinary() - _ = yym31 + yyv31 := &x.ClusterIP + yym32 := z.DecBinary() + _ = yym32 if false { } else { - *((*string)(yyv30)) = r.DecodeString() + *((*string)(yyv31)) = r.DecodeString() } } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44407,16 +46125,16 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.Type = "" } else { - yyv32 := &x.Type - yyv32.CodecDecodeSelf(d) + yyv33 := &x.Type + yyv33.CodecDecodeSelf(d) } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44424,21 +46142,21 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.ExternalIPs = nil } else { - yyv33 := &x.ExternalIPs - yym34 := z.DecBinary() - _ = yym34 + yyv34 := &x.ExternalIPs + yym35 := z.DecBinary() + _ = yym35 if false { } else { - z.F.DecSliceStringX(yyv33, false, d) + z.F.DecSliceStringX(yyv34, false, d) } } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44446,16 +46164,16 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.SessionAffinity = "" } else { - yyv35 := &x.SessionAffinity - yyv35.CodecDecodeSelf(d) + yyv36 := &x.SessionAffinity + yyv36.CodecDecodeSelf(d) } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44463,21 +46181,21 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.LoadBalancerIP = "" } else { - yyv36 := &x.LoadBalancerIP - yym37 := z.DecBinary() - _ = yym37 + yyv37 := &x.LoadBalancerIP + yym38 := z.DecBinary() + _ = yym38 if false { } else { - *((*string)(yyv36)) = r.DecodeString() + *((*string)(yyv37)) = r.DecodeString() } } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44485,21 +46203,21 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.LoadBalancerSourceRanges = nil } else { - yyv38 := &x.LoadBalancerSourceRanges - yym39 := z.DecBinary() - _ = yym39 + yyv39 := &x.LoadBalancerSourceRanges + yym40 := z.DecBinary() + _ = yym40 if false { } else { - z.F.DecSliceStringX(yyv38, false, d) + z.F.DecSliceStringX(yyv39, false, d) } } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44507,21 +46225,21 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.ExternalName = "" } else { - yyv40 := &x.ExternalName - yym41 := z.DecBinary() - _ = yym41 + yyv41 := &x.ExternalName + yym42 := z.DecBinary() + _ = yym42 if false { } else { - *((*string)(yyv40)) = r.DecodeString() + *((*string)(yyv41)) = r.DecodeString() } } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44529,16 +46247,16 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.ExternalTrafficPolicy = "" } else { - yyv42 := &x.ExternalTrafficPolicy - yyv42.CodecDecodeSelf(d) + yyv43 := &x.ExternalTrafficPolicy + yyv43.CodecDecodeSelf(d) } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44546,21 +46264,21 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.HealthCheckNodePort = 0 } else { - yyv43 := &x.HealthCheckNodePort - yym44 := z.DecBinary() - _ = yym44 + yyv44 := &x.HealthCheckNodePort + yym45 := z.DecBinary() + _ = yym45 if false { } else { - *((*int32)(yyv43)) = int32(r.DecodeInt(32)) + *((*int32)(yyv44)) = int32(r.DecodeInt(32)) } } - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -44568,26 +46286,47 @@ func (x *ServiceSpec) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { if r.TryDecodeAsNil() { x.PublishNotReadyAddresses = false } else { - yyv45 := &x.PublishNotReadyAddresses - yym46 := z.DecBinary() - _ = yym46 + yyv46 := &x.PublishNotReadyAddresses + yym47 := z.DecBinary() + _ = yym47 if false { } else { - *((*bool)(yyv45)) = r.DecodeBool() + *((*bool)(yyv46)) = r.DecodeBool() + } + } + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l + } else { + yyb26 = r.CheckBreak() + } + if yyb26 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + if x.SessionAffinityConfig != nil { + x.SessionAffinityConfig = nil } + } else { + if x.SessionAffinityConfig == nil { + x.SessionAffinityConfig = new(SessionAffinityConfig) + } + x.SessionAffinityConfig.CodecDecodeSelf(d) } for { - yyj25++ - if yyhl25 { - yyb25 = yyj25 > l + yyj26++ + if yyhl26 { + yyb26 = yyj26 > l } else { - yyb25 = r.CheckBreak() + yyb26 = r.CheckBreak() } - if yyb25 { + if yyb26 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj25-1, "") + z.DecStructFieldNotFound(yyj26-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } @@ -70186,7 +71925,7 @@ func (x codecSelfer1234) decSlicePersistentVolume(v *[]PersistentVolume, d *code yyrg1 := len(yyv1) > 0 yyv21 := yyv1 - yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 552) + yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 576) if yyrt1 { if yyrl1 <= cap(yyv1) { yyv1 = yyv1[:yyrl1] @@ -73512,7 +75251,7 @@ func (x codecSelfer1234) decSliceService(v *[]Service, d *codec1978.Decoder) { yyrg1 := len(yyv1) > 0 yyv21 := yyv1 - yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 472) + yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 480) if yyrt1 { if yyrl1 <= cap(yyv1) { yyv1 = yyv1[:yyrl1] diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 2444bf0a46da1..f0fa7e6e5762a 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -409,7 +409,7 @@ type PersistentVolumeSource struct { Cinder *CinderVolumeSource `json:"cinder,omitempty" protobuf:"bytes,8,opt,name=cinder"` // CephFS represents a Ceph FS mount on the host that shares a pod's lifetime // +optional - CephFS *CephFSVolumeSource `json:"cephfs,omitempty" protobuf:"bytes,9,opt,name=cephfs"` + CephFS *CephFSPersistentVolumeSource `json:"cephfs,omitempty" protobuf:"bytes,9,opt,name=cephfs"` // FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. // +optional FC *FCVolumeSource `json:"fc,omitempty" protobuf:"bytes,10,opt,name=fc"` @@ -423,7 +423,7 @@ type PersistentVolumeSource struct { FlexVolume *FlexVolumeSource `json:"flexVolume,omitempty" protobuf:"bytes,12,opt,name=flexVolume"` // AzureFile represents an Azure File Service mount on the host and bind mount to the pod. // +optional - AzureFile *AzureFileVolumeSource `json:"azureFile,omitempty" protobuf:"bytes,13,opt,name=azureFile"` + AzureFile *AzureFilePersistentVolumeSource `json:"azureFile,omitempty" protobuf:"bytes,13,opt,name=azureFile"` // VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine // +optional VsphereVolume *VsphereVirtualDiskVolumeSource `json:"vsphereVolume,omitempty" protobuf:"bytes,14,opt,name=vsphereVolume"` @@ -519,6 +519,11 @@ type PersistentVolumeSpec struct { // means that this volume does not belong to any StorageClass. // +optional StorageClassName string `json:"storageClassName,omitempty" protobuf:"bytes,6,opt,name=storageClassName"` + // A list of mount options, e.g. ["ro", "soft"]. Not validated - mount will + // simply fail if one is invalid. + // More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options + // +optional + MountOptions []string `json:"mountOptions,omitempty" protobuf:"bytes,7,opt,name=mountOptions"` } // PersistentVolumeReclaimPolicy describes a policy for end-of-life maintenance of persistent volumes. @@ -681,12 +686,41 @@ const ( ClaimLost PersistentVolumeClaimPhase = "Lost" ) +type HostPathType string + +const ( + // For backwards compatible, leave it empty if unset + HostPathUnset HostPathType = "" + // If nothing exists at the given path, an empty directory will be created there + // as needed with file mode 0755, having the same group and ownership with Kubelet. + HostPathDirectoryOrCreate HostPathType = "DirectoryOrCreate" + // A directory must exist at the given path + HostPathDirectory HostPathType = "Directory" + // If nothing exists at the given path, an empty file will be created there + // as needed with file mode 0644, having the same group and ownership with Kubelet. + HostPathFileOrCreate HostPathType = "FileOrCreate" + // A file must exist at the given path + HostPathFile HostPathType = "File" + // A UNIX socket must exist at the given path + HostPathSocket HostPathType = "Socket" + // A character device must exist at the given path + HostPathCharDev HostPathType = "CharDevice" + // A block device must exist at the given path + HostPathBlockDev HostPathType = "BlockDevice" +) + // Represents a host path mapped into a pod. // Host path volumes do not support ownership management or SELinux relabeling. type HostPathVolumeSource struct { // Path of the directory on the host. + // If the path is a symlink, it will follow the link to the real path. // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath Path string `json:"path" protobuf:"bytes,1,opt,name=path"` + // Type for HostPath Volume + // Defaults to "" + // More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + // +optional + Type *HostPathType `json:"type,omitempty" protobuf:"bytes,2,opt,name=type"` } // Represents an empty directory for a pod. @@ -819,6 +853,45 @@ type CephFSVolumeSource struct { ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,6,opt,name=readOnly"` } +// SecretReference represents a Secret Reference. It has enough information to retrieve secret +// in any namespace +type SecretReference struct { + // Name is unique within a namespace to reference a secret resource. + // +optional + Name string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"` + // Namespace defines the space within which the secret name must be unique. + // +optional + Namespace string `json:"namespace,omitempty" protobuf:"bytes,2,opt,name=namespace"` +} + +// Represents a Ceph Filesystem mount that lasts the lifetime of a pod +// Cephfs volumes do not support ownership management or SELinux relabeling. +type CephFSPersistentVolumeSource struct { + // Required: Monitors is a collection of Ceph monitors + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + Monitors []string `json:"monitors" protobuf:"bytes,1,rep,name=monitors"` + // Optional: Used as the mounted root, rather than the full Ceph tree, default is / + // +optional + Path string `json:"path,omitempty" protobuf:"bytes,2,opt,name=path"` + // Optional: User is the rados user name, default is admin + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + // +optional + User string `json:"user,omitempty" protobuf:"bytes,3,opt,name=user"` + // Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + // +optional + SecretFile string `json:"secretFile,omitempty" protobuf:"bytes,4,opt,name=secretFile"` + // Optional: SecretRef is reference to the authentication secret for User, default is empty. + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + // +optional + SecretRef *SecretReference `json:"secretRef,omitempty" protobuf:"bytes,5,opt,name=secretRef"` + // Optional: Defaults to false (read/write). ReadOnly here will force + // the ReadOnly setting in VolumeMounts. + // More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it + // +optional + ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,6,opt,name=readOnly"` +} + // Represents a Flocker volume mounted by the Flocker agent. // One and only one of datasetName and datasetUUID should be set. // Flocker volumes do not support ownership management or SELinux relabeling. @@ -1095,6 +1168,11 @@ type ISCSIVolumeSource struct { // CHAP secret for iSCSI target and initiator authentication // +optional SecretRef *LocalObjectReference `json:"secretRef,omitempty" protobuf:"bytes,10,opt,name=secretRef"` + // Custom iSCSI initiator name. + // If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + // : will be created for the connection. + // +optional + InitiatorName *string `json:"initiatorName,omitempty" protobuf:"bytes,12,opt,name=initiatorName"` } // Represents a Fibre Channel volume. @@ -1135,6 +1213,22 @@ type AzureFileVolumeSource struct { ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,3,opt,name=readOnly"` } +// AzureFile represents an Azure File Service mount on the host and bind mount to the pod. +type AzureFilePersistentVolumeSource struct { + // the name of secret that contains Azure Storage Account Name and Key + SecretName string `json:"secretName" protobuf:"bytes,1,opt,name=secretName"` + // Share Name + ShareName string `json:"shareName" protobuf:"bytes,2,opt,name=shareName"` + // Defaults to false (read/write). ReadOnly here will force + // the ReadOnly setting in VolumeMounts. + // +optional + ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,3,opt,name=readOnly"` + // the namespace of the secret that contains Azure Storage Account Name and Key + // default is the same as the Pod + // +optional + SecretNamespace *string `json:"secretNamespace" protobuf:"bytes,4,opt,name=secretNamespace"` +} + // Represents a vSphere volume resource. type VsphereVirtualDiskVolumeSource struct { // Path that identifies vSphere volume vmdk @@ -1484,7 +1578,7 @@ type EnvVarSource struct { // +optional FieldRef *ObjectFieldSelector `json:"fieldRef,omitempty" protobuf:"bytes,1,opt,name=fieldRef"` // Selects a resource of the container: only resources limits and requests - // (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + // (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. // +optional ResourceFieldRef *ResourceFieldSelector `json:"resourceFieldRef,omitempty" protobuf:"bytes,2,opt,name=resourceFieldRef"` // Selects a key of a ConfigMap. @@ -2114,9 +2208,7 @@ type NodeSelectorTerm struct { // that relates the key and values. type NodeSelectorRequirement struct { // The label key that the selector applies to. - // +patchMergeKey=key - // +patchStrategy=merge - Key string `json:"key" patchStrategy:"merge" patchMergeKey:"key" protobuf:"bytes,1,opt,name=key"` + Key string `json:"key" protobuf:"bytes,1,opt,name=key"` // Represents a key's relationship to a set of values. // Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. Operator NodeSelectorOperator `json:"operator" protobuf:"bytes,2,opt,name=operator,casttype=NodeSelectorOperator"` @@ -2302,9 +2394,7 @@ type PreferredSchedulingTerm struct { // any pod that does not tolerate the Taint. type Taint struct { // Required. The taint key to be applied to a node. - // +patchMergeKey=key - // +patchStrategy=merge - Key string `json:"key" patchStrategy:"merge" patchMergeKey:"key" protobuf:"bytes,1,opt,name=key"` + Key string `json:"key" protobuf:"bytes,1,opt,name=key"` // Required. The taint value corresponding to the taint key. // +optional Value string `json:"value,omitempty" protobuf:"bytes,2,opt,name=value"` @@ -2346,9 +2436,7 @@ type Toleration struct { // Key is the taint key that the toleration applies to. Empty means match all taint keys. // If the key is empty, operator must be Exists; this combination means to match all values and all keys. // +optional - // +patchMergeKey=key - // +patchStrategy=merge - Key string `json:"key,omitempty" patchStrategy:"merge" patchMergeKey:"key" protobuf:"bytes,1,opt,name=key"` + Key string `json:"key,omitempty" protobuf:"bytes,1,opt,name=key"` // Operator represents a key's relationship to the value. // Valid operators are Exists and Equal. Defaults to Equal. // Exists is equivalent to wildcard for value, so that a pod can @@ -2406,8 +2494,8 @@ type PodSpec struct { // More info: https://kubernetes.io/docs/concepts/storage/volumes // +optional // +patchMergeKey=name - // +patchStrategy=merge - Volumes []Volume `json:"volumes,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,1,rep,name=volumes"` + // +patchStrategy=merge,retainKeys + Volumes []Volume `json:"volumes,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"name" protobuf:"bytes,1,rep,name=volumes"` // List of initialization containers belonging to the pod. // Init containers are executed in order prior to containers being started. If any // init container fails, the pod is considered to have failed and is handled according @@ -2915,6 +3003,22 @@ const ( ServiceAffinityNone ServiceAffinity = "None" ) +// SessionAffinityConfig represents the configurations of session affinity. +type SessionAffinityConfig struct { + // clientIP contains the configurations of Client IP based session affinity. + // +optional + ClientIP *ClientIPConfig `json:"clientIP,omitempty" protobuf:"bytes,1,opt,name=clientIP"` +} + +// ClientIPConfig represents the configurations of Client IP based session affinity. +type ClientIPConfig struct { + // timeoutSeconds specifies the seconds of ClientIP type session sticky time. + // The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + // Default value is 10800(for 3 hours). + // +optional + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty" protobuf:"varint,1,opt,name=timeoutSeconds"` +} + // Service Type string describes ingress methods for a service type ServiceType string @@ -3089,6 +3193,9 @@ type ServiceSpec struct { // field. // +optional PublishNotReadyAddresses bool `json:"publishNotReadyAddresses,omitempty" protobuf:"varint,13,opt,name=publishNotReadyAddresses"` + // sessionAffinityConfig contains the configurations of session affinity. + // +optional + SessionAffinityConfig *SessionAffinityConfig `json:"sessionAffinityConfig,omitempty" protobuf:"bytes,14,opt,name=sessionAffinityConfig"` } // ServicePort contains information on service's port. @@ -3597,15 +3704,11 @@ const ( ResourceMemory ResourceName = "memory" // Volume size, in bytes (e,g. 5Gi = 5GiB = 5 * 1024 * 1024 * 1024) ResourceStorage ResourceName = "storage" - // Local Storage for container overlay filesystem, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) - // The resource name for ResourceStorageOverlay is alpha and it can change across releases. - ResourceStorageOverlay ResourceName = "storage.kubernetes.io/overlay" - // Local Storage for scratch space, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) - // The resource name for ResourceStorageScratch is alpha and it can change across releases. - ResourceStorageScratch ResourceName = "storage.kubernetes.io/scratch" + // Local ephemeral storage, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) + // The resource name for ResourceEphemeralStorage is alpha and it can change across releases. + ResourceEphemeralStorage ResourceName = "ephemeral-storage" // NVIDIA GPU, in devices. Alpha, might change: although fractional and allowing values >1, only one whole device per node is assigned. ResourceNvidiaGPU ResourceName = "alpha.kubernetes.io/nvidia-gpu" - // Number of Pods that may be running on this Node: see ResourcePods ) const ( @@ -4251,10 +4354,14 @@ const ( ResourceRequestsMemory ResourceName = "requests.memory" // Storage request, in bytes ResourceRequestsStorage ResourceName = "requests.storage" + // Local ephemeral storage request, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) + ResourceRequestsEphemeralStorage ResourceName = "requests.ephemeral-storage" // CPU limit, in cores. (500m = .5 cores) ResourceLimitsCPU ResourceName = "limits.cpu" // Memory limit, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) ResourceLimitsMemory ResourceName = "limits.memory" + // Local ephemeral storage limit, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024) + ResourceLimitsEphemeralStorage ResourceName = "limits.ephemeral-storage" ) // A ResourceQuotaScope defines a filter that must match each object tracked by a quota diff --git a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go index 7d6e4ac4e4056..64acca0346202 100644 --- a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -83,6 +83,18 @@ func (AzureDiskVolumeSource) SwaggerDoc() map[string]string { return map_AzureDiskVolumeSource } +var map_AzureFilePersistentVolumeSource = map[string]string{ + "": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", + "secretName": "the name of secret that contains Azure Storage Account Name and Key", + "shareName": "Share Name", + "readOnly": "Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "secretNamespace": "the namespace of the secret that contains Azure Storage Account Name and Key default is the same as the Pod", +} + +func (AzureFilePersistentVolumeSource) SwaggerDoc() map[string]string { + return map_AzureFilePersistentVolumeSource +} + var map_AzureFileVolumeSource = map[string]string{ "": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", "secretName": "the name of secret that contains Azure Storage Account Name and Key", @@ -114,6 +126,20 @@ func (Capabilities) SwaggerDoc() map[string]string { return map_Capabilities } +var map_CephFSPersistentVolumeSource = map[string]string{ + "": "Represents a Ceph Filesystem mount that lasts the lifetime of a pod Cephfs volumes do not support ownership management or SELinux relabeling.", + "monitors": "Required: Monitors is a collection of Ceph monitors More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", + "path": "Optional: Used as the mounted root, rather than the full Ceph tree, default is /", + "user": "Optional: User is the rados user name, default is admin More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", + "secretFile": "Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", + "secretRef": "Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", + "readOnly": "Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", +} + +func (CephFSPersistentVolumeSource) SwaggerDoc() map[string]string { + return map_CephFSPersistentVolumeSource +} + var map_CephFSVolumeSource = map[string]string{ "": "Represents a Ceph Filesystem mount that lasts the lifetime of a pod Cephfs volumes do not support ownership management or SELinux relabeling.", "monitors": "Required: Monitors is a collection of Ceph monitors More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it", @@ -139,6 +165,15 @@ func (CinderVolumeSource) SwaggerDoc() map[string]string { return map_CinderVolumeSource } +var map_ClientIPConfig = map[string]string{ + "": "ClientIPConfig represents the configurations of Client IP based session affinity.", + "timeoutSeconds": "timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == \"ClientIP\". Default value is 10800(for 3 hours).", +} + +func (ClientIPConfig) SwaggerDoc() map[string]string { + return map_ClientIPConfig +} + var map_ComponentCondition = map[string]string{ "": "Information about the condition of a component.", "type": "Type of condition for a component. Valid value: \"Healthy\"", @@ -484,7 +519,7 @@ func (EnvVar) SwaggerDoc() map[string]string { var map_EnvVarSource = map[string]string{ "": "EnvVarSource represents a source for the value of an EnvVar.", "fieldRef": "Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP.", - "resourceFieldRef": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.", + "resourceFieldRef": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.", "configMapKeyRef": "Selects a key of a ConfigMap.", "secretKeyRef": "Selects a key of a secret in the pod's namespace", } @@ -655,7 +690,8 @@ func (HostAlias) SwaggerDoc() map[string]string { var map_HostPathVolumeSource = map[string]string{ "": "Represents a host path mapped into a pod. Host path volumes do not support ownership management or SELinux relabeling.", - "path": "Path of the directory on the host. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "path": "Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "type": "Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", } func (HostPathVolumeSource) SwaggerDoc() map[string]string { @@ -674,6 +710,7 @@ var map_ISCSIVolumeSource = map[string]string{ "chapAuthDiscovery": "whether support iSCSI Discovery CHAP authentication", "chapAuthSession": "whether support iSCSI Session CHAP authentication", "secretRef": "CHAP secret for iSCSI target and initiator authentication", + "initiatorName": "Custom iSCSI initiator name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface : will be created for the connection.", } func (ISCSIVolumeSource) SwaggerDoc() map[string]string { @@ -1175,6 +1212,7 @@ var map_PersistentVolumeSpec = map[string]string{ "claimRef": "ClaimRef is part of a bi-directional binding between PersistentVolume and PersistentVolumeClaim. Expected to be non-nil when bound. claim.VolumeName is the authoritative bind between PV and PVC. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#binding", "persistentVolumeReclaimPolicy": "What happens to a persistent volume when released from its claim. Valid options are Retain (default) and Recycle. Recycling must be supported by the volume plugin underlying this persistent volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#reclaiming", "storageClassName": "Name of StorageClass to which this persistent volume belongs. Empty value means that this volume does not belong to any StorageClass.", + "mountOptions": "A list of mount options, e.g. [\"ro\", \"soft\"]. Not validated - mount will simply fail if one is invalid. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options", } func (PersistentVolumeSpec) SwaggerDoc() map[string]string { @@ -1750,6 +1788,16 @@ func (SecretProjection) SwaggerDoc() map[string]string { return map_SecretProjection } +var map_SecretReference = map[string]string{ + "": "SecretReference represents a Secret Reference. It has enough information to retrieve secret in any namespace", + "name": "Name is unique within a namespace to reference a secret resource.", + "namespace": "Namespace defines the space within which the secret name must be unique.", +} + +func (SecretReference) SwaggerDoc() map[string]string { + return map_SecretReference +} + var map_SecretVolumeSource = map[string]string{ "": "Adapts a Secret into a volume.\n\nThe contents of the target Secret's Data field will be presented in a volume as files using the keys in the Data field as the file names. Secret volumes support ownership management and SELinux relabeling.", "secretName": "Name of the secret in the pod's namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret", @@ -1865,6 +1913,7 @@ var map_ServiceSpec = map[string]string{ "externalTrafficPolicy": "externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints. \"Local\" preserves the client source IP and avoids a second hop for LoadBalancer and Nodeport type services, but risks potentially imbalanced traffic spreading. \"Cluster\" obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.", "healthCheckNodePort": "healthCheckNodePort specifies the healthcheck nodePort for the service. If not specified, HealthCheckNodePort is created by the service api backend with the allocated nodePort. Will use user-specified nodePort value if specified by the client. Only effects when Type is set to LoadBalancer and ExternalTrafficPolicy is set to Local.", "publishNotReadyAddresses": "publishNotReadyAddresses, when set to true, indicates that DNS implementations must publish the notReadyAddresses of subsets for the Endpoints associated with the Service. The default value is false. The primary use case for setting this field is to use a StatefulSet's Headless Service to propagate SRV records for its Pods without respect to their readiness for purpose of peer discovery. This field will replace the service.alpha.kubernetes.io/tolerate-unready-endpoints when that annotation is deprecated and all clients have been converted to use this field.", + "sessionAffinityConfig": "sessionAffinityConfig contains the configurations of session affinity.", } func (ServiceSpec) SwaggerDoc() map[string]string { @@ -1880,6 +1929,15 @@ func (ServiceStatus) SwaggerDoc() map[string]string { return map_ServiceStatus } +var map_SessionAffinityConfig = map[string]string{ + "": "SessionAffinityConfig represents the configurations of session affinity.", + "clientIP": "clientIP contains the configurations of Client IP based session affinity.", +} + +func (SessionAffinityConfig) SwaggerDoc() map[string]string { + return map_SessionAffinityConfig +} + var map_StorageOSPersistentVolumeSource = map[string]string{ "": "Represents a StorageOS persistent volume resource.", "volumeName": "VolumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace.", diff --git a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go index ea1542df8d68a..4c29bf126e665 100644 --- a/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go @@ -58,6 +58,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*AzureDiskVolumeSource).DeepCopyInto(out.(*AzureDiskVolumeSource)) return nil }, InType: reflect.TypeOf(&AzureDiskVolumeSource{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*AzureFilePersistentVolumeSource).DeepCopyInto(out.(*AzureFilePersistentVolumeSource)) + return nil + }, InType: reflect.TypeOf(&AzureFilePersistentVolumeSource{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*AzureFileVolumeSource).DeepCopyInto(out.(*AzureFileVolumeSource)) return nil @@ -70,6 +74,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*Capabilities).DeepCopyInto(out.(*Capabilities)) return nil }, InType: reflect.TypeOf(&Capabilities{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*CephFSPersistentVolumeSource).DeepCopyInto(out.(*CephFSPersistentVolumeSource)) + return nil + }, InType: reflect.TypeOf(&CephFSPersistentVolumeSource{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*CephFSVolumeSource).DeepCopyInto(out.(*CephFSVolumeSource)) return nil @@ -78,6 +86,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*CinderVolumeSource).DeepCopyInto(out.(*CinderVolumeSource)) return nil }, InType: reflect.TypeOf(&CinderVolumeSource{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*ClientIPConfig).DeepCopyInto(out.(*ClientIPConfig)) + return nil + }, InType: reflect.TypeOf(&ClientIPConfig{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*ComponentCondition).DeepCopyInto(out.(*ComponentCondition)) return nil @@ -634,6 +646,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*SecretProjection).DeepCopyInto(out.(*SecretProjection)) return nil }, InType: reflect.TypeOf(&SecretProjection{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*SecretReference).DeepCopyInto(out.(*SecretReference)) + return nil + }, InType: reflect.TypeOf(&SecretReference{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*SecretVolumeSource).DeepCopyInto(out.(*SecretVolumeSource)) return nil @@ -678,6 +694,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*ServiceStatus).DeepCopyInto(out.(*ServiceStatus)) return nil }, InType: reflect.TypeOf(&ServiceStatus{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*SessionAffinityConfig).DeepCopyInto(out.(*SessionAffinityConfig)) + return nil + }, InType: reflect.TypeOf(&SessionAffinityConfig{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*StorageOSPersistentVolumeSource).DeepCopyInto(out.(*StorageOSPersistentVolumeSource)) return nil @@ -879,6 +899,31 @@ func (in *AzureDiskVolumeSource) DeepCopy() *AzureDiskVolumeSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureFilePersistentVolumeSource) DeepCopyInto(out *AzureFilePersistentVolumeSource) { + *out = *in + if in.SecretNamespace != nil { + in, out := &in.SecretNamespace, &out.SecretNamespace + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureFilePersistentVolumeSource. +func (in *AzureFilePersistentVolumeSource) DeepCopy() *AzureFilePersistentVolumeSource { + if in == nil { + return nil + } + out := new(AzureFilePersistentVolumeSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureFileVolumeSource) DeepCopyInto(out *AzureFileVolumeSource) { *out = *in @@ -949,6 +994,36 @@ func (in *Capabilities) DeepCopy() *Capabilities { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CephFSPersistentVolumeSource) DeepCopyInto(out *CephFSPersistentVolumeSource) { + *out = *in + if in.Monitors != nil { + in, out := &in.Monitors, &out.Monitors + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + if *in == nil { + *out = nil + } else { + *out = new(SecretReference) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephFSPersistentVolumeSource. +func (in *CephFSPersistentVolumeSource) DeepCopy() *CephFSPersistentVolumeSource { + if in == nil { + return nil + } + out := new(CephFSPersistentVolumeSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CephFSVolumeSource) DeepCopyInto(out *CephFSVolumeSource) { *out = *in @@ -995,6 +1070,31 @@ func (in *CinderVolumeSource) DeepCopy() *CinderVolumeSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientIPConfig) DeepCopyInto(out *ClientIPConfig) { + *out = *in + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + if *in == nil { + *out = nil + } else { + *out = new(int32) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientIPConfig. +func (in *ClientIPConfig) DeepCopy() *ClientIPConfig { + if in == nil { + return nil + } + out := new(ClientIPConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ComponentCondition) DeepCopyInto(out *ComponentCondition) { *out = *in @@ -2303,6 +2403,15 @@ func (in *HostAlias) DeepCopy() *HostAlias { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HostPathVolumeSource) DeepCopyInto(out *HostPathVolumeSource) { *out = *in + if in.Type != nil { + in, out := &in.Type, &out.Type + if *in == nil { + *out = nil + } else { + *out = new(HostPathType) + **out = **in + } + } return } @@ -2333,6 +2442,15 @@ func (in *ISCSIVolumeSource) DeepCopyInto(out *ISCSIVolumeSource) { **out = **in } } + if in.InitiatorName != nil { + in, out := &in.InitiatorName, &out.InitiatorName + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } return } @@ -3549,7 +3667,7 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) { *out = nil } else { *out = new(HostPathVolumeSource) - **out = **in + (*in).DeepCopyInto(*out) } } if in.Glusterfs != nil { @@ -3602,7 +3720,7 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) { if *in == nil { *out = nil } else { - *out = new(CephFSVolumeSource) + *out = new(CephFSPersistentVolumeSource) (*in).DeepCopyInto(*out) } } @@ -3638,8 +3756,8 @@ func (in *PersistentVolumeSource) DeepCopyInto(out *PersistentVolumeSource) { if *in == nil { *out = nil } else { - *out = new(AzureFileVolumeSource) - **out = **in + *out = new(AzureFilePersistentVolumeSource) + (*in).DeepCopyInto(*out) } } if in.VsphereVolume != nil { @@ -3752,6 +3870,11 @@ func (in *PersistentVolumeSpec) DeepCopyInto(out *PersistentVolumeSpec) { **out = **in } } + if in.MountOptions != nil { + in, out := &in.MountOptions, &out.MountOptions + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -5218,6 +5341,22 @@ func (in *SecretProjection) DeepCopy() *SecretProjection { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretReference) DeepCopyInto(out *SecretReference) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference. +func (in *SecretReference) DeepCopy() *SecretReference { + if in == nil { + return nil + } + out := new(SecretReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretVolumeSource) DeepCopyInto(out *SecretVolumeSource) { *out = *in @@ -5576,6 +5715,15 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.SessionAffinityConfig != nil { + in, out := &in.SessionAffinityConfig, &out.SessionAffinityConfig + if *in == nil { + *out = nil + } else { + *out = new(SessionAffinityConfig) + (*in).DeepCopyInto(*out) + } + } return } @@ -5606,6 +5754,31 @@ func (in *ServiceStatus) DeepCopy() *ServiceStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SessionAffinityConfig) DeepCopyInto(out *SessionAffinityConfig) { + *out = *in + if in.ClientIP != nil { + in, out := &in.ClientIP, &out.ClientIP + if *in == nil { + *out = nil + } else { + *out = new(ClientIPConfig) + (*in).DeepCopyInto(*out) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SessionAffinityConfig. +func (in *SessionAffinityConfig) DeepCopy() *SessionAffinityConfig { + if in == nil { + return nil + } + out := new(SessionAffinityConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageOSPersistentVolumeSource) DeepCopyInto(out *StorageOSPersistentVolumeSource) { *out = *in @@ -5816,7 +5989,7 @@ func (in *VolumeSource) DeepCopyInto(out *VolumeSource) { *out = nil } else { *out = new(HostPathVolumeSource) - **out = **in + (*in).DeepCopyInto(*out) } } if in.EmptyDir != nil { diff --git a/staging/src/k8s.io/api/extensions/v1beta1/generated.pb.go b/staging/src/k8s.io/api/extensions/v1beta1/generated.pb.go index 52d8268e53f97..f7eb3b4d65af9 100644 --- a/staging/src/k8s.io/api/extensions/v1beta1/generated.pb.go +++ b/staging/src/k8s.io/api/extensions/v1beta1/generated.pb.go @@ -47,6 +47,7 @@ limitations under the License. HTTPIngressRuleValue HostPortRange IDRange + IPBlock Ingress IngressBackend IngressList @@ -206,161 +207,165 @@ func (m *IDRange) Reset() { *m = IDRange{} } func (*IDRange) ProtoMessage() {} func (*IDRange) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{21} } +func (m *IPBlock) Reset() { *m = IPBlock{} } +func (*IPBlock) ProtoMessage() {} +func (*IPBlock) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} } + func (m *Ingress) Reset() { *m = Ingress{} } func (*Ingress) ProtoMessage() {} -func (*Ingress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{22} } +func (*Ingress) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} } func (m *IngressBackend) Reset() { *m = IngressBackend{} } func (*IngressBackend) ProtoMessage() {} -func (*IngressBackend) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{23} } +func (*IngressBackend) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{24} } func (m *IngressList) Reset() { *m = IngressList{} } func (*IngressList) ProtoMessage() {} -func (*IngressList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{24} } +func (*IngressList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} } func (m *IngressRule) Reset() { *m = IngressRule{} } func (*IngressRule) ProtoMessage() {} -func (*IngressRule) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{25} } +func (*IngressRule) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} } func (m *IngressRuleValue) Reset() { *m = IngressRuleValue{} } func (*IngressRuleValue) ProtoMessage() {} -func (*IngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{26} } +func (*IngressRuleValue) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{27} } func (m *IngressSpec) Reset() { *m = IngressSpec{} } func (*IngressSpec) ProtoMessage() {} -func (*IngressSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{27} } +func (*IngressSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{28} } func (m *IngressStatus) Reset() { *m = IngressStatus{} } func (*IngressStatus) ProtoMessage() {} -func (*IngressStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{28} } +func (*IngressStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{29} } func (m *IngressTLS) Reset() { *m = IngressTLS{} } func (*IngressTLS) ProtoMessage() {} -func (*IngressTLS) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{29} } +func (*IngressTLS) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} } func (m *NetworkPolicy) Reset() { *m = NetworkPolicy{} } func (*NetworkPolicy) ProtoMessage() {} -func (*NetworkPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{30} } +func (*NetworkPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{31} } func (m *NetworkPolicyIngressRule) Reset() { *m = NetworkPolicyIngressRule{} } func (*NetworkPolicyIngressRule) ProtoMessage() {} func (*NetworkPolicyIngressRule) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{31} + return fileDescriptorGenerated, []int{32} } func (m *NetworkPolicyList) Reset() { *m = NetworkPolicyList{} } func (*NetworkPolicyList) ProtoMessage() {} -func (*NetworkPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{32} } +func (*NetworkPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{33} } func (m *NetworkPolicyPeer) Reset() { *m = NetworkPolicyPeer{} } func (*NetworkPolicyPeer) ProtoMessage() {} -func (*NetworkPolicyPeer) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{33} } +func (*NetworkPolicyPeer) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{34} } func (m *NetworkPolicyPort) Reset() { *m = NetworkPolicyPort{} } func (*NetworkPolicyPort) ProtoMessage() {} -func (*NetworkPolicyPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{34} } +func (*NetworkPolicyPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{35} } func (m *NetworkPolicySpec) Reset() { *m = NetworkPolicySpec{} } func (*NetworkPolicySpec) ProtoMessage() {} -func (*NetworkPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{35} } +func (*NetworkPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} } func (m *PodSecurityPolicy) Reset() { *m = PodSecurityPolicy{} } func (*PodSecurityPolicy) ProtoMessage() {} -func (*PodSecurityPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{36} } +func (*PodSecurityPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} } func (m *PodSecurityPolicyList) Reset() { *m = PodSecurityPolicyList{} } func (*PodSecurityPolicyList) ProtoMessage() {} -func (*PodSecurityPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{37} } +func (*PodSecurityPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} } func (m *PodSecurityPolicySpec) Reset() { *m = PodSecurityPolicySpec{} } func (*PodSecurityPolicySpec) ProtoMessage() {} -func (*PodSecurityPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{38} } +func (*PodSecurityPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} } func (m *ReplicaSet) Reset() { *m = ReplicaSet{} } func (*ReplicaSet) ProtoMessage() {} -func (*ReplicaSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{39} } +func (*ReplicaSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} } func (m *ReplicaSetCondition) Reset() { *m = ReplicaSetCondition{} } func (*ReplicaSetCondition) ProtoMessage() {} -func (*ReplicaSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{40} } +func (*ReplicaSetCondition) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} } func (m *ReplicaSetList) Reset() { *m = ReplicaSetList{} } func (*ReplicaSetList) ProtoMessage() {} -func (*ReplicaSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{41} } +func (*ReplicaSetList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} } func (m *ReplicaSetSpec) Reset() { *m = ReplicaSetSpec{} } func (*ReplicaSetSpec) ProtoMessage() {} -func (*ReplicaSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{42} } +func (*ReplicaSetSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} } func (m *ReplicaSetStatus) Reset() { *m = ReplicaSetStatus{} } func (*ReplicaSetStatus) ProtoMessage() {} -func (*ReplicaSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{43} } +func (*ReplicaSetStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{44} } func (m *ReplicationControllerDummy) Reset() { *m = ReplicationControllerDummy{} } func (*ReplicationControllerDummy) ProtoMessage() {} func (*ReplicationControllerDummy) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{44} + return fileDescriptorGenerated, []int{45} } func (m *RollbackConfig) Reset() { *m = RollbackConfig{} } func (*RollbackConfig) ProtoMessage() {} -func (*RollbackConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{45} } +func (*RollbackConfig) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} } func (m *RollingUpdateDaemonSet) Reset() { *m = RollingUpdateDaemonSet{} } func (*RollingUpdateDaemonSet) ProtoMessage() {} -func (*RollingUpdateDaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{46} } +func (*RollingUpdateDaemonSet) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{47} } func (m *RollingUpdateDeployment) Reset() { *m = RollingUpdateDeployment{} } func (*RollingUpdateDeployment) ProtoMessage() {} func (*RollingUpdateDeployment) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{47} + return fileDescriptorGenerated, []int{48} } func (m *RunAsUserStrategyOptions) Reset() { *m = RunAsUserStrategyOptions{} } func (*RunAsUserStrategyOptions) ProtoMessage() {} func (*RunAsUserStrategyOptions) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{48} + return fileDescriptorGenerated, []int{49} } func (m *SELinuxStrategyOptions) Reset() { *m = SELinuxStrategyOptions{} } func (*SELinuxStrategyOptions) ProtoMessage() {} -func (*SELinuxStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{49} } +func (*SELinuxStrategyOptions) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{50} } func (m *Scale) Reset() { *m = Scale{} } func (*Scale) ProtoMessage() {} -func (*Scale) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{50} } +func (*Scale) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{51} } func (m *ScaleSpec) Reset() { *m = ScaleSpec{} } func (*ScaleSpec) ProtoMessage() {} -func (*ScaleSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{51} } +func (*ScaleSpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{52} } func (m *ScaleStatus) Reset() { *m = ScaleStatus{} } func (*ScaleStatus) ProtoMessage() {} -func (*ScaleStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{52} } +func (*ScaleStatus) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{53} } func (m *SupplementalGroupsStrategyOptions) Reset() { *m = SupplementalGroupsStrategyOptions{} } func (*SupplementalGroupsStrategyOptions) ProtoMessage() {} func (*SupplementalGroupsStrategyOptions) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{53} + return fileDescriptorGenerated, []int{54} } func (m *ThirdPartyResource) Reset() { *m = ThirdPartyResource{} } func (*ThirdPartyResource) ProtoMessage() {} -func (*ThirdPartyResource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{54} } +func (*ThirdPartyResource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} } func (m *ThirdPartyResourceData) Reset() { *m = ThirdPartyResourceData{} } func (*ThirdPartyResourceData) ProtoMessage() {} -func (*ThirdPartyResourceData) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{55} } +func (*ThirdPartyResourceData) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{56} } func (m *ThirdPartyResourceDataList) Reset() { *m = ThirdPartyResourceDataList{} } func (*ThirdPartyResourceDataList) ProtoMessage() {} func (*ThirdPartyResourceDataList) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{56} + return fileDescriptorGenerated, []int{57} } func (m *ThirdPartyResourceList) Reset() { *m = ThirdPartyResourceList{} } func (*ThirdPartyResourceList) ProtoMessage() {} -func (*ThirdPartyResourceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{57} } +func (*ThirdPartyResourceList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{58} } func init() { proto.RegisterType((*APIVersion)(nil), "k8s.io.api.extensions.v1beta1.APIVersion") @@ -385,6 +390,7 @@ func init() { proto.RegisterType((*HTTPIngressRuleValue)(nil), "k8s.io.api.extensions.v1beta1.HTTPIngressRuleValue") proto.RegisterType((*HostPortRange)(nil), "k8s.io.api.extensions.v1beta1.HostPortRange") proto.RegisterType((*IDRange)(nil), "k8s.io.api.extensions.v1beta1.IDRange") + proto.RegisterType((*IPBlock)(nil), "k8s.io.api.extensions.v1beta1.IPBlock") proto.RegisterType((*Ingress)(nil), "k8s.io.api.extensions.v1beta1.Ingress") proto.RegisterType((*IngressBackend)(nil), "k8s.io.api.extensions.v1beta1.IngressBackend") proto.RegisterType((*IngressList)(nil), "k8s.io.api.extensions.v1beta1.IngressList") @@ -1267,6 +1273,43 @@ func (m *IDRange) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *IPBlock) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *IPBlock) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.CIDR))) + i += copy(dAtA[i:], m.CIDR) + if len(m.Except) > 0 { + for _, s := range m.Except { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + func (m *Ingress) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1699,6 +1742,16 @@ func (m *NetworkPolicyPeer) MarshalTo(dAtA []byte) (int, error) { } i += n37 } + if m.IPBlock != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.IPBlock.Size())) + n38, err := m.IPBlock.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n38 + } return i, nil } @@ -1727,11 +1780,11 @@ func (m *NetworkPolicyPort) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Port.Size())) - n38, err := m.Port.MarshalTo(dAtA[i:]) + n39, err := m.Port.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n38 + i += n39 } return i, nil } @@ -1754,11 +1807,11 @@ func (m *NetworkPolicySpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PodSelector.Size())) - n39, err := m.PodSelector.MarshalTo(dAtA[i:]) + n40, err := m.PodSelector.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n39 + i += n40 if len(m.Ingress) > 0 { for _, msg := range m.Ingress { dAtA[i] = 0x12 @@ -1792,19 +1845,19 @@ func (m *PodSecurityPolicy) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n40, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n41, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n40 + i += n41 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n41, err := m.Spec.MarshalTo(dAtA[i:]) + n42, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n41 + i += n42 return i, nil } @@ -1826,11 +1879,11 @@ func (m *PodSecurityPolicyList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n42, err := m.ListMeta.MarshalTo(dAtA[i:]) + n43, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n42 + i += n43 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -1968,35 +2021,35 @@ func (m *PodSecurityPolicySpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x52 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SELinux.Size())) - n43, err := m.SELinux.MarshalTo(dAtA[i:]) + n44, err := m.SELinux.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n43 + i += n44 dAtA[i] = 0x5a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.RunAsUser.Size())) - n44, err := m.RunAsUser.MarshalTo(dAtA[i:]) + n45, err := m.RunAsUser.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n44 + i += n45 dAtA[i] = 0x62 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SupplementalGroups.Size())) - n45, err := m.SupplementalGroups.MarshalTo(dAtA[i:]) + n46, err := m.SupplementalGroups.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n45 + i += n46 dAtA[i] = 0x6a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.FSGroup.Size())) - n46, err := m.FSGroup.MarshalTo(dAtA[i:]) + n47, err := m.FSGroup.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n46 + i += n47 dAtA[i] = 0x70 i++ if m.ReadOnlyRootFilesystem { @@ -2046,27 +2099,27 @@ func (m *ReplicaSet) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n47, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n48, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n47 + i += n48 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n48, err := m.Spec.MarshalTo(dAtA[i:]) + n49, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n48 + i += n49 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n49, err := m.Status.MarshalTo(dAtA[i:]) + n50, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n49 + i += n50 return i, nil } @@ -2096,11 +2149,11 @@ func (m *ReplicaSetCondition) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.LastTransitionTime.Size())) - n50, err := m.LastTransitionTime.MarshalTo(dAtA[i:]) + n51, err := m.LastTransitionTime.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n50 + i += n51 dAtA[i] = 0x22 i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Reason))) @@ -2130,11 +2183,11 @@ func (m *ReplicaSetList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n51, err := m.ListMeta.MarshalTo(dAtA[i:]) + n52, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n51 + i += n52 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -2174,20 +2227,20 @@ func (m *ReplicaSetSpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Selector.Size())) - n52, err := m.Selector.MarshalTo(dAtA[i:]) + n53, err := m.Selector.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n52 + i += n53 } dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Template.Size())) - n53, err := m.Template.MarshalTo(dAtA[i:]) + n54, err := m.Template.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n53 + i += n54 dAtA[i] = 0x20 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.MinReadySeconds)) @@ -2297,11 +2350,11 @@ func (m *RollingUpdateDaemonSet) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.MaxUnavailable.Size())) - n54, err := m.MaxUnavailable.MarshalTo(dAtA[i:]) + n55, err := m.MaxUnavailable.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n54 + i += n55 } return i, nil } @@ -2325,21 +2378,21 @@ func (m *RollingUpdateDeployment) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.MaxUnavailable.Size())) - n55, err := m.MaxUnavailable.MarshalTo(dAtA[i:]) + n56, err := m.MaxUnavailable.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n55 + i += n56 } if m.MaxSurge != nil { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.MaxSurge.Size())) - n56, err := m.MaxSurge.MarshalTo(dAtA[i:]) + n57, err := m.MaxSurge.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n56 + i += n57 } return i, nil } @@ -2401,11 +2454,11 @@ func (m *SELinuxStrategyOptions) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.SELinuxOptions.Size())) - n57, err := m.SELinuxOptions.MarshalTo(dAtA[i:]) + n58, err := m.SELinuxOptions.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n57 + i += n58 } return i, nil } @@ -2428,27 +2481,27 @@ func (m *Scale) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n58, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n59, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n58 + i += n59 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Spec.Size())) - n59, err := m.Spec.MarshalTo(dAtA[i:]) + n60, err := m.Spec.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n59 + i += n60 dAtA[i] = 0x1a i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Status.Size())) - n60, err := m.Status.MarshalTo(dAtA[i:]) + n61, err := m.Status.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n60 + i += n61 return i, nil } @@ -2572,11 +2625,11 @@ func (m *ThirdPartyResource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n61, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n62, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n61 + i += n62 dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(len(m.Description))) @@ -2614,11 +2667,11 @@ func (m *ThirdPartyResourceData) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ObjectMeta.Size())) - n62, err := m.ObjectMeta.MarshalTo(dAtA[i:]) + n63, err := m.ObjectMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n62 + i += n63 if m.Data != nil { dAtA[i] = 0x12 i++ @@ -2646,11 +2699,11 @@ func (m *ThirdPartyResourceDataList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n63, err := m.ListMeta.MarshalTo(dAtA[i:]) + n64, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n63 + i += n64 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -2684,11 +2737,11 @@ func (m *ThirdPartyResourceList) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.ListMeta.Size())) - n64, err := m.ListMeta.MarshalTo(dAtA[i:]) + n65, err := m.ListMeta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n64 + i += n65 if len(m.Items) > 0 { for _, msg := range m.Items { dAtA[i] = 0x12 @@ -3033,6 +3086,20 @@ func (m *IDRange) Size() (n int) { return n } +func (m *IPBlock) Size() (n int) { + var l int + _ = l + l = len(m.CIDR) + n += 1 + l + sovGenerated(uint64(l)) + if len(m.Except) > 0 { + for _, s := range m.Except { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } + return n +} + func (m *Ingress) Size() (n int) { var l int _ = l @@ -3186,6 +3253,10 @@ func (m *NetworkPolicyPeer) Size() (n int) { l = m.NamespaceSelector.Size() n += 1 + l + sovGenerated(uint64(l)) } + if m.IPBlock != nil { + l = m.IPBlock.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -3830,6 +3901,17 @@ func (this *IDRange) String() string { }, "") return s } +func (this *IPBlock) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&IPBlock{`, + `CIDR:` + fmt.Sprintf("%v", this.CIDR) + `,`, + `Except:` + fmt.Sprintf("%v", this.Except) + `,`, + `}`, + }, "") + return s +} func (this *Ingress) String() string { if this == nil { return "nil" @@ -3958,6 +4040,7 @@ func (this *NetworkPolicyPeer) String() string { s := strings.Join([]string{`&NetworkPolicyPeer{`, `PodSelector:` + strings.Replace(fmt.Sprintf("%v", this.PodSelector), "LabelSelector", "k8s_io_apimachinery_pkg_apis_meta_v1.LabelSelector", 1) + `,`, `NamespaceSelector:` + strings.Replace(fmt.Sprintf("%v", this.NamespaceSelector), "LabelSelector", "k8s_io_apimachinery_pkg_apis_meta_v1.LabelSelector", 1) + `,`, + `IPBlock:` + strings.Replace(fmt.Sprintf("%v", this.IPBlock), "IPBlock", "IPBlock", 1) + `,`, `}`, }, "") return s @@ -5362,7 +5445,7 @@ func (m *DaemonSetStatus) Unmarshal(dAtA []byte) error { if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CollisionCount", wireType) } - var v int64 + var v int32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -5372,7 +5455,7 @@ func (m *DaemonSetStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= (int32(b) & 0x7F) << shift if b < 0x80 { break } @@ -6666,7 +6749,7 @@ func (m *DeploymentStatus) Unmarshal(dAtA []byte) error { if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CollisionCount", wireType) } - var v int64 + var v int32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -6676,7 +6759,7 @@ func (m *DeploymentStatus) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int64(b) & 0x7F) << shift + v |= (int32(b) & 0x7F) << shift if b < 0x80 { break } @@ -7291,6 +7374,114 @@ func (m *IDRange) Unmarshal(dAtA []byte) error { } return nil } +func (m *IPBlock) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: IPBlock: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: IPBlock: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CIDR", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CIDR = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Except", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Except = append(m.Except, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Ingress) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -8604,6 +8795,39 @@ func (m *NetworkPolicyPeer) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field IPBlock", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.IPBlock == nil { + m.IPBlock = &IPBlock{} + } + if err := m.IPBlock.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -11959,220 +12183,224 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 3431 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x5b, 0x4f, 0x6c, 0x1b, 0xc7, - 0xd5, 0xf7, 0x8a, 0xa2, 0x44, 0x3d, 0x59, 0x92, 0x35, 0x72, 0x64, 0x46, 0x8e, 0x45, 0x67, 0x03, - 0xf8, 0xb3, 0xf3, 0xd9, 0x64, 0xec, 0xc4, 0x89, 0xbf, 0x18, 0x5f, 0xbe, 0x4f, 0x94, 0xfc, 0x47, - 0xa9, 0x24, 0xd3, 0x43, 0x4a, 0x69, 0x8d, 0xb8, 0xcd, 0x8a, 0x1c, 0x51, 0x6b, 0x2d, 0x77, 0x37, - 0xbb, 0xb3, 0x8a, 0x79, 0x29, 0x7a, 0x0a, 0x50, 0xa0, 0x45, 0xdb, 0x43, 0x8a, 0xf4, 0xd6, 0x5c, - 0x7a, 0x6a, 0xd1, 0xdc, 0xda, 0x43, 0x50, 0xa0, 0x40, 0x0b, 0x18, 0x45, 0x5a, 0xe4, 0xd4, 0xe6, - 0x24, 0x34, 0xca, 0xb1, 0xe7, 0x02, 0x85, 0x0f, 0x45, 0x31, 0xb3, 0xb3, 0xff, 0x77, 0x45, 0x52, - 0xb1, 0x85, 0xa2, 0x37, 0x71, 0xde, 0x7b, 0xbf, 0xf7, 0x67, 0x66, 0xde, 0xbc, 0x99, 0xb7, 0x82, - 0x9b, 0x3b, 0xd7, 0xec, 0xb2, 0x6a, 0x54, 0x76, 0x9c, 0x4d, 0x62, 0xe9, 0x84, 0x12, 0xbb, 0xb2, - 0x4b, 0xf4, 0x96, 0x61, 0x55, 0x04, 0x41, 0x31, 0xd5, 0x0a, 0x79, 0x48, 0x89, 0x6e, 0xab, 0x86, - 0x6e, 0x57, 0x76, 0x2f, 0x6f, 0x12, 0xaa, 0x5c, 0xae, 0xb4, 0x89, 0x4e, 0x2c, 0x85, 0x92, 0x56, - 0xd9, 0xb4, 0x0c, 0x6a, 0xa0, 0x33, 0x2e, 0x7b, 0x59, 0x31, 0xd5, 0x72, 0xc0, 0x5e, 0x16, 0xec, - 0x73, 0x97, 0xda, 0x2a, 0xdd, 0x76, 0x36, 0xcb, 0x4d, 0xa3, 0x53, 0x69, 0x1b, 0x6d, 0xa3, 0xc2, - 0xa5, 0x36, 0x9d, 0x2d, 0xfe, 0x8b, 0xff, 0xe0, 0x7f, 0xb9, 0x68, 0x73, 0x72, 0x48, 0x79, 0xd3, - 0xb0, 0x48, 0x65, 0x37, 0xa1, 0x71, 0xee, 0x42, 0x88, 0xc7, 0x34, 0x34, 0xb5, 0xd9, 0xcd, 0x32, - 0x6e, 0xee, 0x95, 0x80, 0xb5, 0xa3, 0x34, 0xb7, 0x55, 0x9d, 0x58, 0xdd, 0x8a, 0xb9, 0xd3, 0xe6, - 0xb2, 0x16, 0xb1, 0x0d, 0xc7, 0x6a, 0x92, 0x81, 0xa4, 0xec, 0x4a, 0x87, 0x50, 0x25, 0xcd, 0xac, - 0x4a, 0x96, 0x94, 0xe5, 0xe8, 0x54, 0xed, 0x24, 0xd5, 0xbc, 0xda, 0x4b, 0xc0, 0x6e, 0x6e, 0x93, - 0x8e, 0x92, 0x90, 0x7b, 0x39, 0x4b, 0xce, 0xa1, 0xaa, 0x56, 0x51, 0x75, 0x6a, 0x53, 0x2b, 0x2e, - 0x24, 0x97, 0x01, 0x16, 0x6a, 0xcb, 0x1b, 0xc4, 0x62, 0xd3, 0x83, 0xce, 0xc2, 0xb0, 0xae, 0x74, - 0x48, 0x51, 0x3a, 0x2b, 0x9d, 0x1f, 0xab, 0x1e, 0x7f, 0xb4, 0x57, 0x3a, 0xb6, 0xbf, 0x57, 0x1a, - 0x5e, 0x53, 0x3a, 0x04, 0x73, 0x8a, 0xfc, 0x13, 0x09, 0x9e, 0x5d, 0x74, 0x6c, 0x6a, 0x74, 0x56, - 0x09, 0xb5, 0xd4, 0xe6, 0xa2, 0x63, 0x59, 0x44, 0xa7, 0x75, 0xaa, 0x50, 0xc7, 0xee, 0x2d, 0x8f, - 0xee, 0x41, 0x7e, 0x57, 0xd1, 0x1c, 0x52, 0x1c, 0x3a, 0x2b, 0x9d, 0x1f, 0xbf, 0x52, 0x2e, 0x07, - 0xcb, 0xc4, 0x37, 0xba, 0x6c, 0xee, 0xb4, 0xf9, 0xba, 0xf1, 0x66, 0xa2, 0x7c, 0xd7, 0x51, 0x74, - 0xaa, 0xd2, 0x6e, 0xf5, 0xa4, 0x80, 0x3c, 0x2e, 0xf4, 0x6e, 0x30, 0x2c, 0xec, 0x42, 0xca, 0xdf, - 0x86, 0x33, 0x99, 0xa6, 0xad, 0xa8, 0x36, 0x45, 0xf7, 0x21, 0xaf, 0x52, 0xd2, 0xb1, 0x8b, 0xd2, - 0xd9, 0xdc, 0xf9, 0xf1, 0x2b, 0xd7, 0xca, 0x07, 0xae, 0xd1, 0x72, 0x26, 0x58, 0x75, 0x42, 0x98, - 0x91, 0x5f, 0x66, 0x70, 0xd8, 0x45, 0x95, 0x7f, 0x24, 0x01, 0x0a, 0xcb, 0x34, 0x14, 0xab, 0x4d, - 0x68, 0x1f, 0x41, 0xf9, 0xc6, 0x57, 0x0b, 0xca, 0x8c, 0x80, 0x1c, 0x77, 0x15, 0x46, 0x62, 0x62, - 0xc2, 0x6c, 0xd2, 0x24, 0x1e, 0x8c, 0x8d, 0x68, 0x30, 0x2e, 0x0f, 0x10, 0x0c, 0x17, 0x25, 0x23, - 0x0a, 0x1f, 0x0c, 0xc1, 0xd8, 0x92, 0x42, 0x3a, 0x86, 0x5e, 0x27, 0x14, 0xbd, 0x03, 0x05, 0xb6, - 0x31, 0x5a, 0x0a, 0x55, 0x78, 0x00, 0xc6, 0xaf, 0xbc, 0x74, 0x90, 0x77, 0x76, 0x99, 0x71, 0x97, - 0x77, 0x2f, 0x97, 0xef, 0x6c, 0x3e, 0x20, 0x4d, 0xba, 0x4a, 0xa8, 0x52, 0x45, 0x42, 0x0f, 0x04, - 0x63, 0xd8, 0x47, 0x45, 0x6b, 0x30, 0x6c, 0x9b, 0xa4, 0x29, 0x62, 0x77, 0xb1, 0x87, 0x1b, 0xbe, - 0x65, 0x75, 0x93, 0x34, 0x83, 0xc9, 0x60, 0xbf, 0x30, 0xc7, 0x41, 0x1b, 0x30, 0x62, 0xf3, 0x59, - 0x2e, 0xe6, 0x12, 0xb3, 0x71, 0x30, 0xa2, 0xbb, 0x36, 0x26, 0x05, 0xe6, 0x88, 0xfb, 0x1b, 0x0b, - 0x34, 0xf9, 0x13, 0x09, 0x26, 0x7c, 0x5e, 0x3e, 0x03, 0x6f, 0x27, 0x62, 0x53, 0xee, 0x2f, 0x36, - 0x4c, 0x9a, 0x47, 0xe6, 0x84, 0xd0, 0x55, 0xf0, 0x46, 0x42, 0x71, 0x59, 0xf5, 0xe6, 0x77, 0x88, - 0xcf, 0xef, 0xf9, 0x7e, 0xdd, 0xc8, 0x98, 0xd6, 0x1f, 0x0f, 0x87, 0xcc, 0x67, 0xe1, 0x42, 0xf7, - 0xa1, 0x60, 0x13, 0x8d, 0x34, 0xa9, 0x61, 0x09, 0xf3, 0x5f, 0xee, 0xd3, 0x7c, 0x65, 0x93, 0x68, - 0x75, 0x21, 0x5a, 0x3d, 0xce, 0xec, 0xf7, 0x7e, 0x61, 0x1f, 0x12, 0xdd, 0x85, 0x02, 0x25, 0x1d, - 0x53, 0x53, 0xa8, 0xb7, 0x2f, 0x5e, 0x08, 0xbb, 0xc0, 0x4e, 0x01, 0x06, 0x56, 0x33, 0x5a, 0x0d, - 0xc1, 0xc6, 0xa7, 0xd4, 0x0f, 0x89, 0x37, 0x8a, 0x7d, 0x18, 0xb4, 0x0b, 0x93, 0x8e, 0xd9, 0x62, - 0x9c, 0x94, 0xe5, 0xc0, 0x76, 0x57, 0x4c, 0xf1, 0xab, 0xfd, 0xc6, 0x66, 0x3d, 0x22, 0x5d, 0x9d, - 0x15, 0xba, 0x26, 0xa3, 0xe3, 0x38, 0xa6, 0x05, 0x2d, 0xc0, 0x54, 0x47, 0xd5, 0x31, 0x51, 0x5a, - 0xdd, 0x3a, 0x69, 0x1a, 0x7a, 0xcb, 0x2e, 0x0e, 0x9f, 0x95, 0xce, 0xe7, 0xab, 0xa7, 0x04, 0xc0, - 0xd4, 0x6a, 0x94, 0x8c, 0xe3, 0xfc, 0xe8, 0x4d, 0x40, 0x9e, 0x1b, 0xb7, 0xdc, 0x14, 0xae, 0x1a, - 0x7a, 0x31, 0x7f, 0x56, 0x3a, 0x9f, 0xab, 0xce, 0x09, 0x14, 0xd4, 0x48, 0x70, 0xe0, 0x14, 0x29, - 0xb4, 0x02, 0x27, 0x2d, 0xb2, 0xab, 0x32, 0x1f, 0x6f, 0xab, 0x36, 0x35, 0xac, 0xee, 0x8a, 0xda, - 0x51, 0x69, 0x71, 0x84, 0xdb, 0x54, 0xdc, 0xdf, 0x2b, 0x9d, 0xc4, 0x29, 0x74, 0x9c, 0x2a, 0x25, - 0x7f, 0x9c, 0x87, 0xa9, 0xd8, 0x1e, 0x40, 0x1b, 0x30, 0xdb, 0x74, 0x13, 0xe6, 0x9a, 0xd3, 0xd9, - 0x24, 0x56, 0xbd, 0xb9, 0x4d, 0x5a, 0x8e, 0x46, 0x5a, 0x7c, 0xa1, 0xe4, 0xab, 0xf3, 0xc2, 0xe2, - 0xd9, 0xc5, 0x54, 0x2e, 0x9c, 0x21, 0xcd, 0xa2, 0xa0, 0xf3, 0xa1, 0x55, 0xd5, 0xb6, 0x7d, 0xcc, - 0x21, 0x8e, 0xe9, 0x47, 0x61, 0x2d, 0xc1, 0x81, 0x53, 0xa4, 0x98, 0x8d, 0x2d, 0x62, 0xab, 0x16, - 0x69, 0xc5, 0x6d, 0xcc, 0x45, 0x6d, 0x5c, 0x4a, 0xe5, 0xc2, 0x19, 0xd2, 0xe8, 0x2a, 0x8c, 0xbb, - 0xda, 0xf8, 0xfc, 0x89, 0x89, 0xf6, 0x53, 0xf4, 0x5a, 0x40, 0xc2, 0x61, 0x3e, 0xe6, 0x9a, 0xb1, - 0x69, 0x13, 0x6b, 0x97, 0xb4, 0xb2, 0x27, 0xf8, 0x4e, 0x82, 0x03, 0xa7, 0x48, 0x31, 0xd7, 0xdc, - 0x15, 0x98, 0x70, 0x6d, 0x24, 0xea, 0xda, 0x7a, 0x2a, 0x17, 0xce, 0x90, 0x66, 0xeb, 0xd8, 0x35, - 0x79, 0x61, 0x57, 0x51, 0x35, 0x65, 0x53, 0x23, 0xc5, 0xd1, 0xe8, 0x3a, 0x5e, 0x8b, 0x92, 0x71, - 0x9c, 0x1f, 0xdd, 0x82, 0x69, 0x77, 0x68, 0x5d, 0x57, 0x7c, 0x90, 0x02, 0x07, 0x79, 0x56, 0x80, - 0x4c, 0xaf, 0xc5, 0x19, 0x70, 0x52, 0x06, 0xbd, 0x0e, 0x93, 0x4d, 0x43, 0xd3, 0xf8, 0x7a, 0x5c, - 0x34, 0x1c, 0x9d, 0x16, 0xc7, 0x78, 0xac, 0x10, 0xdb, 0x8f, 0x8b, 0x11, 0x0a, 0x8e, 0x71, 0xca, - 0x7f, 0x90, 0xe0, 0x54, 0xc6, 0x9e, 0x46, 0xff, 0x07, 0xc3, 0xb4, 0x6b, 0x7a, 0xa7, 0xf5, 0x7f, - 0x7b, 0x07, 0x44, 0xa3, 0x6b, 0x92, 0xc7, 0x7b, 0xa5, 0xd3, 0x19, 0x62, 0x8c, 0x8c, 0xb9, 0x20, - 0xd2, 0x61, 0xc2, 0x62, 0xea, 0xf4, 0xb6, 0xcb, 0x22, 0x92, 0xd7, 0xd5, 0x1e, 0x39, 0x06, 0x87, - 0x65, 0x82, 0x64, 0x3c, 0xbd, 0xbf, 0x57, 0x9a, 0x88, 0xd0, 0x70, 0x14, 0x5e, 0xfe, 0x70, 0x08, - 0x60, 0x89, 0x98, 0x9a, 0xd1, 0xed, 0x10, 0xfd, 0x28, 0x0e, 0xdc, 0x3b, 0x91, 0x03, 0xf7, 0x52, - 0xaf, 0xdc, 0xe9, 0x9b, 0x96, 0x79, 0xe2, 0xbe, 0x15, 0x3b, 0x71, 0x2b, 0xfd, 0x43, 0x1e, 0x7c, - 0xe4, 0xfe, 0x25, 0x07, 0x33, 0x01, 0xf3, 0xa2, 0xa1, 0xb7, 0x54, 0xbe, 0x3f, 0xae, 0x47, 0xe6, - 0xf8, 0xbf, 0x62, 0x73, 0x7c, 0x2a, 0x45, 0x24, 0x34, 0xbf, 0x2b, 0xbe, 0xb5, 0x43, 0x5c, 0xfc, - 0x95, 0xa8, 0xf2, 0xc7, 0x7b, 0xa5, 0x94, 0xcb, 0x4a, 0xd9, 0x47, 0x8a, 0x9a, 0x88, 0xce, 0xc1, - 0x88, 0x45, 0x14, 0xdb, 0xd0, 0x79, 0xa2, 0x18, 0x0b, 0x5c, 0xc1, 0x7c, 0x14, 0x0b, 0x2a, 0xba, - 0x00, 0xa3, 0x1d, 0x62, 0xdb, 0x4a, 0x9b, 0xf0, 0x9c, 0x30, 0x56, 0x9d, 0x12, 0x8c, 0xa3, 0xab, - 0xee, 0x30, 0xf6, 0xe8, 0xe8, 0x01, 0x4c, 0x6a, 0x8a, 0x2d, 0x16, 0x68, 0x43, 0xed, 0x10, 0xbe, - 0xeb, 0xc7, 0xaf, 0xbc, 0xd8, 0xdf, 0x3a, 0x60, 0x12, 0xc1, 0xc9, 0xb6, 0x12, 0x41, 0xc2, 0x31, - 0x64, 0xb4, 0x0b, 0x88, 0x8d, 0x34, 0x2c, 0x45, 0xb7, 0xdd, 0x40, 0x31, 0x7d, 0xa3, 0x03, 0xeb, - 0xf3, 0x33, 0xdc, 0x4a, 0x02, 0x0d, 0xa7, 0x68, 0x90, 0x7f, 0x23, 0xc1, 0x64, 0x30, 0x4d, 0x47, - 0x50, 0x4d, 0xad, 0x45, 0xab, 0xa9, 0x0b, 0x7d, 0x2f, 0xd1, 0x8c, 0x72, 0xea, 0x1f, 0x43, 0x80, - 0x02, 0x26, 0xb6, 0xc1, 0x37, 0x95, 0xe6, 0x4e, 0x1f, 0x77, 0x85, 0x0f, 0x24, 0x40, 0x22, 0x3d, - 0x2f, 0xe8, 0xba, 0x41, 0x79, 0xc6, 0xf7, 0xcc, 0x5a, 0xee, 0xdb, 0x2c, 0x4f, 0x63, 0x79, 0x3d, - 0x81, 0x75, 0x43, 0xa7, 0x56, 0x37, 0x98, 0x91, 0x24, 0x03, 0x4e, 0x31, 0x00, 0x29, 0x00, 0x96, - 0xc0, 0x6c, 0x18, 0x62, 0x23, 0x5f, 0xea, 0x23, 0xe7, 0x31, 0x81, 0x45, 0x43, 0xdf, 0x52, 0xdb, - 0x41, 0xda, 0xc1, 0x3e, 0x10, 0x0e, 0x81, 0xce, 0xdd, 0x80, 0x53, 0x19, 0xd6, 0xa2, 0x13, 0x90, - 0xdb, 0x21, 0x5d, 0x37, 0x6c, 0x98, 0xfd, 0x89, 0x4e, 0x86, 0xef, 0x54, 0x63, 0xe2, 0x3a, 0xf4, - 0xfa, 0xd0, 0x35, 0x49, 0xfe, 0x24, 0x1f, 0x5e, 0x3b, 0xbc, 0x94, 0x3d, 0x0f, 0x05, 0x8b, 0x98, - 0x9a, 0xda, 0x54, 0x6c, 0x51, 0xa1, 0xf0, 0xaa, 0x14, 0x8b, 0x31, 0xec, 0x53, 0x23, 0x45, 0xef, - 0xd0, 0xd3, 0x2d, 0x7a, 0x73, 0x4f, 0xa6, 0xe8, 0xfd, 0x16, 0x14, 0x6c, 0xaf, 0xdc, 0x1d, 0xe6, - 0x90, 0x97, 0x07, 0xc8, 0xaf, 0xa2, 0xd2, 0xf5, 0x15, 0xf8, 0x35, 0xae, 0x0f, 0x9a, 0x56, 0xdd, - 0xe6, 0x07, 0xac, 0x6e, 0x9f, 0x68, 0x45, 0xca, 0x72, 0xaa, 0xa9, 0x38, 0x36, 0x69, 0xf1, 0x44, - 0x54, 0x08, 0x72, 0x6a, 0x8d, 0x8f, 0x62, 0x41, 0x45, 0xf7, 0x23, 0x4b, 0xb6, 0x70, 0x98, 0x25, - 0x3b, 0x99, 0xbd, 0x5c, 0xd1, 0x3a, 0x9c, 0x32, 0x2d, 0xa3, 0x6d, 0x11, 0xdb, 0x5e, 0x22, 0x4a, - 0x4b, 0x53, 0x75, 0xe2, 0xc5, 0x67, 0x8c, 0xfb, 0x75, 0x7a, 0x7f, 0xaf, 0x74, 0xaa, 0x96, 0xce, - 0x82, 0xb3, 0x64, 0xe5, 0x47, 0xc3, 0x70, 0x22, 0x7e, 0x02, 0x66, 0x54, 0x8f, 0xd2, 0xa1, 0xaa, - 0xc7, 0x8b, 0xa1, 0xcd, 0xe0, 0x96, 0xd6, 0xfe, 0xec, 0xa7, 0x6c, 0x88, 0x05, 0x98, 0x12, 0xd9, - 0xc0, 0x23, 0x8a, 0xfa, 0xd9, 0x9f, 0xfd, 0xf5, 0x28, 0x19, 0xc7, 0xf9, 0x59, 0x4d, 0x18, 0x94, - 0x7a, 0x1e, 0xc8, 0x70, 0xb4, 0x26, 0x5c, 0x88, 0x33, 0xe0, 0xa4, 0x0c, 0x5a, 0x85, 0x19, 0x47, - 0x4f, 0x42, 0xb9, 0xab, 0xf1, 0xb4, 0x80, 0x9a, 0x59, 0x4f, 0xb2, 0xe0, 0x34, 0x39, 0xb4, 0x05, - 0xd0, 0xf4, 0x8e, 0x6d, 0xbb, 0x38, 0xc2, 0x33, 0xec, 0x95, 0xbe, 0xf7, 0x8e, 0x7f, 0xe2, 0x07, - 0x79, 0xcd, 0x1f, 0xb2, 0x71, 0x08, 0x19, 0x5d, 0x87, 0x09, 0x8b, 0x5f, 0x08, 0x3c, 0x83, 0xdd, - 0xa2, 0xfa, 0x19, 0x21, 0x36, 0x81, 0xc3, 0x44, 0x1c, 0xe5, 0x4d, 0xa9, 0x83, 0x0b, 0x7d, 0xd7, - 0xc1, 0xbf, 0x95, 0xc2, 0x87, 0x90, 0x5f, 0x02, 0xbf, 0x1e, 0x29, 0x8f, 0xce, 0xc5, 0xca, 0xa3, - 0xd9, 0xa4, 0x44, 0xa8, 0x3a, 0x32, 0xd2, 0xab, 0xdf, 0x57, 0x07, 0xaa, 0x7e, 0x83, 0xc3, 0xb3, - 0x77, 0xf9, 0xfb, 0x91, 0x04, 0xb3, 0x37, 0xeb, 0xb7, 0x2c, 0xc3, 0x31, 0x3d, 0x73, 0xee, 0x98, - 0x6e, 0x5c, 0x5f, 0x83, 0x61, 0xcb, 0xd1, 0x3c, 0x3f, 0x5e, 0xf0, 0xfc, 0xc0, 0x8e, 0xc6, 0xfc, - 0x98, 0x89, 0x49, 0xb9, 0x4e, 0x30, 0x01, 0xb4, 0x06, 0x23, 0x96, 0xa2, 0xb7, 0x89, 0x77, 0xac, - 0x9e, 0xeb, 0x61, 0xfd, 0xf2, 0x12, 0x66, 0xec, 0xa1, 0xe2, 0x8d, 0x4b, 0x63, 0x81, 0x22, 0x7f, - 0x5f, 0x82, 0xa9, 0xdb, 0x8d, 0x46, 0x6d, 0x59, 0xe7, 0x3b, 0xba, 0xa6, 0xd0, 0x6d, 0x76, 0xd2, - 0x9b, 0x0a, 0xdd, 0x8e, 0x9f, 0xf4, 0x8c, 0x86, 0x39, 0x05, 0x7d, 0x1d, 0x46, 0x59, 0x26, 0x21, - 0x7a, 0xab, 0xcf, 0x52, 0x5b, 0xc0, 0x57, 0x5d, 0xa1, 0xa0, 0x42, 0x14, 0x03, 0xd8, 0x83, 0x93, - 0x77, 0xe0, 0x64, 0xc8, 0x1c, 0x16, 0x0f, 0xfe, 0x66, 0x88, 0xea, 0x90, 0x67, 0x9a, 0xbd, 0x27, - 0xc1, 0x5e, 0x2f, 0x5f, 0x31, 0x97, 0x82, 0x4a, 0x87, 0xfd, 0xb2, 0xb1, 0x8b, 0x25, 0xaf, 0xc2, - 0xc4, 0x6d, 0xc3, 0xa6, 0x35, 0xc3, 0xa2, 0x3c, 0x2c, 0xe8, 0x0c, 0xe4, 0x3a, 0xaa, 0x2e, 0xce, - 0xd9, 0x71, 0x21, 0x93, 0x63, 0x67, 0x04, 0x1b, 0xe7, 0x64, 0xe5, 0xa1, 0xc8, 0x3c, 0x01, 0x59, - 0x79, 0x88, 0xd9, 0xb8, 0x7c, 0x0b, 0x46, 0x45, 0xb8, 0xc3, 0x40, 0xb9, 0x83, 0x81, 0x72, 0x29, - 0x40, 0x3f, 0x18, 0x82, 0x51, 0x61, 0xfd, 0x11, 0x5c, 0x9a, 0x56, 0x22, 0x97, 0xa6, 0x17, 0xfb, - 0x9b, 0xc9, 0xcc, 0x1b, 0x53, 0x23, 0x76, 0x63, 0xba, 0xd8, 0x27, 0xde, 0xc1, 0xd7, 0xa5, 0x8f, - 0x25, 0x98, 0x8c, 0xae, 0x21, 0x74, 0x15, 0xc6, 0xd9, 0xf9, 0xa0, 0x36, 0xc9, 0x5a, 0x50, 0x96, - 0xfa, 0x8f, 0x19, 0xf5, 0x80, 0x84, 0xc3, 0x7c, 0xa8, 0xed, 0x8b, 0xb1, 0x69, 0x17, 0x4e, 0x67, - 0x87, 0xd4, 0xa1, 0xaa, 0x56, 0x76, 0x1b, 0x14, 0xe5, 0x65, 0x9d, 0xde, 0xb1, 0xea, 0xd4, 0x52, - 0xf5, 0x76, 0x42, 0x11, 0x5f, 0x43, 0x61, 0x64, 0xf9, 0xd7, 0x12, 0x8c, 0x0b, 0x93, 0x8f, 0xe0, - 0x12, 0xf0, 0xb5, 0xe8, 0x25, 0xe0, 0x5c, 0x9f, 0xfb, 0x31, 0xfd, 0x06, 0xf0, 0xb3, 0xc0, 0x74, - 0xb6, 0x03, 0x59, 0x42, 0xd8, 0x36, 0x6c, 0x1a, 0x4f, 0x08, 0x6c, 0xef, 0x60, 0x4e, 0x41, 0x0e, - 0x9c, 0x50, 0x63, 0x5b, 0x56, 0x84, 0xb6, 0xd2, 0x9f, 0x25, 0xbe, 0x58, 0xb5, 0x28, 0xe0, 0x4f, - 0xc4, 0x29, 0x38, 0xa1, 0x42, 0x26, 0x90, 0xe0, 0x42, 0x77, 0x61, 0x78, 0x9b, 0x52, 0x33, 0xe5, - 0xdd, 0xb7, 0x47, 0xa2, 0x08, 0x4c, 0x28, 0x70, 0xef, 0x1a, 0x8d, 0x1a, 0xe6, 0x50, 0xf2, 0x3f, - 0x83, 0x78, 0xd4, 0xdd, 0x35, 0xee, 0xa7, 0x3f, 0xe9, 0x30, 0xe9, 0x6f, 0x3c, 0x2d, 0xf5, 0xa1, - 0xdb, 0x90, 0xa3, 0x5a, 0xbf, 0xb7, 0x38, 0x81, 0xd8, 0x58, 0xa9, 0x07, 0xf9, 0xa3, 0xb1, 0x52, - 0xc7, 0x0c, 0x02, 0xdd, 0x81, 0x3c, 0x3b, 0x2c, 0xd8, 0x16, 0xcc, 0xf5, 0xbf, 0xa5, 0x99, 0xff, - 0xc1, 0x82, 0x60, 0xbf, 0x6c, 0xec, 0xe2, 0xc8, 0xef, 0xc2, 0x44, 0x64, 0x9f, 0xa2, 0x77, 0xe0, - 0xb8, 0x66, 0x28, 0xad, 0xaa, 0xa2, 0x29, 0x7a, 0x93, 0x78, 0x8f, 0xec, 0xe7, 0xd2, 0x2e, 0x04, - 0x2b, 0x21, 0x3e, 0xb1, 0xcb, 0xfd, 0x56, 0x59, 0x98, 0x86, 0x23, 0x88, 0xb2, 0x02, 0x10, 0xf8, - 0x88, 0x4a, 0x90, 0x67, 0xeb, 0xcc, 0x4d, 0xff, 0x63, 0xd5, 0x31, 0x66, 0x21, 0x5b, 0x7e, 0x36, - 0x76, 0xc7, 0xd1, 0x15, 0x00, 0x9b, 0x34, 0x2d, 0x42, 0x79, 0x32, 0x70, 0x9f, 0x3f, 0xfc, 0xb4, - 0x57, 0xf7, 0x29, 0x38, 0xc4, 0x25, 0xff, 0x5e, 0x82, 0x89, 0x35, 0x42, 0xdf, 0x33, 0xac, 0x9d, - 0x1a, 0x6f, 0xca, 0x1e, 0x41, 0xb2, 0xc5, 0x91, 0x64, 0xfb, 0x52, 0x8f, 0x99, 0x89, 0x58, 0x97, - 0x95, 0x72, 0x99, 0x1f, 0xc5, 0x08, 0x67, 0x78, 0xef, 0xae, 0x43, 0xde, 0x34, 0x2c, 0xea, 0x1d, - 0x9c, 0x03, 0x69, 0x64, 0x79, 0x2c, 0x74, 0x74, 0x32, 0x18, 0xec, 0xa2, 0x31, 0x3f, 0xb6, 0x2c, - 0xa3, 0x23, 0x56, 0xeb, 0x60, 0xa8, 0x84, 0x58, 0x81, 0x1f, 0x37, 0x2d, 0xa3, 0x83, 0x39, 0x96, - 0xfc, 0x3b, 0x09, 0xa6, 0x23, 0x9c, 0x47, 0x90, 0x37, 0xef, 0x46, 0xf3, 0xe6, 0xc5, 0x41, 0x1c, - 0xc9, 0xc8, 0x9e, 0x7f, 0x8f, 0xbb, 0xc1, 0x1c, 0x46, 0x5b, 0x30, 0x6e, 0x1a, 0xad, 0xfa, 0x13, - 0xe8, 0x4a, 0x4d, 0xb1, 0x63, 0xa7, 0x16, 0x60, 0xe1, 0x30, 0x30, 0x7a, 0x08, 0xd3, 0xba, 0xd2, - 0x21, 0xb6, 0xa9, 0x34, 0x49, 0xfd, 0x09, 0x3c, 0x07, 0x3c, 0xc3, 0x9f, 0xbd, 0xe3, 0x88, 0x38, - 0xa9, 0x44, 0xfe, 0x79, 0xc2, 0x6f, 0xc3, 0xa2, 0xe8, 0x16, 0x14, 0x78, 0x3b, 0xbf, 0x69, 0x68, - 0xde, 0xc3, 0x35, 0x9b, 0x8a, 0x9a, 0x18, 0x7b, 0xbc, 0x57, 0x3a, 0x9d, 0xf2, 0x26, 0xe9, 0x91, - 0xb1, 0x2f, 0x8c, 0xd6, 0x60, 0xd8, 0xfc, 0x2a, 0x27, 0x36, 0x4f, 0xea, 0xfc, 0x98, 0xe6, 0x38, - 0xf2, 0x97, 0x71, 0x73, 0x79, 0x6a, 0x7f, 0xf0, 0xc4, 0xa6, 0xc9, 0xaf, 0x10, 0x32, 0xa7, 0x6a, - 0x13, 0x46, 0xc5, 0x89, 0x26, 0x56, 0xdf, 0x6b, 0x83, 0xac, 0xbe, 0x70, 0xd6, 0xf6, 0xeb, 0x69, - 0x6f, 0xd0, 0x03, 0x96, 0xff, 0x28, 0xc1, 0x34, 0x37, 0xa0, 0xe9, 0x58, 0x2a, 0xed, 0x1e, 0x59, - 0x9e, 0xdb, 0x88, 0xe4, 0xb9, 0x57, 0x7a, 0x38, 0x96, 0xb0, 0x30, 0x33, 0xd7, 0x7d, 0x2a, 0xc1, - 0x33, 0x09, 0xee, 0x23, 0xc8, 0x13, 0xeb, 0xd1, 0x3c, 0xf1, 0xd2, 0xa0, 0x0e, 0x65, 0x55, 0x5a, - 0xe3, 0x29, 0xee, 0xf0, 0x85, 0x78, 0x05, 0xc0, 0xb4, 0xd4, 0x5d, 0x55, 0x23, 0x6d, 0xd1, 0x9b, - 0x2c, 0x04, 0x21, 0xaf, 0xf9, 0x14, 0x1c, 0xe2, 0x42, 0x36, 0xcc, 0xb6, 0xc8, 0x96, 0xe2, 0x68, - 0x74, 0xa1, 0xd5, 0x5a, 0x54, 0x4c, 0x65, 0x53, 0xd5, 0x54, 0xaa, 0x8a, 0xcb, 0xe2, 0x58, 0xf5, - 0xba, 0xdb, 0x33, 0x4c, 0xe3, 0x78, 0xbc, 0x57, 0x3a, 0x93, 0xd6, 0x1b, 0xf0, 0x58, 0xba, 0x38, - 0x03, 0x1a, 0x75, 0xa1, 0x68, 0x91, 0x77, 0x1d, 0xd5, 0x22, 0xad, 0x25, 0xcb, 0x30, 0x23, 0x6a, - 0x73, 0x5c, 0xed, 0xff, 0xee, 0xef, 0x95, 0x8a, 0x38, 0x83, 0xa7, 0xb7, 0xe2, 0x4c, 0x78, 0xf4, - 0x00, 0x66, 0x14, 0x4d, 0x33, 0xde, 0x23, 0x51, 0x67, 0x87, 0xb9, 0xd6, 0x6b, 0xfb, 0x7b, 0xa5, - 0x99, 0x85, 0x24, 0xb9, 0xb7, 0xc2, 0x34, 0x50, 0x54, 0x81, 0xd1, 0x5d, 0x43, 0x73, 0x3a, 0xc4, - 0x2e, 0xe6, 0x39, 0x3e, 0x4b, 0x8c, 0xa3, 0x1b, 0xee, 0xd0, 0xe3, 0xbd, 0xd2, 0xc8, 0xcd, 0x3a, - 0xbf, 0xa6, 0x7b, 0x5c, 0xec, 0x7e, 0xc2, 0x4a, 0x13, 0xb1, 0x67, 0xf9, 0x7b, 0x61, 0x21, 0x48, - 0x0a, 0xb7, 0x03, 0x12, 0x0e, 0xf3, 0xa1, 0xfb, 0x30, 0xb6, 0x2d, 0xee, 0xa4, 0x76, 0x71, 0xb4, - 0xaf, 0x43, 0x29, 0x72, 0x87, 0xad, 0x4e, 0x0b, 0x15, 0x63, 0xde, 0xb0, 0x8d, 0x03, 0x44, 0x74, - 0x01, 0x46, 0xf9, 0x8f, 0xe5, 0x25, 0xfe, 0x18, 0x53, 0x08, 0x52, 0xc7, 0x6d, 0x77, 0x18, 0x7b, - 0x74, 0x8f, 0x75, 0xb9, 0xb6, 0xc8, 0x1f, 0x05, 0x63, 0xac, 0xcb, 0xb5, 0x45, 0xec, 0xd1, 0xd1, - 0x3b, 0x30, 0x6a, 0x93, 0x15, 0x55, 0x77, 0x1e, 0x16, 0xa1, 0xaf, 0x96, 0x62, 0xfd, 0x06, 0xe7, - 0x8e, 0x3d, 0x8b, 0x04, 0x1a, 0x04, 0x1d, 0x7b, 0xb0, 0x68, 0x1b, 0xc6, 0x2c, 0x47, 0x5f, 0xb0, - 0xd7, 0x6d, 0x62, 0x15, 0xc7, 0xb9, 0x8e, 0x5e, 0xd9, 0x12, 0x7b, 0xfc, 0x71, 0x2d, 0x7e, 0x84, - 0x7c, 0x0e, 0x1c, 0x80, 0xa3, 0xef, 0x49, 0x80, 0x6c, 0xc7, 0x34, 0x35, 0xd2, 0x21, 0x3a, 0x55, - 0x34, 0xfe, 0x12, 0x63, 0x17, 0x8f, 0x73, 0x9d, 0xff, 0xdf, 0xcb, 0xaf, 0x84, 0x60, 0x5c, 0xb9, - 0xff, 0xe4, 0x99, 0x64, 0xc5, 0x29, 0x7a, 0x59, 0x68, 0xb7, 0x6c, 0xfe, 0x77, 0x71, 0xa2, 0xaf, - 0xd0, 0xa6, 0xbf, 0x38, 0x05, 0xa1, 0x15, 0x74, 0xec, 0xc1, 0xa2, 0x0d, 0x98, 0xb5, 0x88, 0xd2, - 0xba, 0xa3, 0x6b, 0x5d, 0x6c, 0x18, 0xf4, 0xa6, 0xaa, 0x11, 0xbb, 0x6b, 0x53, 0xd2, 0x29, 0x4e, - 0xf2, 0x69, 0xf7, 0x5b, 0xf2, 0x38, 0x95, 0x0b, 0x67, 0x48, 0xa3, 0x0e, 0x94, 0xbc, 0x94, 0xc1, - 0xf6, 0x93, 0x9f, 0xb3, 0x6e, 0xd8, 0x4d, 0x45, 0x73, 0x5f, 0x81, 0xa7, 0xb8, 0x82, 0x17, 0xf6, - 0xf7, 0x4a, 0xa5, 0xa5, 0x83, 0x59, 0x71, 0x2f, 0x2c, 0xf4, 0x36, 0x14, 0x95, 0x2c, 0x3d, 0x27, - 0xb8, 0x9e, 0xb3, 0xc2, 0x91, 0x62, 0xa6, 0x92, 0x4c, 0x04, 0xde, 0xca, 0x16, 0x0f, 0x9b, 0x47, - 0xf3, 0xed, 0xd8, 0x60, 0xad, 0xec, 0xc0, 0xb4, 0x27, 0xd6, 0xca, 0x0e, 0x41, 0x1e, 0xfc, 0x36, - 0xf3, 0xb7, 0x21, 0x98, 0x09, 0x98, 0xfb, 0x6e, 0x65, 0xa7, 0x88, 0x3c, 0xb5, 0x56, 0x76, 0x7a, - 0x2f, 0x38, 0xf7, 0xb4, 0x7b, 0xc1, 0x4f, 0xa1, 0x85, 0xce, 0xdb, 0xcb, 0x41, 0xe8, 0xfe, 0xfd, - 0xda, 0xcb, 0x81, 0x6d, 0x19, 0x25, 0xcf, 0x2f, 0x87, 0xc2, 0x0e, 0xfc, 0xc7, 0xf7, 0x38, 0xbf, - 0xfa, 0x07, 0x76, 0xf2, 0xa7, 0x39, 0x38, 0x11, 0xdf, 0x8d, 0x91, 0x56, 0x98, 0xd4, 0xb3, 0x15, - 0x56, 0x83, 0x93, 0x5b, 0x8e, 0xa6, 0x75, 0x79, 0x18, 0x42, 0xfd, 0x30, 0xf7, 0x29, 0xfb, 0x39, - 0x21, 0x79, 0xf2, 0x66, 0x0a, 0x0f, 0x4e, 0x95, 0xcc, 0x68, 0xeb, 0xe5, 0x0e, 0xd5, 0xd6, 0x4b, - 0x74, 0x99, 0x86, 0x07, 0xe8, 0x32, 0xa5, 0xb6, 0xe8, 0xf2, 0x87, 0x68, 0xd1, 0x1d, 0xa6, 0xa7, - 0x96, 0x92, 0xc4, 0x7a, 0xf5, 0xd4, 0xe4, 0xe7, 0x60, 0x4e, 0x88, 0x51, 0xde, 0xee, 0xd2, 0xa9, - 0x65, 0x68, 0x1a, 0xb1, 0x96, 0x9c, 0x4e, 0xa7, 0x2b, 0xbf, 0x01, 0x93, 0xd1, 0x46, 0xae, 0x3b, - 0xd3, 0x6e, 0x2f, 0x59, 0x34, 0x14, 0x42, 0x33, 0xed, 0x8e, 0x63, 0x9f, 0x43, 0x7e, 0x5f, 0x82, - 0xd9, 0xf4, 0x0f, 0xb6, 0x90, 0x06, 0x93, 0x1d, 0xe5, 0x61, 0xf8, 0xeb, 0x36, 0xe9, 0x90, 0x77, - 0x69, 0xde, 0xc1, 0x5b, 0x8d, 0x60, 0xe1, 0x18, 0x36, 0xbb, 0x5f, 0x9f, 0xca, 0xe8, 0x9d, 0x1d, - 0xad, 0x25, 0xe8, 0x1e, 0x14, 0x3a, 0xca, 0xc3, 0xba, 0x63, 0xb5, 0xc9, 0xa1, 0x5f, 0x0f, 0x78, - 0xc6, 0x58, 0x15, 0x28, 0xd8, 0xc7, 0x93, 0x3f, 0x92, 0xa0, 0x98, 0x55, 0x68, 0xa2, 0xab, 0x91, - 0x2e, 0xdf, 0xf3, 0xb1, 0x2e, 0xdf, 0x74, 0x42, 0xee, 0x29, 0xf5, 0xf8, 0x7e, 0x21, 0xc1, 0x6c, - 0x7a, 0xc1, 0x8d, 0x5e, 0x8e, 0x58, 0x58, 0x8a, 0x59, 0x38, 0x15, 0x93, 0x12, 0xf6, 0x7d, 0x13, - 0x26, 0x45, 0x59, 0x2e, 0x60, 0x44, 0x54, 0xe5, 0xb4, 0x5c, 0x29, 0x20, 0xbc, 0x32, 0x94, 0xcf, - 0x57, 0x74, 0x0c, 0xc7, 0xd0, 0xe4, 0xef, 0x0e, 0x41, 0xbe, 0xde, 0x54, 0x34, 0x72, 0x04, 0x65, - 0xd6, 0x9b, 0x91, 0x32, 0xab, 0xd7, 0x97, 0xe8, 0xdc, 0xaa, 0xcc, 0x0a, 0x0b, 0xc7, 0x2a, 0xac, - 0x17, 0xfb, 0x42, 0x3b, 0xb8, 0xb8, 0xfa, 0x1f, 0x18, 0xf3, 0x95, 0x0e, 0x96, 0xf3, 0xe5, 0x9f, - 0x0e, 0xc1, 0x78, 0x48, 0xc5, 0x80, 0x27, 0xc6, 0x56, 0xe4, 0xa4, 0xed, 0xe7, 0x7f, 0x52, 0x42, - 0xba, 0xca, 0xde, 0xd9, 0xea, 0x7e, 0xb0, 0x15, 0x7c, 0xa2, 0x93, 0x3c, 0x72, 0xdf, 0x80, 0x49, - 0xca, 0xff, 0x67, 0xc3, 0x7f, 0x73, 0xcb, 0xf1, 0xb5, 0xe8, 0x7f, 0xe6, 0xd7, 0x88, 0x50, 0x71, - 0x8c, 0x7b, 0xee, 0x3a, 0x4c, 0x44, 0x94, 0x0d, 0xf4, 0xbd, 0xd5, 0xaf, 0x24, 0x78, 0xbe, 0xe7, - 0x95, 0x0d, 0x55, 0x23, 0x9b, 0xa4, 0x1c, 0xdb, 0x24, 0xf3, 0xd9, 0x00, 0x4f, 0xb1, 0x6f, 0xff, - 0xfe, 0x10, 0xa0, 0xc6, 0xb6, 0x6a, 0xb5, 0x6a, 0x8a, 0x45, 0xbb, 0x58, 0xfc, 0xe3, 0xcd, 0x11, - 0x6c, 0x98, 0xab, 0x30, 0xde, 0x22, 0x76, 0xd3, 0x52, 0x79, 0x70, 0x44, 0x75, 0xee, 0x3f, 0x6b, - 0x2c, 0x05, 0x24, 0x1c, 0xe6, 0x43, 0x6f, 0x41, 0x61, 0xd7, 0xfd, 0x4f, 0x2e, 0xaf, 0x2b, 0xd5, - 0xab, 0x90, 0x0c, 0xfe, 0xf7, 0x2b, 0x58, 0x3f, 0x62, 0xc0, 0xc6, 0x3e, 0x98, 0xfc, 0xa1, 0x04, - 0xb3, 0xc9, 0x40, 0x2c, 0x31, 0x53, 0x9f, 0x7e, 0x30, 0x9e, 0x83, 0x61, 0x8e, 0xce, 0xa2, 0x70, - 0xdc, 0x7d, 0x61, 0x66, 0x9a, 0x31, 0x1f, 0x95, 0xff, 0x2c, 0xc1, 0x5c, 0xba, 0x69, 0x47, 0x50, - 0xb6, 0xdf, 0x8b, 0x96, 0xed, 0xbd, 0x5e, 0x0d, 0xd2, 0xed, 0xcc, 0x28, 0xe1, 0xff, 0x94, 0x1a, - 0xf3, 0x23, 0x70, 0x6a, 0x23, 0xea, 0xd4, 0xe5, 0x81, 0x9d, 0x4a, 0x77, 0xa8, 0x7a, 0xe9, 0xd1, - 0x17, 0xf3, 0xc7, 0x3e, 0xfb, 0x62, 0xfe, 0xd8, 0xe7, 0x5f, 0xcc, 0x1f, 0xfb, 0xce, 0xfe, 0xbc, - 0xf4, 0x68, 0x7f, 0x5e, 0xfa, 0x6c, 0x7f, 0x5e, 0xfa, 0x7c, 0x7f, 0x5e, 0xfa, 0xeb, 0xfe, 0xbc, - 0xf4, 0xc3, 0x2f, 0xe7, 0x8f, 0xdd, 0x1b, 0x15, 0xb8, 0xff, 0x0a, 0x00, 0x00, 0xff, 0xff, 0xc0, - 0x14, 0x27, 0xd4, 0x89, 0x3a, 0x00, 0x00, + // 3495 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x5b, 0xcd, 0x6f, 0x1b, 0xd7, + 0xb5, 0xf7, 0x90, 0xa2, 0x48, 0x1d, 0x5a, 0x92, 0x75, 0xe5, 0xc8, 0x8c, 0x1c, 0x8b, 0xce, 0x04, + 0xf0, 0xb3, 0xf3, 0x6c, 0x32, 0x76, 0xe2, 0xc4, 0x2f, 0xc6, 0xcb, 0x7b, 0xa2, 0xe4, 0x0f, 0xa5, + 0xfa, 0xa0, 0x2f, 0x29, 0xa5, 0x35, 0xe2, 0x36, 0x23, 0xf2, 0x8a, 0x1a, 0x6b, 0x38, 0x33, 0x99, + 0x0f, 0x45, 0xdc, 0x14, 0x5d, 0x05, 0x28, 0xd0, 0xa2, 0xed, 0x22, 0x45, 0xba, 0x6b, 0x36, 0x5d, + 0xb5, 0x68, 0x76, 0xed, 0x22, 0x28, 0x50, 0x20, 0x05, 0x8c, 0x22, 0x2d, 0xb2, 0x6a, 0xb3, 0x12, + 0x1a, 0x65, 0xd9, 0x7f, 0xa0, 0xf0, 0xa2, 0x28, 0xee, 0x9d, 0x3b, 0xdf, 0x33, 0x22, 0xa9, 0xd8, + 0x42, 0xd1, 0x9d, 0x78, 0xef, 0x39, 0xbf, 0xf3, 0x71, 0xef, 0x3d, 0xe7, 0xdc, 0x7b, 0x46, 0x70, + 0x7b, 0xe7, 0x86, 0x59, 0x91, 0xb5, 0xea, 0x8e, 0xbd, 0x49, 0x0c, 0x95, 0x58, 0xc4, 0xac, 0xee, + 0x12, 0xb5, 0xad, 0x19, 0x55, 0x3e, 0x21, 0xe9, 0x72, 0x95, 0xec, 0x59, 0x44, 0x35, 0x65, 0x4d, + 0x35, 0xab, 0xbb, 0x57, 0x37, 0x89, 0x25, 0x5d, 0xad, 0x76, 0x88, 0x4a, 0x0c, 0xc9, 0x22, 0xed, + 0x8a, 0x6e, 0x68, 0x96, 0x86, 0xce, 0x39, 0xe4, 0x15, 0x49, 0x97, 0x2b, 0x3e, 0x79, 0x85, 0x93, + 0xcf, 0x5e, 0xe9, 0xc8, 0xd6, 0xb6, 0xbd, 0x59, 0x69, 0x69, 0xdd, 0x6a, 0x47, 0xeb, 0x68, 0x55, + 0xc6, 0xb5, 0x69, 0x6f, 0xb1, 0x5f, 0xec, 0x07, 0xfb, 0xcb, 0x41, 0x9b, 0x15, 0x03, 0xc2, 0x5b, + 0x9a, 0x41, 0xaa, 0xbb, 0x31, 0x89, 0xb3, 0x97, 0x02, 0x34, 0xba, 0xa6, 0xc8, 0xad, 0x5e, 0x9a, + 0x72, 0xb3, 0xaf, 0xf8, 0xa4, 0x5d, 0xa9, 0xb5, 0x2d, 0xab, 0xc4, 0xe8, 0x55, 0xf5, 0x9d, 0x0e, + 0xe3, 0x35, 0x88, 0xa9, 0xd9, 0x46, 0x8b, 0x0c, 0xc5, 0x65, 0x56, 0xbb, 0xc4, 0x92, 0x92, 0xd4, + 0xaa, 0xa6, 0x71, 0x19, 0xb6, 0x6a, 0xc9, 0xdd, 0xb8, 0x98, 0x57, 0xfb, 0x31, 0x98, 0xad, 0x6d, + 0xd2, 0x95, 0x62, 0x7c, 0x2f, 0xa7, 0xf1, 0xd9, 0x96, 0xac, 0x54, 0x65, 0xd5, 0x32, 0x2d, 0x23, + 0xca, 0x24, 0x56, 0x00, 0xe6, 0xeb, 0x4b, 0x1b, 0xc4, 0xa0, 0xcb, 0x83, 0xce, 0xc3, 0x88, 0x2a, + 0x75, 0x49, 0x49, 0x38, 0x2f, 0x5c, 0x1c, 0xab, 0x9d, 0x7c, 0xb4, 0x5f, 0x3e, 0x71, 0xb0, 0x5f, + 0x1e, 0x59, 0x95, 0xba, 0x04, 0xb3, 0x19, 0xf1, 0x67, 0x02, 0x3c, 0xbb, 0x60, 0x9b, 0x96, 0xd6, + 0x5d, 0x21, 0x96, 0x21, 0xb7, 0x16, 0x6c, 0xc3, 0x20, 0xaa, 0xd5, 0xb0, 0x24, 0xcb, 0x36, 0xfb, + 0xf3, 0xa3, 0xfb, 0x90, 0xdb, 0x95, 0x14, 0x9b, 0x94, 0x32, 0xe7, 0x85, 0x8b, 0xc5, 0x6b, 0x95, + 0x8a, 0xbf, 0x4d, 0x3c, 0xa5, 0x2b, 0xfa, 0x4e, 0x87, 0xed, 0x1b, 0x77, 0x25, 0x2a, 0xf7, 0x6c, + 0x49, 0xb5, 0x64, 0xab, 0x57, 0x3b, 0xcd, 0x21, 0x4f, 0x72, 0xb9, 0x1b, 0x14, 0x0b, 0x3b, 0x90, + 0xe2, 0x77, 0xe1, 0x5c, 0xaa, 0x6a, 0xcb, 0xb2, 0x69, 0xa1, 0x07, 0x90, 0x93, 0x2d, 0xd2, 0x35, + 0x4b, 0xc2, 0xf9, 0xec, 0xc5, 0xe2, 0xb5, 0x1b, 0x95, 0x43, 0xf7, 0x68, 0x25, 0x15, 0xac, 0x36, + 0xce, 0xd5, 0xc8, 0x2d, 0x51, 0x38, 0xec, 0xa0, 0x8a, 0x3f, 0x11, 0x00, 0x05, 0x79, 0x9a, 0x92, + 0xd1, 0x21, 0xd6, 0x00, 0x4e, 0xf9, 0xd6, 0xd7, 0x73, 0xca, 0x34, 0x87, 0x2c, 0x3a, 0x02, 0x43, + 0x3e, 0xd1, 0x61, 0x26, 0xae, 0x12, 0x73, 0xc6, 0x46, 0xd8, 0x19, 0x57, 0x87, 0x70, 0x86, 0x83, + 0x92, 0xe2, 0x85, 0x0f, 0x32, 0x30, 0xb6, 0x28, 0x91, 0xae, 0xa6, 0x36, 0x88, 0x85, 0xde, 0x81, + 0x02, 0x3d, 0x18, 0x6d, 0xc9, 0x92, 0x98, 0x03, 0x8a, 0xd7, 0x5e, 0x3a, 0xcc, 0x3a, 0xb3, 0x42, + 0xa9, 0x2b, 0xbb, 0x57, 0x2b, 0x6b, 0x9b, 0x0f, 0x49, 0xcb, 0x5a, 0x21, 0x96, 0x54, 0x43, 0x5c, + 0x0e, 0xf8, 0x63, 0xd8, 0x43, 0x45, 0xab, 0x30, 0x62, 0xea, 0xa4, 0xc5, 0x7d, 0x77, 0xb9, 0x8f, + 0x19, 0x9e, 0x66, 0x0d, 0x9d, 0xb4, 0xfc, 0xc5, 0xa0, 0xbf, 0x30, 0xc3, 0x41, 0x1b, 0x30, 0x6a, + 0xb2, 0x55, 0x2e, 0x65, 0x63, 0xab, 0x71, 0x38, 0xa2, 0xb3, 0x37, 0x26, 0x38, 0xe6, 0xa8, 0xf3, + 0x1b, 0x73, 0x34, 0xf1, 0x13, 0x01, 0xc6, 0x3d, 0x5a, 0xb6, 0x02, 0x6f, 0xc7, 0x7c, 0x53, 0x19, + 0xcc, 0x37, 0x94, 0x9b, 0x79, 0xe6, 0x14, 0x97, 0x55, 0x70, 0x47, 0x02, 0x7e, 0x59, 0x71, 0xd7, + 0x37, 0xc3, 0xd6, 0xf7, 0xe2, 0xa0, 0x66, 0xa4, 0x2c, 0xeb, 0x4f, 0x47, 0x02, 0xea, 0x53, 0x77, + 0xa1, 0x07, 0x50, 0x30, 0x89, 0x42, 0x5a, 0x96, 0x66, 0x70, 0xf5, 0x5f, 0x1e, 0x50, 0x7d, 0x69, + 0x93, 0x28, 0x0d, 0xce, 0x5a, 0x3b, 0x49, 0xf5, 0x77, 0x7f, 0x61, 0x0f, 0x12, 0xdd, 0x83, 0x82, + 0x45, 0xba, 0xba, 0x22, 0x59, 0xee, 0xb9, 0x78, 0x21, 0x68, 0x02, 0xcd, 0x02, 0x14, 0xac, 0xae, + 0xb5, 0x9b, 0x9c, 0x8c, 0x2d, 0xa9, 0xe7, 0x12, 0x77, 0x14, 0x7b, 0x30, 0x68, 0x17, 0x26, 0x6c, + 0xbd, 0x4d, 0x29, 0x2d, 0x1a, 0x03, 0x3b, 0x3d, 0xbe, 0xc4, 0xaf, 0x0e, 0xea, 0x9b, 0xf5, 0x10, + 0x77, 0x6d, 0x86, 0xcb, 0x9a, 0x08, 0x8f, 0xe3, 0x88, 0x14, 0x34, 0x0f, 0x93, 0x5d, 0x59, 0xc5, + 0x44, 0x6a, 0xf7, 0x1a, 0xa4, 0xa5, 0xa9, 0x6d, 0xb3, 0x34, 0x72, 0x5e, 0xb8, 0x98, 0xab, 0x9d, + 0xe1, 0x00, 0x93, 0x2b, 0xe1, 0x69, 0x1c, 0xa5, 0x47, 0x6f, 0x02, 0x72, 0xcd, 0xb8, 0xe3, 0x84, + 0x70, 0x59, 0x53, 0x4b, 0xb9, 0xf3, 0xc2, 0xc5, 0x6c, 0x6d, 0x96, 0xa3, 0xa0, 0x66, 0x8c, 0x02, + 0x27, 0x70, 0xa1, 0x65, 0x38, 0x6d, 0x90, 0x5d, 0x99, 0xda, 0x78, 0x57, 0x36, 0x2d, 0xcd, 0xe8, + 0x2d, 0xcb, 0x5d, 0xd9, 0x2a, 0x8d, 0x32, 0x9d, 0x4a, 0x07, 0xfb, 0xe5, 0xd3, 0x38, 0x61, 0x1e, + 0x27, 0x72, 0x89, 0x1f, 0xe7, 0x60, 0x32, 0x72, 0x06, 0xd0, 0x06, 0xcc, 0xb4, 0x9c, 0x80, 0xb9, + 0x6a, 0x77, 0x37, 0x89, 0xd1, 0x68, 0x6d, 0x93, 0xb6, 0xad, 0x90, 0x36, 0xdb, 0x28, 0xb9, 0xda, + 0x1c, 0xd7, 0x78, 0x66, 0x21, 0x91, 0x0a, 0xa7, 0x70, 0x53, 0x2f, 0xa8, 0x6c, 0x68, 0x45, 0x36, + 0x4d, 0x0f, 0x33, 0xc3, 0x30, 0x3d, 0x2f, 0xac, 0xc6, 0x28, 0x70, 0x02, 0x17, 0xd5, 0xb1, 0x4d, + 0x4c, 0xd9, 0x20, 0xed, 0xa8, 0x8e, 0xd9, 0xb0, 0x8e, 0x8b, 0x89, 0x54, 0x38, 0x85, 0x1b, 0x5d, + 0x87, 0xa2, 0x23, 0x8d, 0xad, 0x1f, 0x5f, 0x68, 0x2f, 0x44, 0xaf, 0xfa, 0x53, 0x38, 0x48, 0x47, + 0x4d, 0xd3, 0x36, 0x4d, 0x62, 0xec, 0x92, 0x76, 0xfa, 0x02, 0xaf, 0xc5, 0x28, 0x70, 0x02, 0x17, + 0x35, 0xcd, 0xd9, 0x81, 0x31, 0xd3, 0x46, 0xc3, 0xa6, 0xad, 0x27, 0x52, 0xe1, 0x14, 0x6e, 0xba, + 0x8f, 0x1d, 0x95, 0xe7, 0x77, 0x25, 0x59, 0x91, 0x36, 0x15, 0x52, 0xca, 0x87, 0xf7, 0xf1, 0x6a, + 0x78, 0x1a, 0x47, 0xe9, 0xd1, 0x1d, 0x98, 0x72, 0x86, 0xd6, 0x55, 0xc9, 0x03, 0x29, 0x30, 0x90, + 0x67, 0x39, 0xc8, 0xd4, 0x6a, 0x94, 0x00, 0xc7, 0x79, 0xd0, 0xeb, 0x30, 0xd1, 0xd2, 0x14, 0x85, + 0xed, 0xc7, 0x05, 0xcd, 0x56, 0xad, 0xd2, 0x18, 0x43, 0x41, 0xf4, 0x3c, 0x2e, 0x84, 0x66, 0x70, + 0x84, 0x52, 0xfc, 0xa3, 0x00, 0x67, 0x52, 0xce, 0x34, 0xfa, 0x3f, 0x18, 0xb1, 0x7a, 0xba, 0x9b, + 0xad, 0xff, 0xdb, 0x4d, 0x10, 0xcd, 0x9e, 0x4e, 0x1e, 0xef, 0x97, 0xcf, 0xa6, 0xb0, 0xd1, 0x69, + 0xcc, 0x18, 0x91, 0x0a, 0xe3, 0x06, 0x15, 0xa7, 0x76, 0x1c, 0x12, 0x1e, 0xbc, 0xae, 0xf7, 0x89, + 0x31, 0x38, 0xc8, 0xe3, 0x07, 0xe3, 0xa9, 0x83, 0xfd, 0xf2, 0x78, 0x68, 0x0e, 0x87, 0xe1, 0xc5, + 0x0f, 0x33, 0x00, 0x8b, 0x44, 0x57, 0xb4, 0x5e, 0x97, 0xa8, 0xc7, 0x91, 0x70, 0xd7, 0x42, 0x09, + 0xf7, 0x4a, 0xbf, 0xd8, 0xe9, 0xa9, 0x96, 0x9a, 0x71, 0xdf, 0x8a, 0x64, 0xdc, 0xea, 0xe0, 0x90, + 0x87, 0xa7, 0xdc, 0xbf, 0x66, 0x61, 0xda, 0x27, 0x5e, 0xd0, 0xd4, 0xb6, 0xcc, 0xce, 0xc7, 0xcd, + 0xd0, 0x1a, 0xff, 0x57, 0x64, 0x8d, 0xcf, 0x24, 0xb0, 0x04, 0xd6, 0x77, 0xd9, 0xd3, 0x36, 0xc3, + 0xd8, 0x5f, 0x09, 0x0b, 0x7f, 0xbc, 0x5f, 0x4e, 0xb8, 0xac, 0x54, 0x3c, 0xa4, 0xb0, 0x8a, 0xe8, + 0x02, 0x8c, 0x1a, 0x44, 0x32, 0x35, 0x95, 0x05, 0x8a, 0x31, 0xdf, 0x14, 0xcc, 0x46, 0x31, 0x9f, + 0x45, 0x97, 0x20, 0xdf, 0x25, 0xa6, 0x29, 0x75, 0x08, 0x8b, 0x09, 0x63, 0xb5, 0x49, 0x4e, 0x98, + 0x5f, 0x71, 0x86, 0xb1, 0x3b, 0x8f, 0x1e, 0xc2, 0x84, 0x22, 0x99, 0x7c, 0x83, 0x36, 0xe5, 0x2e, + 0x61, 0xa7, 0xbe, 0x78, 0xed, 0xc5, 0xc1, 0xf6, 0x01, 0xe5, 0xf0, 0x33, 0xdb, 0x72, 0x08, 0x09, + 0x47, 0x90, 0xd1, 0x2e, 0x20, 0x3a, 0xd2, 0x34, 0x24, 0xd5, 0x74, 0x1c, 0x45, 0xe5, 0xe5, 0x87, + 0x96, 0xe7, 0x45, 0xb8, 0xe5, 0x18, 0x1a, 0x4e, 0x90, 0x20, 0xfe, 0x4e, 0x80, 0x09, 0x7f, 0x99, + 0x8e, 0xa1, 0x9a, 0x5a, 0x0d, 0x57, 0x53, 0x97, 0x06, 0xde, 0xa2, 0x29, 0xe5, 0xd4, 0x3f, 0x32, + 0x80, 0x7c, 0x22, 0x7a, 0xc0, 0x37, 0xa5, 0xd6, 0xce, 0x00, 0x77, 0x85, 0x0f, 0x04, 0x40, 0x3c, + 0x3c, 0xcf, 0xab, 0xaa, 0x66, 0xb1, 0x88, 0xef, 0xaa, 0xb5, 0x34, 0xb0, 0x5a, 0xae, 0xc4, 0xca, + 0x7a, 0x0c, 0xeb, 0x96, 0x6a, 0x19, 0x3d, 0x7f, 0x45, 0xe2, 0x04, 0x38, 0x41, 0x01, 0x24, 0x01, + 0x18, 0x1c, 0xb3, 0xa9, 0xf1, 0x83, 0x7c, 0x65, 0x80, 0x98, 0x47, 0x19, 0x16, 0x34, 0x75, 0x4b, + 0xee, 0xf8, 0x61, 0x07, 0x7b, 0x40, 0x38, 0x00, 0x3a, 0x7b, 0x0b, 0xce, 0xa4, 0x68, 0x8b, 0x4e, + 0x41, 0x76, 0x87, 0xf4, 0x1c, 0xb7, 0x61, 0xfa, 0x27, 0x3a, 0x1d, 0xbc, 0x53, 0x8d, 0xf1, 0xeb, + 0xd0, 0xeb, 0x99, 0x1b, 0x82, 0xf8, 0x49, 0x2e, 0xb8, 0x77, 0x58, 0x29, 0x7b, 0x11, 0x0a, 0x06, + 0xd1, 0x15, 0xb9, 0x25, 0x99, 0xbc, 0x42, 0x61, 0x55, 0x29, 0xe6, 0x63, 0xd8, 0x9b, 0x0d, 0x15, + 0xbd, 0x99, 0xa7, 0x5b, 0xf4, 0x66, 0x9f, 0x4c, 0xd1, 0xfb, 0x1d, 0x28, 0x98, 0x6e, 0xb9, 0x3b, + 0xc2, 0x20, 0xaf, 0x0e, 0x11, 0x5f, 0x79, 0xa5, 0xeb, 0x09, 0xf0, 0x6a, 0x5c, 0x0f, 0x34, 0xa9, + 0xba, 0xcd, 0x0d, 0x59, 0xdd, 0x3e, 0xd1, 0x8a, 0x94, 0xc6, 0x54, 0x5d, 0xb2, 0x4d, 0xd2, 0x66, + 0x81, 0xa8, 0xe0, 0xc7, 0xd4, 0x3a, 0x1b, 0xc5, 0x7c, 0x16, 0x3d, 0x08, 0x6d, 0xd9, 0xc2, 0x51, + 0xb6, 0xec, 0x44, 0xfa, 0x76, 0x45, 0xeb, 0x70, 0x46, 0x37, 0xb4, 0x8e, 0x41, 0x4c, 0x73, 0x91, + 0x48, 0x6d, 0x45, 0x56, 0x89, 0xeb, 0x1f, 0xa7, 0x54, 0x39, 0x7b, 0xb0, 0x5f, 0x3e, 0x53, 0x4f, + 0x26, 0xc1, 0x69, 0xbc, 0xe2, 0xa3, 0x11, 0x38, 0x15, 0xcd, 0x80, 0x29, 0xd5, 0xa3, 0x70, 0xa4, + 0xea, 0xf1, 0x72, 0xe0, 0x30, 0x38, 0xa5, 0xb5, 0xb7, 0xfa, 0x09, 0x07, 0x62, 0x1e, 0x26, 0x79, + 0x34, 0x70, 0x27, 0x79, 0xfd, 0xec, 0xad, 0xfe, 0x7a, 0x78, 0x1a, 0x47, 0xe9, 0x69, 0x4d, 0xe8, + 0x97, 0x7a, 0x2e, 0xc8, 0x48, 0xb8, 0x26, 0x9c, 0x8f, 0x12, 0xe0, 0x38, 0x0f, 0x5a, 0x81, 0x69, + 0x5b, 0x8d, 0x43, 0x39, 0xbb, 0xf1, 0x2c, 0x87, 0x9a, 0x5e, 0x8f, 0x93, 0xe0, 0x24, 0x3e, 0xb4, + 0x05, 0xd0, 0x72, 0xd3, 0xb6, 0x59, 0x1a, 0x65, 0x11, 0xf6, 0xda, 0xc0, 0x67, 0xc7, 0xcb, 0xf8, + 0x7e, 0x5c, 0xf3, 0x86, 0x4c, 0x1c, 0x40, 0x46, 0x37, 0x61, 0xdc, 0x60, 0x17, 0x02, 0x57, 0x61, + 0xa7, 0xa8, 0x7e, 0x86, 0xb3, 0x8d, 0xe3, 0xe0, 0x24, 0x0e, 0xd3, 0x26, 0xd4, 0xc1, 0x85, 0x81, + 0xeb, 0xe0, 0xdf, 0x0b, 0xc1, 0x24, 0xe4, 0x95, 0xc0, 0xaf, 0x87, 0xca, 0xa3, 0x0b, 0x91, 0xf2, + 0x68, 0x26, 0xce, 0x11, 0xa8, 0x8e, 0xb4, 0xe4, 0xea, 0xf7, 0xd5, 0xa1, 0xaa, 0x5f, 0x3f, 0x79, + 0xf6, 0x2f, 0x7f, 0x3f, 0x12, 0x60, 0xe6, 0x76, 0xe3, 0x8e, 0xa1, 0xd9, 0xba, 0xab, 0xce, 0x9a, + 0xee, 0xf8, 0xf5, 0x35, 0x18, 0x31, 0x6c, 0xc5, 0xb5, 0xe3, 0x05, 0xd7, 0x0e, 0x6c, 0x2b, 0xd4, + 0x8e, 0xe9, 0x08, 0x97, 0x63, 0x04, 0x65, 0x40, 0xab, 0x30, 0x6a, 0x48, 0x6a, 0x87, 0xb8, 0x69, + 0xf5, 0x42, 0x1f, 0xed, 0x97, 0x16, 0x31, 0x25, 0x0f, 0x14, 0x6f, 0x8c, 0x1b, 0x73, 0x14, 0xf1, + 0x87, 0x02, 0x4c, 0xde, 0x6d, 0x36, 0xeb, 0x4b, 0x2a, 0x3b, 0xd1, 0x75, 0xc9, 0xda, 0xa6, 0x99, + 0x5e, 0x97, 0xac, 0xed, 0x68, 0xa6, 0xa7, 0x73, 0x98, 0xcd, 0xa0, 0x6f, 0x42, 0x9e, 0x46, 0x12, + 0xa2, 0xb6, 0x07, 0x2c, 0xb5, 0x39, 0x7c, 0xcd, 0x61, 0xf2, 0x2b, 0x44, 0x3e, 0x80, 0x5d, 0x38, + 0x71, 0x07, 0x4e, 0x07, 0xd4, 0xa1, 0xfe, 0x60, 0x6f, 0x86, 0xa8, 0x01, 0x39, 0x2a, 0xd9, 0x7d, + 0x12, 0xec, 0xf7, 0xf2, 0x15, 0x31, 0xc9, 0xaf, 0x74, 0xe8, 0x2f, 0x13, 0x3b, 0x58, 0xe2, 0x0a, + 0x8c, 0xdf, 0xd5, 0x4c, 0xab, 0xae, 0x19, 0x16, 0x73, 0x0b, 0x3a, 0x07, 0xd9, 0xae, 0xac, 0xf2, + 0x3c, 0x5b, 0xe4, 0x3c, 0x59, 0x9a, 0x23, 0xe8, 0x38, 0x9b, 0x96, 0xf6, 0x78, 0xe4, 0xf1, 0xa7, + 0xa5, 0x3d, 0x4c, 0xc7, 0xc5, 0x3b, 0x90, 0xe7, 0xee, 0x0e, 0x02, 0x65, 0x0f, 0x07, 0xca, 0x26, + 0x00, 0xad, 0x41, 0x7e, 0xa9, 0x5e, 0x53, 0x34, 0xa7, 0xea, 0x6a, 0xc9, 0x6d, 0x23, 0xba, 0x16, + 0x0b, 0x4b, 0x8b, 0x18, 0xb3, 0x19, 0x24, 0xc2, 0x28, 0xd9, 0x6b, 0x11, 0xdd, 0x62, 0x3b, 0x62, + 0xac, 0x06, 0x74, 0x95, 0x6f, 0xb1, 0x11, 0xcc, 0x67, 0xc4, 0x1f, 0x65, 0x20, 0xcf, 0xdd, 0x71, + 0x0c, 0xb7, 0xb0, 0xe5, 0xd0, 0x2d, 0xec, 0xc5, 0xc1, 0xb6, 0x46, 0xea, 0x15, 0xac, 0x19, 0xb9, + 0x82, 0x5d, 0x1e, 0x10, 0xef, 0xf0, 0xfb, 0xd7, 0xc7, 0x02, 0x4c, 0x84, 0x37, 0x25, 0xba, 0x0e, + 0x45, 0x9a, 0x70, 0xe4, 0x16, 0x59, 0xf5, 0xeb, 0x5c, 0xef, 0x75, 0xa4, 0xe1, 0x4f, 0xe1, 0x20, + 0x1d, 0xea, 0x78, 0x6c, 0x74, 0x1f, 0x71, 0xa3, 0xd3, 0x5d, 0x6a, 0x5b, 0xb2, 0x52, 0x71, 0x3a, + 0x1e, 0x95, 0x25, 0xd5, 0x5a, 0x33, 0x1a, 0x96, 0x21, 0xab, 0x9d, 0x98, 0x20, 0xb6, 0x29, 0x83, + 0xc8, 0xe2, 0x6f, 0x05, 0x28, 0x72, 0x95, 0x8f, 0xe1, 0x56, 0xf1, 0x8d, 0xf0, 0xad, 0xe2, 0xc2, + 0x80, 0x07, 0x3c, 0xf9, 0x4a, 0xf1, 0x0b, 0x5f, 0x75, 0x7a, 0xa4, 0xe9, 0xae, 0xde, 0xd6, 0x4c, + 0x2b, 0xba, 0xab, 0xe9, 0x61, 0xc4, 0x6c, 0x06, 0xd9, 0x70, 0x4a, 0x8e, 0xc4, 0x00, 0xee, 0xda, + 0xea, 0x60, 0x9a, 0x78, 0x6c, 0xb5, 0x12, 0x87, 0x3f, 0x15, 0x9d, 0xc1, 0x31, 0x11, 0x22, 0x81, + 0x18, 0x15, 0xba, 0x07, 0x23, 0xdb, 0x96, 0xa5, 0x27, 0x3c, 0x24, 0xf7, 0x89, 0x3c, 0xbe, 0x0a, + 0x05, 0x66, 0x5d, 0xb3, 0x59, 0xc7, 0x0c, 0x4a, 0xfc, 0xa7, 0xef, 0x8f, 0x86, 0xb3, 0xc7, 0xbd, + 0x78, 0x2a, 0x1c, 0x25, 0x9e, 0x16, 0x93, 0x62, 0x29, 0xba, 0x0b, 0x59, 0x4b, 0x19, 0xf4, 0x5a, + 0xc8, 0x11, 0x9b, 0xcb, 0x0d, 0x3f, 0x20, 0x35, 0x97, 0x1b, 0x98, 0x42, 0xa0, 0x35, 0xc8, 0xd1, + 0xec, 0x43, 0x8f, 0x60, 0x76, 0xf0, 0x23, 0x4d, 0xed, 0xf7, 0x37, 0x04, 0xfd, 0x65, 0x62, 0x07, + 0x47, 0x7c, 0x17, 0xc6, 0x43, 0xe7, 0x14, 0xbd, 0x03, 0x27, 0x15, 0x4d, 0x6a, 0xd7, 0x24, 0x45, + 0x52, 0x5b, 0xc4, 0x7d, 0xb5, 0xbf, 0x90, 0x74, 0xc3, 0x58, 0x0e, 0xd0, 0xf1, 0x53, 0xee, 0xf5, + 0xde, 0x82, 0x73, 0x38, 0x84, 0x28, 0x4a, 0x00, 0xbe, 0x8d, 0xa8, 0x0c, 0x39, 0xba, 0xcf, 0x9c, + 0x7c, 0x32, 0x56, 0x1b, 0xa3, 0x1a, 0xd2, 0xed, 0x67, 0x62, 0x67, 0x1c, 0x5d, 0x03, 0x30, 0x49, + 0xcb, 0x20, 0x16, 0x0b, 0x06, 0xce, 0x7b, 0x8a, 0x17, 0xf6, 0x1a, 0xde, 0x0c, 0x0e, 0x50, 0x89, + 0x7f, 0x10, 0x60, 0x7c, 0x95, 0x58, 0xef, 0x69, 0xc6, 0x4e, 0x9d, 0x75, 0x79, 0x8f, 0x21, 0xd8, + 0xe2, 0x50, 0xb0, 0x7d, 0xa9, 0xcf, 0xca, 0x84, 0xb4, 0x4b, 0x0b, 0xb9, 0xd4, 0x8e, 0x52, 0x88, + 0x32, 0x78, 0x76, 0xd7, 0x21, 0xa7, 0x6b, 0x86, 0xe5, 0x66, 0xe2, 0xa1, 0x24, 0xd2, 0x38, 0x16, + 0xc8, 0xc5, 0x14, 0x06, 0x3b, 0x68, 0xd4, 0x8e, 0x2d, 0x43, 0xeb, 0xf2, 0xdd, 0x3a, 0x1c, 0x2a, + 0x21, 0x86, 0x6f, 0xc7, 0x6d, 0x43, 0xeb, 0x62, 0x86, 0x25, 0x7e, 0x2a, 0xc0, 0x54, 0x88, 0xf2, + 0x18, 0xe2, 0xe6, 0xbd, 0x70, 0xdc, 0xbc, 0x3c, 0x8c, 0x21, 0x29, 0xd1, 0xf3, 0xd3, 0x4c, 0xc4, + 0x0c, 0x6a, 0x30, 0xda, 0x82, 0xa2, 0xae, 0xb5, 0x1b, 0x4f, 0xa0, 0xcd, 0x35, 0x49, 0xd3, 0x4e, + 0xdd, 0xc7, 0xc2, 0x41, 0x60, 0xb4, 0x07, 0x53, 0xaa, 0xd4, 0x25, 0xa6, 0x2e, 0xb5, 0x48, 0xe3, + 0x09, 0xbc, 0x2f, 0x3c, 0xc3, 0xde, 0xd1, 0xa3, 0x88, 0x38, 0x2e, 0x04, 0xad, 0x40, 0x5e, 0xd6, + 0x59, 0x19, 0xc4, 0x53, 0x7f, 0xdf, 0x24, 0xe4, 0x14, 0x4d, 0x4e, 0x38, 0xe4, 0x3f, 0xb0, 0x8b, + 0x21, 0xfe, 0x32, 0xba, 0x1b, 0xe8, 0xfe, 0x43, 0x77, 0xa0, 0xc0, 0x3e, 0x37, 0x68, 0x69, 0x8a, + 0xfb, 0xb0, 0x4e, 0x57, 0xb6, 0xce, 0xc7, 0x1e, 0xef, 0x97, 0xcf, 0x26, 0xbc, 0x99, 0xba, 0xd3, + 0xd8, 0x63, 0x46, 0xab, 0x30, 0xa2, 0x7f, 0x9d, 0x02, 0x80, 0xe5, 0x08, 0x96, 0xf5, 0x19, 0x8e, + 0xf8, 0x55, 0x54, 0x5d, 0x96, 0x29, 0x1e, 0x3e, 0xb1, 0x55, 0xf7, 0x0a, 0x8e, 0xd4, 0x95, 0xdf, + 0x84, 0x3c, 0x4f, 0x90, 0x7c, 0x33, 0xbf, 0x36, 0xcc, 0x66, 0x0e, 0x26, 0x01, 0xaf, 0xde, 0x77, + 0x07, 0x5d, 0x60, 0xf1, 0x4f, 0x02, 0x4c, 0x31, 0x05, 0x5a, 0xb6, 0x21, 0x5b, 0xbd, 0x63, 0x0b, + 0x9b, 0x1b, 0xa1, 0xb0, 0xf9, 0x4a, 0x1f, 0xc3, 0x62, 0x1a, 0xa6, 0x86, 0xce, 0xcf, 0x04, 0x78, + 0x26, 0x46, 0x7d, 0x0c, 0x61, 0x67, 0x3d, 0x1c, 0x76, 0x5e, 0x1a, 0xd6, 0xa0, 0xb4, 0xc2, 0xad, + 0x98, 0x60, 0x0e, 0xdb, 0x88, 0xd7, 0x00, 0x74, 0x43, 0xde, 0x95, 0x15, 0xd2, 0xe1, 0xbd, 0xd3, + 0x82, 0xef, 0xf2, 0xba, 0x37, 0x83, 0x03, 0x54, 0xc8, 0x84, 0x99, 0x36, 0xd9, 0x92, 0x6c, 0xc5, + 0x9a, 0x6f, 0xb7, 0x17, 0x24, 0x5d, 0xda, 0x94, 0x15, 0xd9, 0x92, 0xf9, 0x65, 0x76, 0xac, 0x76, + 0xd3, 0xe9, 0x69, 0x26, 0x51, 0x3c, 0xde, 0x2f, 0x9f, 0x4b, 0xea, 0x5d, 0xb8, 0x24, 0x3d, 0x9c, + 0x02, 0x8d, 0x7a, 0x50, 0x32, 0xc8, 0xbb, 0xb6, 0x6c, 0x90, 0xf6, 0xa2, 0xa1, 0xe9, 0x21, 0xb1, + 0x59, 0x26, 0xf6, 0x7f, 0x0f, 0xf6, 0xcb, 0x25, 0x9c, 0x42, 0xd3, 0x5f, 0x70, 0x2a, 0x3c, 0x7a, + 0x08, 0xd3, 0x92, 0xa2, 0x68, 0xef, 0x91, 0xb0, 0xb1, 0x23, 0x4c, 0xea, 0x8d, 0x83, 0xfd, 0xf2, + 0xf4, 0x7c, 0x7c, 0xba, 0xbf, 0xc0, 0x24, 0x50, 0x54, 0x85, 0xfc, 0xae, 0xa6, 0xd8, 0x5d, 0x62, + 0x96, 0x72, 0x0c, 0x9f, 0xc6, 0xd9, 0xfc, 0x86, 0x33, 0xf4, 0x78, 0xbf, 0x3c, 0x7a, 0xbb, 0xc1, + 0x9e, 0x11, 0x5c, 0x2a, 0x7a, 0xdd, 0xa1, 0x95, 0x0e, 0x3f, 0xb3, 0xec, 0x3d, 0xb3, 0xe0, 0x07, + 0x85, 0xbb, 0xfe, 0x14, 0x0e, 0xd2, 0xa1, 0x07, 0x30, 0xb6, 0xcd, 0xef, 0xcc, 0x66, 0x29, 0x3f, + 0x50, 0x8e, 0x0b, 0xdd, 0xb1, 0x6b, 0x53, 0x5c, 0xc4, 0x98, 0x3b, 0x6c, 0x62, 0x1f, 0x11, 0x5d, + 0x82, 0x3c, 0xfb, 0xb1, 0xb4, 0xc8, 0x1e, 0x8b, 0x0a, 0x7e, 0xe8, 0xb8, 0xeb, 0x0c, 0x63, 0x77, + 0xde, 0x25, 0x5d, 0xaa, 0x2f, 0xb0, 0x47, 0xcb, 0x08, 0xe9, 0x52, 0x7d, 0x01, 0xbb, 0xf3, 0xe8, + 0x1d, 0xc8, 0x9b, 0x64, 0x59, 0x56, 0xed, 0xbd, 0x12, 0x0c, 0xd4, 0xf2, 0x6c, 0xdc, 0x62, 0xd4, + 0x91, 0x67, 0x1b, 0x5f, 0x02, 0x9f, 0xc7, 0x2e, 0x2c, 0xda, 0x86, 0x31, 0xc3, 0x56, 0xe7, 0xcd, + 0x75, 0x93, 0x18, 0xa5, 0x22, 0x93, 0xd1, 0x2f, 0x5a, 0x62, 0x97, 0x3e, 0x2a, 0xc5, 0xf3, 0x90, + 0x47, 0x81, 0x7d, 0x70, 0xf4, 0x03, 0x01, 0x90, 0x69, 0xeb, 0xba, 0x42, 0xba, 0x44, 0xb5, 0x24, + 0x85, 0xbd, 0x14, 0x99, 0xa5, 0x93, 0x4c, 0xe6, 0xff, 0xf7, 0xb3, 0x2b, 0xc6, 0x18, 0x15, 0xee, + 0x3d, 0xc9, 0xc6, 0x49, 0x71, 0x82, 0x5c, 0xea, 0xda, 0x2d, 0x93, 0xfd, 0x5d, 0x1a, 0x1f, 0xc8, + 0xb5, 0xc9, 0x2f, 0x62, 0xbe, 0x6b, 0xf9, 0x3c, 0x76, 0x61, 0xd1, 0x06, 0xcc, 0x18, 0x44, 0x6a, + 0xaf, 0xa9, 0x4a, 0x0f, 0x6b, 0x9a, 0x75, 0x5b, 0x56, 0x88, 0xd9, 0x33, 0x2d, 0xd2, 0x2d, 0x4d, + 0xb0, 0x65, 0xf7, 0x3e, 0x19, 0xc0, 0x89, 0x54, 0x38, 0x85, 0x1b, 0x75, 0xa1, 0xec, 0x86, 0x0c, + 0x7a, 0x9e, 0xbc, 0x98, 0x75, 0xcb, 0x6c, 0x49, 0x8a, 0xf3, 0x4a, 0x3d, 0xc9, 0x04, 0xbc, 0x70, + 0xb0, 0x5f, 0x2e, 0x2f, 0x1e, 0x4e, 0x8a, 0xfb, 0x61, 0xa1, 0xb7, 0xa1, 0x24, 0xa5, 0xc9, 0x39, + 0xc5, 0xe4, 0x9c, 0xe7, 0x86, 0x94, 0x52, 0x85, 0xa4, 0x22, 0xb0, 0x56, 0x3b, 0x7f, 0x78, 0x3d, + 0x9e, 0x6f, 0xdb, 0x86, 0x6b, 0xb5, 0xfb, 0xaa, 0x3d, 0xb1, 0x56, 0x7b, 0x00, 0xf2, 0xf0, 0xa7, + 0x9e, 0xbf, 0x67, 0x60, 0xda, 0x27, 0x1e, 0xb8, 0xd5, 0x9e, 0xc0, 0xf2, 0xd4, 0x5a, 0xed, 0xc9, + 0xbd, 0xea, 0xec, 0xd3, 0xee, 0x55, 0x3f, 0x85, 0x16, 0x3f, 0x6b, 0x7f, 0xfb, 0xae, 0xfb, 0xf7, + 0x6b, 0x7f, 0xfb, 0xba, 0xa5, 0x94, 0x3c, 0xbf, 0xce, 0x04, 0x0d, 0xf8, 0x8f, 0xef, 0xc1, 0x7e, + 0xfd, 0x0f, 0x00, 0xc5, 0xcf, 0xb2, 0x70, 0x2a, 0x7a, 0x1a, 0x43, 0xad, 0x3a, 0xa1, 0x6f, 0xab, + 0xae, 0x0e, 0xa7, 0xb7, 0x6c, 0x45, 0xe9, 0x31, 0x37, 0x04, 0xfa, 0x75, 0xce, 0x53, 0xfb, 0x73, + 0x9c, 0xf3, 0xf4, 0xed, 0x04, 0x1a, 0x9c, 0xc8, 0x99, 0xd2, 0x76, 0xcc, 0x1e, 0xa9, 0xed, 0x18, + 0xeb, 0x82, 0x8d, 0x0c, 0xd1, 0x05, 0x4b, 0x6c, 0x21, 0xe6, 0x8e, 0xd0, 0x42, 0x3c, 0x4a, 0xcf, + 0x2f, 0x21, 0x88, 0xf5, 0xeb, 0xf9, 0x89, 0xcf, 0xc1, 0x2c, 0x67, 0xb3, 0x58, 0x3b, 0x4e, 0xb5, + 0x0c, 0x4d, 0x51, 0x88, 0xb1, 0x68, 0x77, 0xbb, 0x3d, 0xf1, 0x0d, 0x98, 0x08, 0x37, 0x9a, 0x9d, + 0x95, 0x76, 0x7a, 0xdd, 0xbc, 0xe1, 0x11, 0x58, 0x69, 0x67, 0x1c, 0x7b, 0x14, 0xe2, 0xfb, 0x02, + 0xcc, 0x24, 0x7f, 0x50, 0x86, 0x14, 0x98, 0xe8, 0x4a, 0x7b, 0xc1, 0xaf, 0xef, 0x84, 0x23, 0xde, + 0xa5, 0x59, 0x87, 0x71, 0x25, 0x84, 0x85, 0x23, 0xd8, 0xf4, 0x7e, 0x7d, 0x26, 0xa5, 0xb7, 0x77, + 0xbc, 0x9a, 0xa0, 0xfb, 0x50, 0xe8, 0x4a, 0x7b, 0x0d, 0xdb, 0xe8, 0x90, 0x23, 0xbf, 0x1e, 0xb0, + 0x88, 0xb1, 0xc2, 0x51, 0xb0, 0x87, 0x27, 0x7e, 0x24, 0x40, 0x29, 0xad, 0xd0, 0x44, 0xd7, 0x43, + 0x5d, 0xc8, 0xe7, 0x23, 0x5d, 0xc8, 0xa9, 0x18, 0xdf, 0x53, 0xea, 0x41, 0xfe, 0x4a, 0x80, 0x99, + 0xe4, 0x82, 0x1b, 0xbd, 0x1c, 0xd2, 0xb0, 0x1c, 0xd1, 0x70, 0x32, 0xc2, 0xc5, 0xf5, 0xfb, 0x36, + 0x4c, 0xf0, 0xb2, 0x9c, 0xc3, 0x70, 0xaf, 0x8a, 0x49, 0xb1, 0x92, 0x43, 0xb8, 0x65, 0x28, 0x5b, + 0xaf, 0xf0, 0x18, 0x8e, 0xa0, 0x89, 0xdf, 0xcf, 0x40, 0xae, 0xd1, 0x92, 0x14, 0x72, 0x0c, 0x65, + 0xd6, 0x9b, 0xa1, 0x32, 0xab, 0xdf, 0x97, 0xf2, 0x4c, 0xab, 0xd4, 0x0a, 0x0b, 0x47, 0x2a, 0xac, + 0x17, 0x07, 0x42, 0x3b, 0xbc, 0xb8, 0xfa, 0x1f, 0x18, 0xf3, 0x84, 0x0e, 0x17, 0xf3, 0xc5, 0x9f, + 0x67, 0xa0, 0x18, 0x10, 0x31, 0x64, 0xc6, 0xd8, 0x0a, 0x65, 0xda, 0x41, 0xfe, 0x67, 0x26, 0x20, + 0xab, 0xe2, 0xe6, 0x56, 0xe7, 0x83, 0x32, 0xff, 0x13, 0xa2, 0x78, 0xca, 0x7d, 0x03, 0x26, 0x2c, + 0xf6, 0x3f, 0x25, 0xde, 0x9b, 0x5b, 0x96, 0xed, 0x45, 0xef, 0x33, 0xc4, 0x66, 0x68, 0x16, 0x47, + 0xa8, 0x67, 0x6f, 0xc2, 0x78, 0x48, 0xd8, 0x50, 0xdf, 0x83, 0xfd, 0x46, 0x80, 0xe7, 0xfb, 0x5e, + 0xd9, 0x50, 0x2d, 0x74, 0x48, 0x2a, 0x91, 0x43, 0x32, 0x97, 0x0e, 0xf0, 0x14, 0xbf, 0x2b, 0x78, + 0x3f, 0x03, 0xa8, 0xb9, 0x2d, 0x1b, 0xed, 0xba, 0x64, 0x58, 0x3d, 0xcc, 0xff, 0x31, 0xe8, 0x18, + 0x0e, 0xcc, 0x75, 0x28, 0xb6, 0x89, 0xd9, 0x32, 0x64, 0xe6, 0x1c, 0x5e, 0x9d, 0x7b, 0xcf, 0x1a, + 0x8b, 0xfe, 0x14, 0x0e, 0xd2, 0xa1, 0xb7, 0xa0, 0xb0, 0xeb, 0xfc, 0xa7, 0x99, 0xdb, 0xe4, 0xea, + 0x57, 0x48, 0xfa, 0xff, 0x9b, 0xe6, 0xef, 0x1f, 0x3e, 0x60, 0x62, 0x0f, 0x4c, 0xfc, 0x50, 0x80, + 0x99, 0xb8, 0x23, 0x16, 0xa9, 0xaa, 0x4f, 0xdf, 0x19, 0xcf, 0xc1, 0x08, 0x43, 0xa7, 0x5e, 0x38, + 0xe9, 0xbc, 0x30, 0x53, 0xc9, 0x98, 0x8d, 0x8a, 0x7f, 0x11, 0x60, 0x36, 0x59, 0xb5, 0x63, 0x28, + 0xdb, 0xef, 0x87, 0xcb, 0xf6, 0x7e, 0xaf, 0x06, 0xc9, 0x7a, 0xa6, 0x94, 0xf0, 0x7f, 0x4e, 0xf4, + 0xf9, 0x31, 0x18, 0xb5, 0x11, 0x36, 0xea, 0xea, 0xd0, 0x46, 0x25, 0x1b, 0x54, 0xbb, 0xf2, 0xe8, + 0xcb, 0xb9, 0x13, 0x9f, 0x7f, 0x39, 0x77, 0xe2, 0x8b, 0x2f, 0xe7, 0x4e, 0x7c, 0xef, 0x60, 0x4e, + 0x78, 0x74, 0x30, 0x27, 0x7c, 0x7e, 0x30, 0x27, 0x7c, 0x71, 0x30, 0x27, 0xfc, 0xed, 0x60, 0x4e, + 0xf8, 0xf1, 0x57, 0x73, 0x27, 0xee, 0xe7, 0x39, 0xee, 0xbf, 0x02, 0x00, 0x00, 0xff, 0xff, 0x6f, + 0x30, 0xd5, 0x91, 0x29, 0x3b, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/extensions/v1beta1/generated.proto b/staging/src/k8s.io/api/extensions/v1beta1/generated.proto index 91510a4a33e1e..0b319f997122a 100644 --- a/staging/src/k8s.io/api/extensions/v1beta1/generated.proto +++ b/staging/src/k8s.io/api/extensions/v1beta1/generated.proto @@ -181,7 +181,7 @@ message DaemonSetStatus { // uses this field as a collision avoidance mechanism when it needs to // create the name for the newest ControllerRevision. // +optional - optional int64 collisionCount = 9; + optional int32 collisionCount = 9; } message DaemonSetUpdateStrategy { @@ -276,6 +276,7 @@ message DeploymentSpec { // The deployment strategy to use to replace existing pods with new ones. // +optional + // +patchStrategy=retainKeys optional DeploymentStrategy strategy = 4; // Minimum number of seconds for which a newly created pod should be ready @@ -344,7 +345,7 @@ message DeploymentStatus { // field as a collision avoidance mechanism when it needs to create the name for the // newest ReplicaSet. // +optional - optional int64 collisionCount = 8; + optional int32 collisionCount = 8; } // DeploymentStrategy describes how to replace existing pods with new ones. @@ -421,6 +422,21 @@ message IDRange { optional int64 max = 2; } +// IPBlock describes a particular CIDR (Ex. "192.168.1.1/24") that is allowed to the pods +// matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should +// not be included within this rule. +message IPBlock { + // CIDR is a string representing the IP Block + // Valid examples are "192.168.1.1/24" + optional string cidr = 1; + + // Except is a slice of CIDRs that should not be included within an IP Block + // Valid examples are "192.168.1.1/24" + // Except values will be rejected if they are outside the CIDR range + // +optional + repeated string except = 2; +} + // Ingress is a collection of rules that allow inbound connections to reach the // endpoints defined by a backend. An Ingress can be configured to give services // externally-reachable urls, load balance traffic, terminate SSL, offer name @@ -602,6 +618,10 @@ message NetworkPolicyPeer { // If present but empty, this selector selects all namespaces. // +optional optional k8s.io.apimachinery.pkg.apis.meta.v1.LabelSelector namespaceSelector = 2; + + // IPBlock defines policy on a particular IPBlock + // +optional + optional IPBlock ipBlock = 3; } message NetworkPolicyPort { diff --git a/staging/src/k8s.io/api/extensions/v1beta1/types.generated.go b/staging/src/k8s.io/api/extensions/v1beta1/types.generated.go index ee6d2082b77dc..37120e62d103e 100644 --- a/staging/src/k8s.io/api/extensions/v1beta1/types.generated.go +++ b/staging/src/k8s.io/api/extensions/v1beta1/types.generated.go @@ -6000,13 +6000,13 @@ func (x *DeploymentStatus) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { } } else { if x.CollisionCount == nil { - x.CollisionCount = new(int64) + x.CollisionCount = new(int32) } yym19 := z.DecBinary() _ = yym19 if false { } else { - *((*int64)(x.CollisionCount)) = int64(r.DecodeInt(64)) + *((*int32)(x.CollisionCount)) = int32(r.DecodeInt(32)) } } default: @@ -6194,13 +6194,13 @@ func (x *DeploymentStatus) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) } } else { if x.CollisionCount == nil { - x.CollisionCount = new(int64) + x.CollisionCount = new(int32) } yym36 := z.DecBinary() _ = yym36 if false { } else { - *((*int64)(x.CollisionCount)) = int64(r.DecodeInt(64)) + *((*int32)(x.CollisionCount)) = int32(r.DecodeInt(32)) } } for { @@ -8424,13 +8424,13 @@ func (x *DaemonSetStatus) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { } } else { if x.CollisionCount == nil { - x.CollisionCount = new(int64) + x.CollisionCount = new(int32) } yym21 := z.DecBinary() _ = yym21 if false { } else { - *((*int64)(x.CollisionCount)) = int64(r.DecodeInt(64)) + *((*int32)(x.CollisionCount)) = int32(r.DecodeInt(32)) } } default: @@ -8640,13 +8640,13 @@ func (x *DaemonSetStatus) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) } } else { if x.CollisionCount == nil { - x.CollisionCount = new(int64) + x.CollisionCount = new(int32) } yym40 := z.DecBinary() _ = yym40 if false { } else { - *((*int64)(x.CollisionCount)) = int64(r.DecodeInt(64)) + *((*int32)(x.CollisionCount)) = int32(r.DecodeInt(32)) } } for { @@ -18741,7 +18741,7 @@ func (x *NetworkPolicyPort) codecDecodeSelfFromArray(l int, d *codec1978.Decoder z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } -func (x *NetworkPolicyPeer) CodecEncodeSelf(e *codec1978.Encoder) { +func (x *IPBlock) CodecEncodeSelf(e *codec1978.Encoder) { var h codecSelfer1234 z, r := codec1978.GenHelperEncoder(e) _, _, _ = h, z, r @@ -18758,11 +18758,254 @@ func (x *NetworkPolicyPeer) CodecEncodeSelf(e *codec1978.Encoder) { var yyq2 [2]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false + yyq2[1] = len(x.Except) != 0 + var yynn2 int + if yyr2 || yy2arr2 { + r.EncodeArrayStart(2) + } else { + yynn2 = 1 + for _, b := range yyq2 { + if b { + yynn2++ + } + } + r.EncodeMapStart(yynn2) + yynn2 = 0 + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + yym4 := z.EncBinary() + _ = yym4 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.CIDR)) + } + } else { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("cidr")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym5 := z.EncBinary() + _ = yym5 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.CIDR)) + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[1] { + if x.Except == nil { + r.EncodeNil() + } else { + yym7 := z.EncBinary() + _ = yym7 + if false { + } else { + z.F.EncSliceStringV(x.Except, false, e) + } + } + } else { + r.EncodeNil() + } + } else { + if yyq2[1] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("except")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.Except == nil { + r.EncodeNil() + } else { + yym8 := z.EncBinary() + _ = yym8 + if false { + } else { + z.F.EncSliceStringV(x.Except, false, e) + } + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + z.EncSendContainerState(codecSelfer_containerMapEnd1234) + } + } + } +} + +func (x *IPBlock) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerMapEnd1234) + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(codecSelferOnlyMapOrArrayEncodeToStructErr1234) + } + } +} + +func (x *IPBlock) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yys3Slc = z.DecScratchBuffer() // default slice to decode into + _ = yys3Slc + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + z.DecSendContainerState(codecSelfer_containerMapKey1234) + yys3Slc = r.DecodeBytes(yys3Slc, true, true) + yys3 := string(yys3Slc) + z.DecSendContainerState(codecSelfer_containerMapValue1234) + switch yys3 { + case "cidr": + if r.TryDecodeAsNil() { + x.CIDR = "" + } else { + yyv4 := &x.CIDR + yym5 := z.DecBinary() + _ = yym5 + if false { + } else { + *((*string)(yyv4)) = r.DecodeString() + } + } + case "except": + if r.TryDecodeAsNil() { + x.Except = nil + } else { + yyv6 := &x.Except + yym7 := z.DecBinary() + _ = yym7 + if false { + } else { + z.F.DecSliceStringX(yyv6, false, d) + } + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + z.DecSendContainerState(codecSelfer_containerMapEnd1234) +} + +func (x *IPBlock) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj8 int + var yyb8 bool + var yyhl8 bool = l >= 0 + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.CIDR = "" + } else { + yyv9 := &x.CIDR + yym10 := z.DecBinary() + _ = yym10 + if false { + } else { + *((*string)(yyv9)) = r.DecodeString() + } + } + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.Except = nil + } else { + yyv11 := &x.Except + yym12 := z.DecBinary() + _ = yym12 + if false { + } else { + z.F.DecSliceStringX(yyv11, false, d) + } + } + for { + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + break + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + z.DecStructFieldNotFound(yyj8-1, "") + } + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) +} + +func (x *NetworkPolicyPeer) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + var yyq2 [3]bool + _, _, _ = yysep2, yyq2, yy2arr2 + const yyr2 bool = false yyq2[0] = x.PodSelector != nil yyq2[1] = x.NamespaceSelector != nil + yyq2[2] = x.IPBlock != nil var yynn2 int if yyr2 || yy2arr2 { - r.EncodeArrayStart(2) + r.EncodeArrayStart(3) } else { yynn2 = 0 for _, b := range yyq2 { @@ -18843,6 +19086,29 @@ func (x *NetworkPolicyPeer) CodecEncodeSelf(e *codec1978.Encoder) { } } } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[2] { + if x.IPBlock == nil { + r.EncodeNil() + } else { + x.IPBlock.CodecEncodeSelf(e) + } + } else { + r.EncodeNil() + } + } else { + if yyq2[2] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("ipBlock")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.IPBlock == nil { + r.EncodeNil() + } else { + x.IPBlock.CodecEncodeSelf(e) + } + } + } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayEnd1234) } else { @@ -18938,6 +19204,17 @@ func (x *NetworkPolicyPeer) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) z.DecFallback(x.NamespaceSelector, false) } } + case "ipBlock": + if r.TryDecodeAsNil() { + if x.IPBlock != nil { + x.IPBlock = nil + } + } else { + if x.IPBlock == nil { + x.IPBlock = new(IPBlock) + } + x.IPBlock.CodecDecodeSelf(d) + } default: z.DecStructFieldNotFound(-1, yys3) } // end switch yys3 @@ -18949,16 +19226,16 @@ func (x *NetworkPolicyPeer) codecDecodeSelfFromArray(l int, d *codec1978.Decoder var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj8 int - var yyb8 bool - var yyhl8 bool = l >= 0 - yyj8++ - if yyhl8 { - yyb8 = yyj8 > l + var yyj9 int + var yyb9 bool + var yyhl9 bool = l >= 0 + yyj9++ + if yyhl9 { + yyb9 = yyj9 > l } else { - yyb8 = r.CheckBreak() + yyb9 = r.CheckBreak() } - if yyb8 { + if yyb9 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -18971,21 +19248,21 @@ func (x *NetworkPolicyPeer) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if x.PodSelector == nil { x.PodSelector = new(pkg1_v1.LabelSelector) } - yym10 := z.DecBinary() - _ = yym10 + yym11 := z.DecBinary() + _ = yym11 if false { } else if z.HasExtensions() && z.DecExt(x.PodSelector) { } else { z.DecFallback(x.PodSelector, false) } } - yyj8++ - if yyhl8 { - yyb8 = yyj8 > l + yyj9++ + if yyhl9 { + yyb9 = yyj9 > l } else { - yyb8 = r.CheckBreak() + yyb9 = r.CheckBreak() } - if yyb8 { + if yyb9 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -18998,26 +19275,47 @@ func (x *NetworkPolicyPeer) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if x.NamespaceSelector == nil { x.NamespaceSelector = new(pkg1_v1.LabelSelector) } - yym12 := z.DecBinary() - _ = yym12 + yym13 := z.DecBinary() + _ = yym13 if false { } else if z.HasExtensions() && z.DecExt(x.NamespaceSelector) { } else { z.DecFallback(x.NamespaceSelector, false) } } + yyj9++ + if yyhl9 { + yyb9 = yyj9 > l + } else { + yyb9 = r.CheckBreak() + } + if yyb9 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + if x.IPBlock != nil { + x.IPBlock = nil + } + } else { + if x.IPBlock == nil { + x.IPBlock = new(IPBlock) + } + x.IPBlock.CodecDecodeSelf(d) + } for { - yyj8++ - if yyhl8 { - yyb8 = yyj8 > l + yyj9++ + if yyhl9 { + yyb9 = yyj9 > l } else { - yyb8 = r.CheckBreak() + yyb9 = r.CheckBreak() } - if yyb8 { + if yyb9 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj8-1, "") + z.DecStructFieldNotFound(yyj9-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } @@ -21917,7 +22215,7 @@ func (x codecSelfer1234) decSliceNetworkPolicyPeer(v *[]NetworkPolicyPeer, d *co yyrg1 := len(yyv1) > 0 yyv21 := yyv1 - yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 16) + yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 24) if yyrt1 { if yyrl1 <= cap(yyv1) { yyv1 = yyv1[:yyrl1] diff --git a/staging/src/k8s.io/api/extensions/v1beta1/types.go b/staging/src/k8s.io/api/extensions/v1beta1/types.go index 98bd22488c5bb..971db945bd0e5 100644 --- a/staging/src/k8s.io/api/extensions/v1beta1/types.go +++ b/staging/src/k8s.io/api/extensions/v1beta1/types.go @@ -193,7 +193,8 @@ type DeploymentSpec struct { // The deployment strategy to use to replace existing pods with new ones. // +optional - Strategy DeploymentStrategy `json:"strategy,omitempty" protobuf:"bytes,4,opt,name=strategy"` + // +patchStrategy=retainKeys + Strategy DeploymentStrategy `json:"strategy,omitempty" patchStrategy:"retainKeys" protobuf:"bytes,4,opt,name=strategy"` // Minimum number of seconds for which a newly created pod should be ready // without any of its container crashing, for it to be considered available. @@ -345,7 +346,7 @@ type DeploymentStatus struct { // field as a collision avoidance mechanism when it needs to create the name for the // newest ReplicaSet. // +optional - CollisionCount *int64 `json:"collisionCount,omitempty" protobuf:"varint,8,opt,name=collisionCount"` + CollisionCount *int32 `json:"collisionCount,omitempty" protobuf:"varint,8,opt,name=collisionCount"` } type DeploymentConditionType string @@ -524,7 +525,7 @@ type DaemonSetStatus struct { // uses this field as a collision avoidance mechanism when it needs to // create the name for the newest ControllerRevision. // +optional - CollisionCount *int64 `json:"collisionCount,omitempty" protobuf:"varint,9,opt,name=collisionCount"` + CollisionCount *int32 `json:"collisionCount,omitempty" protobuf:"varint,9,opt,name=collisionCount"` } // +genclient @@ -1181,6 +1182,20 @@ type NetworkPolicyPort struct { Port *intstr.IntOrString `json:"port,omitempty" protobuf:"bytes,2,opt,name=port"` } +// IPBlock describes a particular CIDR (Ex. "192.168.1.1/24") that is allowed to the pods +// matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should +// not be included within this rule. +type IPBlock struct { + // CIDR is a string representing the IP Block + // Valid examples are "192.168.1.1/24" + CIDR string `json:"cidr" protobuf:"bytes,1,name=cidr"` + // Except is a slice of CIDRs that should not be included within an IP Block + // Valid examples are "192.168.1.1/24" + // Except values will be rejected if they are outside the CIDR range + // +optional + Except []string `json:"except,omitempty" protobuf:"bytes,2,rep,name=except"` +} + type NetworkPolicyPeer struct { // Exactly one of the following must be specified. @@ -1196,6 +1211,10 @@ type NetworkPolicyPeer struct { // If present but empty, this selector selects all namespaces. // +optional NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty" protobuf:"bytes,2,opt,name=namespaceSelector"` + + // IPBlock defines policy on a particular IPBlock + // +optional + IPBlock *IPBlock `json:"ipBlock,omitempty" protobuf:"bytes,3,rep,name=ipBlock"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/staging/src/k8s.io/api/extensions/v1beta1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/extensions/v1beta1/types_swagger_doc_generated.go index 2a0259548c85a..2944ec44ab263 100644 --- a/staging/src/k8s.io/api/extensions/v1beta1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/extensions/v1beta1/types_swagger_doc_generated.go @@ -254,6 +254,16 @@ func (IDRange) SwaggerDoc() map[string]string { return map_IDRange } +var map_IPBlock = map[string]string{ + "": "IPBlock describes a particular CIDR (Ex. \"192.168.1.1/24\") that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should not be included within this rule.", + "cidr": "CIDR is a string representing the IP Block Valid examples are \"192.168.1.1/24\"", + "except": "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are \"192.168.1.1/24\" Except values will be rejected if they are outside the CIDR range", +} + +func (IPBlock) SwaggerDoc() map[string]string { + return map_IPBlock +} + var map_Ingress = map[string]string{ "": "Ingress is a collection of rules that allow inbound connections to reach the endpoints defined by a backend. An Ingress can be configured to give services externally-reachable urls, load balance traffic, terminate SSL, offer name based virtual hosting etc.", "metadata": "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", @@ -365,6 +375,7 @@ func (NetworkPolicyList) SwaggerDoc() map[string]string { var map_NetworkPolicyPeer = map[string]string{ "podSelector": "This is a label selector which selects Pods in this namespace. This field follows standard label selector semantics. If present but empty, this selector selects all pods in this namespace.", "namespaceSelector": "Selects Namespaces using cluster scoped-labels. This matches all pods in all namespaces selected by this label selector. This field follows standard label selector semantics. If present but empty, this selector selects all namespaces.", + "ipBlock": "IPBlock defines policy on a particular IPBlock", } func (NetworkPolicyPeer) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/extensions/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/extensions/v1beta1/zz_generated.deepcopy.go index 6f2c4aa7e1a60..c0aa22dad1851 100644 --- a/staging/src/k8s.io/api/extensions/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/extensions/v1beta1/zz_generated.deepcopy.go @@ -127,6 +127,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { in.(*IDRange).DeepCopyInto(out.(*IDRange)) return nil }, InType: reflect.TypeOf(&IDRange{})}, + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*IPBlock).DeepCopyInto(out.(*IPBlock)) + return nil + }, InType: reflect.TypeOf(&IPBlock{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*Ingress).DeepCopyInto(out.(*Ingress)) return nil @@ -477,7 +481,7 @@ func (in *DaemonSetStatus) DeepCopyInto(out *DaemonSetStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } @@ -712,7 +716,7 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) { if *in == nil { *out = nil } else { - *out = new(int64) + *out = new(int32) **out = **in } } @@ -845,6 +849,27 @@ func (in *IDRange) DeepCopy() *IDRange { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPBlock) DeepCopyInto(out *IPBlock) { + *out = *in + if in.Except != nil { + in, out := &in.Except, &out.Except + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPBlock. +func (in *IPBlock) DeepCopy() *IPBlock { + if in == nil { + return nil + } + out := new(IPBlock) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Ingress) DeepCopyInto(out *Ingress) { *out = *in @@ -1157,6 +1182,15 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { (*in).DeepCopyInto(*out) } } + if in.IPBlock != nil { + in, out := &in.IPBlock, &out.IPBlock + if *in == nil { + *out = nil + } else { + *out = new(IPBlock) + (*in).DeepCopyInto(*out) + } + } return } diff --git a/staging/src/k8s.io/api/networking/v1/generated.pb.go b/staging/src/k8s.io/api/networking/v1/generated.pb.go index 186b288824bb7..226b55997ce32 100644 --- a/staging/src/k8s.io/api/networking/v1/generated.pb.go +++ b/staging/src/k8s.io/api/networking/v1/generated.pb.go @@ -25,6 +25,7 @@ limitations under the License. k8s.io/kubernetes/vendor/k8s.io/api/networking/v1/generated.proto It has these top-level messages: + IPBlock NetworkPolicy NetworkPolicyIngressRule NetworkPolicyList @@ -60,33 +61,38 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +func (m *IPBlock) Reset() { *m = IPBlock{} } +func (*IPBlock) ProtoMessage() {} +func (*IPBlock) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{0} } + func (m *NetworkPolicy) Reset() { *m = NetworkPolicy{} } func (*NetworkPolicy) ProtoMessage() {} -func (*NetworkPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{0} } +func (*NetworkPolicy) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{1} } func (m *NetworkPolicyIngressRule) Reset() { *m = NetworkPolicyIngressRule{} } func (*NetworkPolicyIngressRule) ProtoMessage() {} func (*NetworkPolicyIngressRule) Descriptor() ([]byte, []int) { - return fileDescriptorGenerated, []int{1} + return fileDescriptorGenerated, []int{2} } func (m *NetworkPolicyList) Reset() { *m = NetworkPolicyList{} } func (*NetworkPolicyList) ProtoMessage() {} -func (*NetworkPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{2} } +func (*NetworkPolicyList) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{3} } func (m *NetworkPolicyPeer) Reset() { *m = NetworkPolicyPeer{} } func (*NetworkPolicyPeer) ProtoMessage() {} -func (*NetworkPolicyPeer) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{3} } +func (*NetworkPolicyPeer) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{4} } func (m *NetworkPolicyPort) Reset() { *m = NetworkPolicyPort{} } func (*NetworkPolicyPort) ProtoMessage() {} -func (*NetworkPolicyPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{4} } +func (*NetworkPolicyPort) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{5} } func (m *NetworkPolicySpec) Reset() { *m = NetworkPolicySpec{} } func (*NetworkPolicySpec) ProtoMessage() {} -func (*NetworkPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{5} } +func (*NetworkPolicySpec) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{6} } func init() { + proto.RegisterType((*IPBlock)(nil), "k8s.io.api.networking.v1.IPBlock") proto.RegisterType((*NetworkPolicy)(nil), "k8s.io.api.networking.v1.NetworkPolicy") proto.RegisterType((*NetworkPolicyIngressRule)(nil), "k8s.io.api.networking.v1.NetworkPolicyIngressRule") proto.RegisterType((*NetworkPolicyList)(nil), "k8s.io.api.networking.v1.NetworkPolicyList") @@ -94,6 +100,43 @@ func init() { proto.RegisterType((*NetworkPolicyPort)(nil), "k8s.io.api.networking.v1.NetworkPolicyPort") proto.RegisterType((*NetworkPolicySpec)(nil), "k8s.io.api.networking.v1.NetworkPolicySpec") } +func (m *IPBlock) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *IPBlock) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.CIDR))) + i += copy(dAtA[i:], m.CIDR) + if len(m.Except) > 0 { + for _, s := range m.Except { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + func (m *NetworkPolicy) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -243,6 +286,16 @@ func (m *NetworkPolicyPeer) MarshalTo(dAtA []byte) (int, error) { } i += n5 } + if m.IPBlock != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.IPBlock.Size())) + n6, err := m.IPBlock.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n6 + } return i, nil } @@ -271,11 +324,11 @@ func (m *NetworkPolicyPort) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintGenerated(dAtA, i, uint64(m.Port.Size())) - n6, err := m.Port.MarshalTo(dAtA[i:]) + n7, err := m.Port.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n6 + i += n7 } return i, nil } @@ -298,11 +351,11 @@ func (m *NetworkPolicySpec) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintGenerated(dAtA, i, uint64(m.PodSelector.Size())) - n7, err := m.PodSelector.MarshalTo(dAtA[i:]) + n8, err := m.PodSelector.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n7 + i += n8 if len(m.Ingress) > 0 { for _, msg := range m.Ingress { dAtA[i] = 0x12 @@ -345,6 +398,20 @@ func encodeVarintGenerated(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } +func (m *IPBlock) Size() (n int) { + var l int + _ = l + l = len(m.CIDR) + n += 1 + l + sovGenerated(uint64(l)) + if len(m.Except) > 0 { + for _, s := range m.Except { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } + return n +} + func (m *NetworkPolicy) Size() (n int) { var l int _ = l @@ -398,6 +465,10 @@ func (m *NetworkPolicyPeer) Size() (n int) { l = m.NamespaceSelector.Size() n += 1 + l + sovGenerated(uint64(l)) } + if m.IPBlock != nil { + l = m.IPBlock.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -442,6 +513,17 @@ func sovGenerated(x uint64) (n int) { func sozGenerated(x uint64) (n int) { return sovGenerated(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } +func (this *IPBlock) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&IPBlock{`, + `CIDR:` + fmt.Sprintf("%v", this.CIDR) + `,`, + `Except:` + fmt.Sprintf("%v", this.Except) + `,`, + `}`, + }, "") + return s +} func (this *NetworkPolicy) String() string { if this == nil { return "nil" @@ -482,6 +564,7 @@ func (this *NetworkPolicyPeer) String() string { s := strings.Join([]string{`&NetworkPolicyPeer{`, `PodSelector:` + strings.Replace(fmt.Sprintf("%v", this.PodSelector), "LabelSelector", "k8s_io_apimachinery_pkg_apis_meta_v1.LabelSelector", 1) + `,`, `NamespaceSelector:` + strings.Replace(fmt.Sprintf("%v", this.NamespaceSelector), "LabelSelector", "k8s_io_apimachinery_pkg_apis_meta_v1.LabelSelector", 1) + `,`, + `IPBlock:` + strings.Replace(fmt.Sprintf("%v", this.IPBlock), "IPBlock", "IPBlock", 1) + `,`, `}`, }, "") return s @@ -516,6 +599,114 @@ func valueToStringGenerated(v interface{}) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("*%v", pv) } +func (m *IPBlock) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: IPBlock: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: IPBlock: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CIDR", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CIDR = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Except", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Except = append(m.Except, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NetworkPolicy) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -944,6 +1135,39 @@ func (m *NetworkPolicyPeer) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field IPBlock", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.IPBlock == nil { + m.IPBlock = &IPBlock{} + } + if err := m.IPBlock.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -1299,47 +1523,52 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 670 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0x4d, 0x4f, 0xdb, 0x4a, - 0x14, 0x8d, 0xf3, 0x40, 0xe4, 0x0d, 0x0f, 0xf1, 0xf0, 0xd3, 0x93, 0x22, 0x2a, 0x39, 0xc8, 0x9b, - 0x52, 0x21, 0x66, 0x1a, 0xa8, 0xaa, 0x6e, 0xeb, 0x45, 0x2b, 0x24, 0x3e, 0x22, 0xb3, 0xab, 0x5a, - 0xa9, 0x13, 0xe7, 0x62, 0x86, 0xc4, 0x33, 0xd6, 0xcc, 0x24, 0x85, 0x5d, 0x7f, 0x42, 0x7f, 0x48, - 0xff, 0x41, 0x17, 0xed, 0x92, 0x25, 0x4b, 0x56, 0x51, 0x71, 0x7f, 0x43, 0x37, 0x5d, 0x55, 0x63, - 0x3b, 0x71, 0x88, 0x89, 0x1a, 0xaa, 0xee, 0x98, 0xeb, 0x73, 0xce, 0xbd, 0x87, 0x73, 0x6f, 0xd0, - 0xf3, 0xee, 0x33, 0x85, 0x99, 0x20, 0xdd, 0x7e, 0x1b, 0x24, 0x07, 0x0d, 0x8a, 0x0c, 0x80, 0x77, - 0x84, 0x24, 0xf9, 0x07, 0x1a, 0x33, 0xc2, 0x41, 0xbf, 0x13, 0xb2, 0xcb, 0x78, 0x48, 0x06, 0x4d, - 0x12, 0x02, 0x07, 0x49, 0x35, 0x74, 0x70, 0x2c, 0x85, 0x16, 0x76, 0x3d, 0x43, 0x62, 0x1a, 0x33, - 0x5c, 0x20, 0xf1, 0xa0, 0xb9, 0xbe, 0x1d, 0x32, 0x7d, 0xda, 0x6f, 0xe3, 0x40, 0x44, 0x24, 0x14, - 0xa1, 0x20, 0x29, 0xa1, 0xdd, 0x3f, 0x49, 0x5f, 0xe9, 0x23, 0xfd, 0x2b, 0x13, 0x5a, 0x77, 0x27, - 0x5a, 0x06, 0x42, 0xc2, 0x1d, 0xcd, 0xd6, 0xb7, 0x27, 0x30, 0x70, 0xae, 0x81, 0x2b, 0x26, 0xb8, - 0x22, 0x83, 0x66, 0x1b, 0x34, 0x2d, 0xc3, 0x1f, 0x4d, 0xc0, 0x63, 0xd1, 0x63, 0xc1, 0xc5, 0x4c, - 0xe8, 0x93, 0x02, 0x1a, 0xd1, 0xe0, 0x94, 0x71, 0x90, 0x17, 0x24, 0xee, 0x86, 0xa6, 0xa0, 0x48, - 0x04, 0x9a, 0xde, 0x35, 0x0f, 0x99, 0xc5, 0x92, 0x7d, 0xae, 0x59, 0x04, 0x25, 0xc2, 0xd3, 0x5f, - 0x11, 0x54, 0x70, 0x0a, 0x11, 0x2d, 0xf1, 0x76, 0x67, 0xf1, 0xfa, 0x9a, 0xf5, 0x08, 0xe3, 0x5a, - 0x69, 0x39, 0x4d, 0x72, 0xbf, 0x58, 0x68, 0xe5, 0x30, 0x8b, 0xa4, 0x95, 0xba, 0xb7, 0xdf, 0xa2, - 0x9a, 0xb1, 0xd2, 0xa1, 0x9a, 0xd6, 0xad, 0x0d, 0x6b, 0x73, 0x79, 0xe7, 0x31, 0x2e, 0xf2, 0x1b, - 0x2b, 0xe3, 0xb8, 0x1b, 0x9a, 0x82, 0xc2, 0x06, 0x8d, 0x07, 0x4d, 0x7c, 0xd4, 0x3e, 0x83, 0x40, - 0x1f, 0x80, 0xa6, 0x9e, 0x7d, 0x39, 0x6c, 0x54, 0x92, 0x61, 0x03, 0x15, 0x35, 0x7f, 0xac, 0x6a, - 0x1f, 0xa0, 0x05, 0x15, 0x43, 0x50, 0xaf, 0xa6, 0xea, 0x5b, 0x78, 0xd6, 0x76, 0xe0, 0x5b, 0x83, - 0x1d, 0xc7, 0x10, 0x78, 0xff, 0xe4, 0xc2, 0x0b, 0xe6, 0xe5, 0xa7, 0x32, 0xee, 0x27, 0x0b, 0xd5, - 0x6f, 0x21, 0xf7, 0x78, 0x28, 0x41, 0x29, 0xbf, 0xdf, 0x03, 0xbb, 0x85, 0x16, 0x63, 0x21, 0xb5, - 0xaa, 0x5b, 0x1b, 0x7f, 0xdd, 0xa3, 0x59, 0x4b, 0x48, 0xed, 0xad, 0xe4, 0xcd, 0x16, 0xcd, 0x4b, - 0xf9, 0x99, 0x90, 0x99, 0xfe, 0x44, 0x8a, 0xa8, 0x5e, 0xbd, 0x9f, 0x20, 0x80, 0x2c, 0xa6, 0x7f, - 0x21, 0x45, 0xe4, 0xa7, 0x32, 0xee, 0x67, 0x0b, 0xad, 0xdd, 0x42, 0xee, 0x33, 0xa5, 0xed, 0xd7, - 0xa5, 0x10, 0xf0, 0x7c, 0x21, 0x18, 0x76, 0x1a, 0xc1, 0xbf, 0x79, 0xaf, 0xda, 0xa8, 0x32, 0x11, - 0xc0, 0x3e, 0x5a, 0x64, 0x1a, 0x22, 0x95, 0x7b, 0x78, 0x38, 0xa7, 0x87, 0xe2, 0x1f, 0xb2, 0x67, - 0xd8, 0x7e, 0x26, 0xe2, 0x7e, 0x9f, 0x76, 0x60, 0xbc, 0xda, 0x27, 0x68, 0x39, 0x16, 0x9d, 0x63, - 0xe8, 0x41, 0xa0, 0x85, 0xcc, 0x4d, 0xec, 0xce, 0x69, 0x82, 0xb6, 0xa1, 0x37, 0xa2, 0x7a, 0xab, - 0xc9, 0xb0, 0xb1, 0xdc, 0x2a, 0xb4, 0xfc, 0x49, 0x61, 0xfb, 0x1c, 0xad, 0x71, 0x1a, 0x81, 0x8a, - 0x69, 0x00, 0xe3, 0x6e, 0xd5, 0xdf, 0xef, 0xf6, 0x7f, 0x32, 0x6c, 0xac, 0x1d, 0x4e, 0x2b, 0xfa, - 0xe5, 0x26, 0xee, 0xc7, 0x92, 0x6f, 0x21, 0xb5, 0xfd, 0x12, 0xd5, 0xd2, 0xcb, 0x0a, 0x44, 0x2f, - 0x35, 0xfd, 0xb7, 0xb7, 0x65, 0x52, 0x68, 0xe5, 0xb5, 0x1f, 0xc3, 0xc6, 0x83, 0xf2, 0x8f, 0x18, - 0x1e, 0x7d, 0xf6, 0xc7, 0x64, 0xfb, 0x10, 0x2d, 0x98, 0x85, 0xcb, 0xbd, 0xcc, 0xbe, 0x41, 0x73, - 0xdd, 0x38, 0xbb, 0x6e, 0xbc, 0xc7, 0xf5, 0x91, 0x3c, 0xd6, 0x92, 0xf1, 0xd0, 0xab, 0x99, 0x45, - 0x33, 0x23, 0xf9, 0xa9, 0x8e, 0x3b, 0x9c, 0x1e, 0xd7, 0x9c, 0x90, 0x7d, 0xf6, 0xc7, 0x62, 0xfa, - 0x2f, 0x5f, 0x8e, 0xd9, 0x51, 0xbd, 0x41, 0x4b, 0x2c, 0x3b, 0xcd, 0x7c, 0xf1, 0x76, 0xe6, 0x5c, - 0xbc, 0x89, 0x83, 0xf6, 0x56, 0xf3, 0x36, 0x4b, 0xa3, 0xe2, 0x48, 0xd3, 0xdb, 0xbc, 0xbc, 0x71, - 0x2a, 0x57, 0x37, 0x4e, 0xe5, 0xfa, 0xc6, 0xa9, 0xbc, 0x4f, 0x1c, 0xeb, 0x32, 0x71, 0xac, 0xab, - 0xc4, 0xb1, 0xae, 0x13, 0xc7, 0xfa, 0x9a, 0x38, 0xd6, 0x87, 0x6f, 0x4e, 0xe5, 0x55, 0x75, 0xd0, - 0xfc, 0x19, 0x00, 0x00, 0xff, 0xff, 0x75, 0x6b, 0xae, 0xb3, 0xd3, 0x06, 0x00, 0x00, + // 745 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcd, 0x4e, 0xdb, 0x4a, + 0x14, 0x8e, 0x43, 0x20, 0x61, 0x72, 0x11, 0x17, 0x5f, 0x5d, 0x29, 0xa2, 0x92, 0x93, 0x7a, 0x53, + 0x2a, 0xc4, 0xb8, 0x81, 0xaa, 0xea, 0xb6, 0xee, 0x6f, 0x24, 0x7e, 0x22, 0xb3, 0xab, 0x5a, 0xa9, + 0x8e, 0x73, 0x30, 0x43, 0x62, 0x8f, 0x35, 0x33, 0x49, 0x61, 0xd7, 0x47, 0xe8, 0x83, 0xf4, 0x0d, + 0xba, 0xa0, 0x4b, 0x96, 0x2c, 0x59, 0x59, 0xc5, 0x7d, 0x8b, 0xae, 0xaa, 0x99, 0x38, 0x71, 0x48, + 0x88, 0x1a, 0xaa, 0xee, 0x66, 0xce, 0x9c, 0xef, 0xfb, 0xce, 0xef, 0xa0, 0x67, 0x9d, 0xa7, 0x1c, + 0x13, 0x6a, 0x75, 0x7a, 0x2d, 0x60, 0x21, 0x08, 0xe0, 0x56, 0x1f, 0xc2, 0x36, 0x65, 0x56, 0xfa, + 0xe0, 0x46, 0xc4, 0x0a, 0x41, 0x7c, 0xa4, 0xac, 0x43, 0x42, 0xdf, 0xea, 0xd7, 0x2d, 0x1f, 0x42, + 0x60, 0xae, 0x80, 0x36, 0x8e, 0x18, 0x15, 0x54, 0xaf, 0x0c, 0x3c, 0xb1, 0x1b, 0x11, 0x9c, 0x79, + 0xe2, 0x7e, 0x7d, 0x7d, 0xcb, 0x27, 0xe2, 0xb8, 0xd7, 0xc2, 0x1e, 0x0d, 0x2c, 0x9f, 0xfa, 0xd4, + 0x52, 0x80, 0x56, 0xef, 0x48, 0xdd, 0xd4, 0x45, 0x9d, 0x06, 0x44, 0xeb, 0xe6, 0x98, 0xa4, 0x47, + 0x19, 0xdc, 0x22, 0xb6, 0xbe, 0x35, 0xe6, 0x03, 0xa7, 0x02, 0x42, 0x4e, 0x68, 0xc8, 0xad, 0x7e, + 0xbd, 0x05, 0xc2, 0x9d, 0x76, 0x7f, 0x38, 0xe6, 0x1e, 0xd1, 0x2e, 0xf1, 0xce, 0x66, 0xba, 0x3e, + 0xce, 0x5c, 0x03, 0xd7, 0x3b, 0x26, 0x21, 0xb0, 0x33, 0x2b, 0xea, 0xf8, 0xd2, 0xc0, 0xad, 0x00, + 0x84, 0x7b, 0x5b, 0x3c, 0xd6, 0x2c, 0x14, 0xeb, 0x85, 0x82, 0x04, 0x30, 0x05, 0x78, 0xf2, 0x3b, + 0x00, 0xf7, 0x8e, 0x21, 0x70, 0xa7, 0x70, 0x3b, 0xb3, 0x70, 0x3d, 0x41, 0xba, 0x16, 0x09, 0x05, + 0x17, 0x6c, 0x12, 0x64, 0x1e, 0xa0, 0x62, 0xa3, 0x69, 0x77, 0xa9, 0xd7, 0xd1, 0x6b, 0xa8, 0xe0, + 0x91, 0x36, 0xab, 0x68, 0x35, 0x6d, 0x63, 0xd9, 0xfe, 0xe7, 0x22, 0xae, 0xe6, 0x92, 0xb8, 0x5a, + 0x78, 0xde, 0x78, 0xe1, 0x38, 0xea, 0x45, 0x37, 0xd1, 0x12, 0x9c, 0x7a, 0x10, 0x89, 0x4a, 0xbe, + 0xb6, 0xb0, 0xb1, 0x6c, 0xa3, 0x24, 0xae, 0x2e, 0xbd, 0x54, 0x16, 0x27, 0x7d, 0x31, 0xbf, 0x69, + 0x68, 0x65, 0x7f, 0xd0, 0xe3, 0xa6, 0x2a, 0xa7, 0xfe, 0x01, 0x95, 0x64, 0x6d, 0xda, 0xae, 0x70, + 0x15, 0x77, 0x79, 0xfb, 0x11, 0xce, 0x06, 0x62, 0x14, 0x2a, 0x8e, 0x3a, 0xbe, 0x34, 0x70, 0x2c, + 0xbd, 0x71, 0xbf, 0x8e, 0x0f, 0x5a, 0x27, 0xe0, 0x89, 0x3d, 0x10, 0xae, 0xad, 0xa7, 0xd1, 0xa0, + 0xcc, 0xe6, 0x8c, 0x58, 0xf5, 0x3d, 0x54, 0xe0, 0x11, 0x78, 0x95, 0xbc, 0x62, 0xdf, 0xc4, 0xb3, + 0xc6, 0x0d, 0xdf, 0x08, 0xec, 0x30, 0x02, 0x2f, 0x4b, 0x53, 0xde, 0x1c, 0x45, 0x63, 0x7e, 0xd5, + 0x50, 0xe5, 0x86, 0x67, 0x23, 0xf4, 0x19, 0x70, 0xee, 0xf4, 0xba, 0xa0, 0x37, 0xd1, 0x62, 0x44, + 0x99, 0xe0, 0x15, 0xad, 0xb6, 0x70, 0x07, 0xb1, 0x26, 0x65, 0xc2, 0x5e, 0x49, 0xc5, 0x16, 0xe5, + 0x8d, 0x3b, 0x03, 0x22, 0x19, 0xfd, 0x11, 0xa3, 0x81, 0xaa, 0xe9, 0x1d, 0x08, 0x01, 0x58, 0x16, + 0xfd, 0x2b, 0x46, 0x03, 0x47, 0xd1, 0x98, 0xe7, 0x1a, 0x5a, 0xbb, 0xe1, 0xb9, 0x4b, 0xb8, 0xd0, + 0xdf, 0x4d, 0x35, 0x01, 0xcf, 0xd7, 0x04, 0x89, 0x56, 0x2d, 0xf8, 0x37, 0xd5, 0x2a, 0x0d, 0x2d, + 0x63, 0x0d, 0xd8, 0x45, 0x8b, 0x44, 0x40, 0xc0, 0xd3, 0x1c, 0x1e, 0xcc, 0x99, 0x43, 0x56, 0x90, + 0x86, 0x44, 0x3b, 0x03, 0x12, 0xf3, 0x3c, 0x3f, 0x91, 0x81, 0xcc, 0x55, 0x3f, 0x42, 0xe5, 0x88, + 0xb6, 0x0f, 0xa1, 0x0b, 0x9e, 0xa0, 0x2c, 0x4d, 0x62, 0x67, 0xce, 0x24, 0xdc, 0x16, 0x74, 0x87, + 0x50, 0x7b, 0x35, 0x89, 0xab, 0xe5, 0x66, 0xc6, 0xe5, 0x8c, 0x13, 0xeb, 0xa7, 0x68, 0x2d, 0x74, + 0x03, 0xe0, 0x91, 0xeb, 0xc1, 0x48, 0x2d, 0xff, 0xe7, 0x6a, 0xff, 0x27, 0x71, 0x75, 0x6d, 0x7f, + 0x92, 0xd1, 0x99, 0x16, 0xd1, 0xdf, 0xa0, 0x22, 0x89, 0xd4, 0x2e, 0x56, 0x16, 0x94, 0xde, 0xfd, + 0xd9, 0x75, 0x4c, 0x97, 0xd6, 0x2e, 0x27, 0x71, 0x75, 0xb8, 0xc1, 0xce, 0x10, 0x6e, 0x7e, 0x99, + 0x9c, 0x01, 0x39, 0x70, 0xfa, 0x6b, 0x54, 0x52, 0x4b, 0xef, 0xd1, 0x6e, 0xba, 0xe4, 0x9b, 0xb2, + 0x9f, 0xcd, 0xd4, 0xf6, 0x33, 0xae, 0xde, 0x9b, 0xfe, 0x5f, 0xf1, 0xf0, 0xd9, 0x19, 0x81, 0xf5, + 0x7d, 0x54, 0x90, 0xa3, 0x9b, 0x56, 0x65, 0xf6, 0x36, 0xcb, 0x8f, 0x07, 0x0f, 0x3e, 0x1e, 0xdc, + 0x08, 0xc5, 0x01, 0x3b, 0x14, 0x8c, 0x84, 0xbe, 0x5d, 0x92, 0x23, 0x2b, 0x43, 0x72, 0x14, 0x8f, + 0x19, 0x4f, 0x86, 0x2b, 0x97, 0x51, 0x3f, 0xf9, 0x6b, 0x0d, 0xff, 0x2f, 0x1d, 0xb3, 0xd9, 0x4d, + 0x7f, 0x8f, 0x8a, 0x64, 0xb0, 0xe4, 0xe9, 0x08, 0x6f, 0xcf, 0x39, 0xc2, 0x63, 0x5f, 0x83, 0xbd, + 0x9a, 0xca, 0x14, 0x87, 0xc6, 0x21, 0xa7, 0xbd, 0x71, 0x71, 0x6d, 0xe4, 0x2e, 0xaf, 0x8d, 0xdc, + 0xd5, 0xb5, 0x91, 0xfb, 0x94, 0x18, 0xda, 0x45, 0x62, 0x68, 0x97, 0x89, 0xa1, 0x5d, 0x25, 0x86, + 0xf6, 0x3d, 0x31, 0xb4, 0xcf, 0x3f, 0x8c, 0xdc, 0xdb, 0x7c, 0xbf, 0xfe, 0x2b, 0x00, 0x00, 0xff, + 0xff, 0x3b, 0x3f, 0x8c, 0x26, 0x6e, 0x07, 0x00, 0x00, } diff --git a/staging/src/k8s.io/api/networking/v1/generated.proto b/staging/src/k8s.io/api/networking/v1/generated.proto index 0c0e5bc6af128..2dee2c9f78b4b 100644 --- a/staging/src/k8s.io/api/networking/v1/generated.proto +++ b/staging/src/k8s.io/api/networking/v1/generated.proto @@ -32,6 +32,21 @@ import "k8s.io/apimachinery/pkg/util/intstr/generated.proto"; // Package-wide variables from generator "generated". option go_package = "v1"; +// IPBlock describes a particular CIDR (Ex. "192.168.1.1/24") that is allowed to the pods +// matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should +// not be included within this rule. +message IPBlock { + // CIDR is a string representing the IP Block + // Valid examples are "192.168.1.1/24" + optional string cidr = 1; + + // Except is a slice of CIDRs that should not be included within an IP Block + // Valid examples are "192.168.1.1/24" + // Except values will be rejected if they are outside the CIDR range + // +optional + repeated string except = 2; +} + // NetworkPolicy describes what network traffic is allowed for a set of Pods message NetworkPolicy { // Standard object's metadata. @@ -89,6 +104,10 @@ message NetworkPolicyPeer { // selector semantics. If present but empty, this selector selects all namespaces. // +optional optional k8s.io.apimachinery.pkg.apis.meta.v1.LabelSelector namespaceSelector = 2; + + // IPBlock defines policy on a particular IPBlock + // +optional + optional IPBlock ipBlock = 3; } // NetworkPolicyPort describes a port to allow traffic on diff --git a/staging/src/k8s.io/api/networking/v1/types.generated.go b/staging/src/k8s.io/api/networking/v1/types.generated.go index 6b89e36d7da66..d81d40a22f7ca 100644 --- a/staging/src/k8s.io/api/networking/v1/types.generated.go +++ b/staging/src/k8s.io/api/networking/v1/types.generated.go @@ -1196,7 +1196,7 @@ func (x *NetworkPolicyPort) codecDecodeSelfFromArray(l int, d *codec1978.Decoder z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } -func (x *NetworkPolicyPeer) CodecEncodeSelf(e *codec1978.Encoder) { +func (x *IPBlock) CodecEncodeSelf(e *codec1978.Encoder) { var h codecSelfer1234 z, r := codec1978.GenHelperEncoder(e) _, _, _ = h, z, r @@ -1213,11 +1213,254 @@ func (x *NetworkPolicyPeer) CodecEncodeSelf(e *codec1978.Encoder) { var yyq2 [2]bool _, _, _ = yysep2, yyq2, yy2arr2 const yyr2 bool = false + yyq2[1] = len(x.Except) != 0 + var yynn2 int + if yyr2 || yy2arr2 { + r.EncodeArrayStart(2) + } else { + yynn2 = 1 + for _, b := range yyq2 { + if b { + yynn2++ + } + } + r.EncodeMapStart(yynn2) + yynn2 = 0 + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + yym4 := z.EncBinary() + _ = yym4 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.CIDR)) + } + } else { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("cidr")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + yym5 := z.EncBinary() + _ = yym5 + if false { + } else { + r.EncodeString(codecSelferC_UTF81234, string(x.CIDR)) + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[1] { + if x.Except == nil { + r.EncodeNil() + } else { + yym7 := z.EncBinary() + _ = yym7 + if false { + } else { + z.F.EncSliceStringV(x.Except, false, e) + } + } + } else { + r.EncodeNil() + } + } else { + if yyq2[1] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("except")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.Except == nil { + r.EncodeNil() + } else { + yym8 := z.EncBinary() + _ = yym8 + if false { + } else { + z.F.EncSliceStringV(x.Except, false, e) + } + } + } + } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + z.EncSendContainerState(codecSelfer_containerMapEnd1234) + } + } + } +} + +func (x *IPBlock) CodecDecodeSelf(d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + yym1 := z.DecBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.DecExt(x) { + } else { + yyct2 := r.ContainerType() + if yyct2 == codecSelferValueTypeMap1234 { + yyl2 := r.ReadMapStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerMapEnd1234) + } else { + x.codecDecodeSelfFromMap(yyl2, d) + } + } else if yyct2 == codecSelferValueTypeArray1234 { + yyl2 := r.ReadArrayStart() + if yyl2 == 0 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + } else { + x.codecDecodeSelfFromArray(yyl2, d) + } + } else { + panic(codecSelferOnlyMapOrArrayEncodeToStructErr1234) + } + } +} + +func (x *IPBlock) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yys3Slc = z.DecScratchBuffer() // default slice to decode into + _ = yys3Slc + var yyhl3 bool = l >= 0 + for yyj3 := 0; ; yyj3++ { + if yyhl3 { + if yyj3 >= l { + break + } + } else { + if r.CheckBreak() { + break + } + } + z.DecSendContainerState(codecSelfer_containerMapKey1234) + yys3Slc = r.DecodeBytes(yys3Slc, true, true) + yys3 := string(yys3Slc) + z.DecSendContainerState(codecSelfer_containerMapValue1234) + switch yys3 { + case "cidr": + if r.TryDecodeAsNil() { + x.CIDR = "" + } else { + yyv4 := &x.CIDR + yym5 := z.DecBinary() + _ = yym5 + if false { + } else { + *((*string)(yyv4)) = r.DecodeString() + } + } + case "except": + if r.TryDecodeAsNil() { + x.Except = nil + } else { + yyv6 := &x.Except + yym7 := z.DecBinary() + _ = yym7 + if false { + } else { + z.F.DecSliceStringX(yyv6, false, d) + } + } + default: + z.DecStructFieldNotFound(-1, yys3) + } // end switch yys3 + } // end for yyj3 + z.DecSendContainerState(codecSelfer_containerMapEnd1234) +} + +func (x *IPBlock) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperDecoder(d) + _, _, _ = h, z, r + var yyj8 int + var yyb8 bool + var yyhl8 bool = l >= 0 + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.CIDR = "" + } else { + yyv9 := &x.CIDR + yym10 := z.DecBinary() + _ = yym10 + if false { + } else { + *((*string)(yyv9)) = r.DecodeString() + } + } + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + x.Except = nil + } else { + yyv11 := &x.Except + yym12 := z.DecBinary() + _ = yym12 + if false { + } else { + z.F.DecSliceStringX(yyv11, false, d) + } + } + for { + yyj8++ + if yyhl8 { + yyb8 = yyj8 > l + } else { + yyb8 = r.CheckBreak() + } + if yyb8 { + break + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + z.DecStructFieldNotFound(yyj8-1, "") + } + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) +} + +func (x *NetworkPolicyPeer) CodecEncodeSelf(e *codec1978.Encoder) { + var h codecSelfer1234 + z, r := codec1978.GenHelperEncoder(e) + _, _, _ = h, z, r + if x == nil { + r.EncodeNil() + } else { + yym1 := z.EncBinary() + _ = yym1 + if false { + } else if z.HasExtensions() && z.EncExt(x) { + } else { + yysep2 := !z.EncBinary() + yy2arr2 := z.EncBasicHandle().StructToArray + var yyq2 [3]bool + _, _, _ = yysep2, yyq2, yy2arr2 + const yyr2 bool = false yyq2[0] = x.PodSelector != nil yyq2[1] = x.NamespaceSelector != nil + yyq2[2] = x.IPBlock != nil var yynn2 int if yyr2 || yy2arr2 { - r.EncodeArrayStart(2) + r.EncodeArrayStart(3) } else { yynn2 = 0 for _, b := range yyq2 { @@ -1298,6 +1541,29 @@ func (x *NetworkPolicyPeer) CodecEncodeSelf(e *codec1978.Encoder) { } } } + if yyr2 || yy2arr2 { + z.EncSendContainerState(codecSelfer_containerArrayElem1234) + if yyq2[2] { + if x.IPBlock == nil { + r.EncodeNil() + } else { + x.IPBlock.CodecEncodeSelf(e) + } + } else { + r.EncodeNil() + } + } else { + if yyq2[2] { + z.EncSendContainerState(codecSelfer_containerMapKey1234) + r.EncodeString(codecSelferC_UTF81234, string("ipBlock")) + z.EncSendContainerState(codecSelfer_containerMapValue1234) + if x.IPBlock == nil { + r.EncodeNil() + } else { + x.IPBlock.CodecEncodeSelf(e) + } + } + } if yyr2 || yy2arr2 { z.EncSendContainerState(codecSelfer_containerArrayEnd1234) } else { @@ -1393,6 +1659,17 @@ func (x *NetworkPolicyPeer) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) z.DecFallback(x.NamespaceSelector, false) } } + case "ipBlock": + if r.TryDecodeAsNil() { + if x.IPBlock != nil { + x.IPBlock = nil + } + } else { + if x.IPBlock == nil { + x.IPBlock = new(IPBlock) + } + x.IPBlock.CodecDecodeSelf(d) + } default: z.DecStructFieldNotFound(-1, yys3) } // end switch yys3 @@ -1404,16 +1681,16 @@ func (x *NetworkPolicyPeer) codecDecodeSelfFromArray(l int, d *codec1978.Decoder var h codecSelfer1234 z, r := codec1978.GenHelperDecoder(d) _, _, _ = h, z, r - var yyj8 int - var yyb8 bool - var yyhl8 bool = l >= 0 - yyj8++ - if yyhl8 { - yyb8 = yyj8 > l + var yyj9 int + var yyb9 bool + var yyhl9 bool = l >= 0 + yyj9++ + if yyhl9 { + yyb9 = yyj9 > l } else { - yyb8 = r.CheckBreak() + yyb9 = r.CheckBreak() } - if yyb8 { + if yyb9 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1426,21 +1703,21 @@ func (x *NetworkPolicyPeer) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if x.PodSelector == nil { x.PodSelector = new(pkg1_v1.LabelSelector) } - yym10 := z.DecBinary() - _ = yym10 + yym11 := z.DecBinary() + _ = yym11 if false { } else if z.HasExtensions() && z.DecExt(x.PodSelector) { } else { z.DecFallback(x.PodSelector, false) } } - yyj8++ - if yyhl8 { - yyb8 = yyj8 > l + yyj9++ + if yyhl9 { + yyb9 = yyj9 > l } else { - yyb8 = r.CheckBreak() + yyb9 = r.CheckBreak() } - if yyb8 { + if yyb9 { z.DecSendContainerState(codecSelfer_containerArrayEnd1234) return } @@ -1453,26 +1730,47 @@ func (x *NetworkPolicyPeer) codecDecodeSelfFromArray(l int, d *codec1978.Decoder if x.NamespaceSelector == nil { x.NamespaceSelector = new(pkg1_v1.LabelSelector) } - yym12 := z.DecBinary() - _ = yym12 + yym13 := z.DecBinary() + _ = yym13 if false { } else if z.HasExtensions() && z.DecExt(x.NamespaceSelector) { } else { z.DecFallback(x.NamespaceSelector, false) } } + yyj9++ + if yyhl9 { + yyb9 = yyj9 > l + } else { + yyb9 = r.CheckBreak() + } + if yyb9 { + z.DecSendContainerState(codecSelfer_containerArrayEnd1234) + return + } + z.DecSendContainerState(codecSelfer_containerArrayElem1234) + if r.TryDecodeAsNil() { + if x.IPBlock != nil { + x.IPBlock = nil + } + } else { + if x.IPBlock == nil { + x.IPBlock = new(IPBlock) + } + x.IPBlock.CodecDecodeSelf(d) + } for { - yyj8++ - if yyhl8 { - yyb8 = yyj8 > l + yyj9++ + if yyhl9 { + yyb9 = yyj9 > l } else { - yyb8 = r.CheckBreak() + yyb9 = r.CheckBreak() } - if yyb8 { + if yyb9 { break } z.DecSendContainerState(codecSelfer_containerArrayElem1234) - z.DecStructFieldNotFound(yyj8-1, "") + z.DecStructFieldNotFound(yyj9-1, "") } z.DecSendContainerState(codecSelfer_containerArrayEnd1234) } @@ -2122,7 +2420,7 @@ func (x codecSelfer1234) decSliceNetworkPolicyPeer(v *[]NetworkPolicyPeer, d *co yyrg1 := len(yyv1) > 0 yyv21 := yyv1 - yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 16) + yyrl1, yyrt1 = z.DecInferLen(yyl1, z.DecBasicHandle().MaxInitLen, 24) if yyrt1 { if yyrl1 <= cap(yyv1) { yyv1 = yyv1[:yyrl1] diff --git a/staging/src/k8s.io/api/networking/v1/types.go b/staging/src/k8s.io/api/networking/v1/types.go index 1a6ea7b96a640..a0c7721b2d67f 100644 --- a/staging/src/k8s.io/api/networking/v1/types.go +++ b/staging/src/k8s.io/api/networking/v1/types.go @@ -92,6 +92,20 @@ type NetworkPolicyPort struct { Port *intstr.IntOrString `json:"port,omitempty" protobuf:"bytes,2,opt,name=port"` } +// IPBlock describes a particular CIDR (Ex. "192.168.1.1/24") that is allowed to the pods +// matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should +// not be included within this rule. +type IPBlock struct { + // CIDR is a string representing the IP Block + // Valid examples are "192.168.1.1/24" + CIDR string `json:"cidr" protobuf:"bytes,1,name=cidr"` + // Except is a slice of CIDRs that should not be included within an IP Block + // Valid examples are "192.168.1.1/24" + // Except values will be rejected if they are outside the CIDR range + // +optional + Except []string `json:"except,omitempty" protobuf:"bytes,2,rep,name=except"` +} + // NetworkPolicyPeer describes a peer to allow traffic from. Exactly one of its fields // must be specified. type NetworkPolicyPeer struct { @@ -106,6 +120,10 @@ type NetworkPolicyPeer struct { // selector semantics. If present but empty, this selector selects all namespaces. // +optional NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty" protobuf:"bytes,2,opt,name=namespaceSelector"` + + // IPBlock defines policy on a particular IPBlock + // +optional + IPBlock *IPBlock `json:"ipBlock,omitempty" protobuf:"bytes,3,rep,name=ipBlock"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/staging/src/k8s.io/api/networking/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/networking/v1/types_swagger_doc_generated.go index c3c9116313f92..87170082d6dc0 100644 --- a/staging/src/k8s.io/api/networking/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/networking/v1/types_swagger_doc_generated.go @@ -27,6 +27,16 @@ package v1 // Those methods can be generated by using hack/update-generated-swagger-docs.sh // AUTO-GENERATED FUNCTIONS START HERE +var map_IPBlock = map[string]string{ + "": "IPBlock describes a particular CIDR (Ex. \"192.168.1.1/24\") that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should not be included within this rule.", + "cidr": "CIDR is a string representing the IP Block Valid examples are \"192.168.1.1/24\"", + "except": "Except is a slice of CIDRs that should not be included within an IP Block Valid examples are \"192.168.1.1/24\" Except values will be rejected if they are outside the CIDR range", +} + +func (IPBlock) SwaggerDoc() map[string]string { + return map_IPBlock +} + var map_NetworkPolicy = map[string]string{ "": "NetworkPolicy describes what network traffic is allowed for a set of Pods", "metadata": "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", @@ -61,6 +71,7 @@ var map_NetworkPolicyPeer = map[string]string{ "": "NetworkPolicyPeer describes a peer to allow traffic from. Exactly one of its fields must be specified.", "podSelector": "This is a label selector which selects Pods in this namespace. This field follows standard label selector semantics. If present but empty, this selector selects all pods in this namespace.", "namespaceSelector": "Selects Namespaces using cluster scoped-labels. This matches all pods in all namespaces selected by this label selector. This field follows standard label selector semantics. If present but empty, this selector selects all namespaces.", + "ipBlock": "IPBlock defines policy on a particular IPBlock", } func (NetworkPolicyPeer) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/networking/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/networking/v1/zz_generated.deepcopy.go index f6a0a5f809e33..94948b14168e1 100644 --- a/staging/src/k8s.io/api/networking/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/networking/v1/zz_generated.deepcopy.go @@ -39,6 +39,10 @@ func init() { // Deprecated: deepcopy registration will go away when static deepcopy is fully implemented. func RegisterDeepCopies(scheme *runtime.Scheme) error { return scheme.AddGeneratedDeepCopyFuncs( + conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { + in.(*IPBlock).DeepCopyInto(out.(*IPBlock)) + return nil + }, InType: reflect.TypeOf(&IPBlock{})}, conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error { in.(*NetworkPolicy).DeepCopyInto(out.(*NetworkPolicy)) return nil @@ -66,6 +70,27 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { ) } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IPBlock) DeepCopyInto(out *IPBlock) { + *out = *in + if in.Except != nil { + in, out := &in.Except, &out.Except + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPBlock. +func (in *IPBlock) DeepCopy() *IPBlock { + if in == nil { + return nil + } + out := new(IPBlock) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkPolicy) DeepCopyInto(out *NetworkPolicy) { *out = *in @@ -179,6 +204,15 @@ func (in *NetworkPolicyPeer) DeepCopyInto(out *NetworkPolicyPeer) { (*in).DeepCopyInto(*out) } } + if in.IPBlock != nil { + in, out := &in.IPBlock, &out.IPBlock + if *in == nil { + *out = nil + } else { + *out = new(IPBlock) + (*in).DeepCopyInto(*out) + } + } return } diff --git a/staging/src/k8s.io/api/rbac/OWNERS b/staging/src/k8s.io/api/rbac/OWNERS index a35477b920d78..53710111cde18 100755 --- a/staging/src/k8s.io/api/rbac/OWNERS +++ b/staging/src/k8s.io/api/rbac/OWNERS @@ -15,3 +15,4 @@ reviewers: - lixiaobing10051267 - jianhuiz - liggitt +- enj diff --git a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json index d55a95c8eafa5..329d72542387a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json @@ -304,43 +304,43 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/html", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/html/atom", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/internal/timeseries", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/trace", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/sys/unix", @@ -348,79 +348,107 @@ }, { "ImportPath": "golang.org/x/text/cases", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" + }, + { + "ImportPath": "golang.org/x/text/internal", + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/internal/tag", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/language", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/runes", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/bidirule", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/precis", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/transform", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/bidi", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/norm", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/width", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" + }, + { + "ImportPath": "google.golang.org/genproto/googleapis/rpc/status", + "Rev": "09f6ed296fc66555a25fe4ce95173148778dfa85" }, { "ImportPath": "google.golang.org/grpc", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/codes", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/credentials", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/grpclb/grpc_lb_v1", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/grpclog", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/internal", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/keepalive", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/metadata", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/naming", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/peer", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/stats", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/status", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/tap", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/transport", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "gopkg.in/inf.v0", diff --git a/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh b/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh index 7a30bf1356b81..efc52ecd886f6 100755 --- a/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh +++ b/staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh @@ -20,7 +20,7 @@ set -o pipefail SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/.. SCRIPT_BASE=${SCRIPT_ROOT}/../.. -KUBEGEN_PKG=${KUBEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/kube-gen 2>/dev/null || echo k8s.io/kube-gen)} +CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo k8s.io/code-generator)} if LANG=C sed --help 2>&1 | grep -q GNU; then SED="sed" @@ -41,7 +41,7 @@ trap cleanup EXIT echo "Building client-gen" CLIENTGEN="${PWD}/client-gen-binary" -go build -o "${CLIENTGEN}" ${KUBEGEN_PKG}/cmd/client-gen +go build -o "${CLIENTGEN}" ${CODEGEN_PKG}/cmd/client-gen PREFIX=k8s.io/apiextensions-apiserver/pkg/apis INPUT_BASE="--input-base ${PREFIX}" @@ -58,7 +58,7 @@ ${CLIENTGEN} --clientset-name="clientset" ${INPUT_BASE} --input apiextensions/v1 echo "Building lister-gen" listergen="${PWD}/lister-gen" -go build -o "${listergen}" ${KUBEGEN_PKG}/cmd/lister-gen +go build -o "${listergen}" ${CODEGEN_PKG}/cmd/lister-gen LISTER_INPUT="--input-dirs k8s.io/apiextensions-apiserver/pkg/apis/apiextensions --input-dirs k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" LISTER_PATH="--output-package k8s.io/apiextensions-apiserver/pkg/client/listers" @@ -67,7 +67,7 @@ ${listergen} ${LISTER_INPUT} ${LISTER_PATH} --output-base ${SCRIPT_BASE} echo "Building informer-gen" informergen="${PWD}/informer-gen" -go build -o "${informergen}" ${KUBEGEN_PKG}/cmd/informer-gen +go build -o "${informergen}" ${CODEGEN_PKG}/cmd/informer-gen ${informergen} \ --output-base ${SCRIPT_BASE} \ diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go index 15b1255a3d8a2..cb19f35d7c7a0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go @@ -123,11 +123,7 @@ func Convert_v1beta1_CustomResourceDefinitionList_To_apiextensions_CustomResourc func autoConvert_apiextensions_CustomResourceDefinitionList_To_v1beta1_CustomResourceDefinitionList(in *apiextensions.CustomResourceDefinitionList, out *CustomResourceDefinitionList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]CustomResourceDefinition, 0) - } else { - out.Items = *(*[]CustomResourceDefinition)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]CustomResourceDefinition)(unsafe.Pointer(&in.Items)) return nil } @@ -208,11 +204,7 @@ func Convert_v1beta1_CustomResourceDefinitionStatus_To_apiextensions_CustomResou } func autoConvert_apiextensions_CustomResourceDefinitionStatus_To_v1beta1_CustomResourceDefinitionStatus(in *apiextensions.CustomResourceDefinitionStatus, out *CustomResourceDefinitionStatus, s conversion.Scope) error { - if in.Conditions == nil { - out.Conditions = make([]CustomResourceDefinitionCondition, 0) - } else { - out.Conditions = *(*[]CustomResourceDefinitionCondition)(unsafe.Pointer(&in.Conditions)) - } + out.Conditions = *(*[]CustomResourceDefinitionCondition)(unsafe.Pointer(&in.Conditions)) if err := Convert_apiextensions_CustomResourceDefinitionNames_To_v1beta1_CustomResourceDefinitionNames(&in.AcceptedNames, &out.AcceptedNames, s); err != nil { return err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/BUILD index 6bf81ab317c87..058f6e2562388 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/BUILD @@ -17,7 +17,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go index f73c7db37e9c1..b60900ab1bafb 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go @@ -26,7 +26,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/conversion" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -42,8 +41,6 @@ import ( listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion" ) -var cloner = conversion.NewCloner() - // CRDFinalizer is a controller that finalizes the CRD by deleting all the CRs associated with it. type CRDFinalizer struct { crdClient client.CustomResourceDefinitionsGetter diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/BUILD index 3ca71007cba18..2737a2efa49cf 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/BUILD @@ -29,7 +29,6 @@ go_library( "//vendor/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go index 4789f0acaf834..271d11adf8674 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go @@ -26,7 +26,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/conversion" "k8s.io/apimachinery/pkg/labels" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -41,8 +40,6 @@ import ( listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion" ) -var cloner = conversion.NewCloner() - // This controller is reserving names. To avoid conflicts, be sure to run only one instance of the worker at a time. // This could eventually be lifted, but starting simple. type NamingConditionController struct { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd.go index 6abd528354bb3..5f953b1c699a5 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/etcd.go @@ -24,13 +24,13 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" ) -// rest implements a RESTStorage for API services against etcd +// REST implements a RESTStorage for API services against etcd type REST struct { *genericregistry.Store } // NewREST returns a RESTStorage object that will work against API services. -func NewREST(resource schema.GroupResource, listKind schema.GroupVersionKind, copier runtime.ObjectCopier, strategy CustomResourceDefinitionStorageStrategy, optsGetter generic.RESTOptionsGetter) *REST { +func NewREST(resource schema.GroupResource, listKind schema.GroupVersionKind, copier runtime.ObjectCopier, strategy customResourceDefinitionStorageStrategy, optsGetter generic.RESTOptionsGetter) *REST { store := &genericregistry.Store{ Copier: copier, NewFunc: func() runtime.Object { return &unstructured.Unstructured{} }, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy.go index 8ebc90c179970..6b97d07531a78 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/strategy.go @@ -32,7 +32,7 @@ import ( "k8s.io/apiserver/pkg/storage/names" ) -type CustomResourceDefinitionStorageStrategy struct { +type customResourceDefinitionStorageStrategy struct { runtime.ObjectTyper names.NameGenerator @@ -40,8 +40,8 @@ type CustomResourceDefinitionStorageStrategy struct { validator customResourceValidator } -func NewStrategy(typer runtime.ObjectTyper, namespaceScoped bool, kind schema.GroupVersionKind) CustomResourceDefinitionStorageStrategy { - return CustomResourceDefinitionStorageStrategy{ +func NewStrategy(typer runtime.ObjectTyper, namespaceScoped bool, kind schema.GroupVersionKind) customResourceDefinitionStorageStrategy { + return customResourceDefinitionStorageStrategy{ ObjectTyper: typer, NameGenerator: names.SimpleNameGenerator, namespaceScoped: namespaceScoped, @@ -52,36 +52,36 @@ func NewStrategy(typer runtime.ObjectTyper, namespaceScoped bool, kind schema.Gr } } -func (a CustomResourceDefinitionStorageStrategy) NamespaceScoped() bool { +func (a customResourceDefinitionStorageStrategy) NamespaceScoped() bool { return a.namespaceScoped } -func (CustomResourceDefinitionStorageStrategy) PrepareForCreate(ctx genericapirequest.Context, obj runtime.Object) { +func (customResourceDefinitionStorageStrategy) PrepareForCreate(ctx genericapirequest.Context, obj runtime.Object) { } -func (CustomResourceDefinitionStorageStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old runtime.Object) { +func (customResourceDefinitionStorageStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old runtime.Object) { } -func (a CustomResourceDefinitionStorageStrategy) Validate(ctx genericapirequest.Context, obj runtime.Object) field.ErrorList { +func (a customResourceDefinitionStorageStrategy) Validate(ctx genericapirequest.Context, obj runtime.Object) field.ErrorList { return a.validator.Validate(ctx, obj) } -func (CustomResourceDefinitionStorageStrategy) AllowCreateOnUpdate() bool { +func (customResourceDefinitionStorageStrategy) AllowCreateOnUpdate() bool { return false } -func (CustomResourceDefinitionStorageStrategy) AllowUnconditionalUpdate() bool { +func (customResourceDefinitionStorageStrategy) AllowUnconditionalUpdate() bool { return false } -func (CustomResourceDefinitionStorageStrategy) Canonicalize(obj runtime.Object) { +func (customResourceDefinitionStorageStrategy) Canonicalize(obj runtime.Object) { } -func (a CustomResourceDefinitionStorageStrategy) ValidateUpdate(ctx genericapirequest.Context, obj, old runtime.Object) field.ErrorList { +func (a customResourceDefinitionStorageStrategy) ValidateUpdate(ctx genericapirequest.Context, obj, old runtime.Object) field.ErrorList { return a.validator.ValidateUpdate(ctx, obj, old) } -func (a CustomResourceDefinitionStorageStrategy) GetAttrs(obj runtime.Object) (labels.Set, fields.Set, bool, error) { +func (a customResourceDefinitionStorageStrategy) GetAttrs(obj runtime.Object) (labels.Set, fields.Set, bool, error) { accessor, err := meta.Accessor(obj) if err != nil { return nil, nil, false, err @@ -102,7 +102,7 @@ func objectMetaFieldsSet(objectMeta metav1.Object, namespaceScoped bool) fields. } } -func (a CustomResourceDefinitionStorageStrategy) MatchCustomResourceDefinitionStorage(label labels.Selector, field fields.Selector) storage.SelectionPredicate { +func (a customResourceDefinitionStorageStrategy) MatchCustomResourceDefinitionStorage(label labels.Selector, field fields.Selector) storage.SelectionPredicate { return storage.SelectionPredicate{ Label: label, Field: field, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/BUILD index ff0bd54d2e08f..42e88f9887f27 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/BUILD @@ -14,6 +14,7 @@ go_library( deps = [ "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go index de6e34a74cf8a..b6314b0c5c8a3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/strategy.go @@ -19,6 +19,7 @@ package customresourcedefinition import ( "fmt" + apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -46,9 +47,27 @@ func (strategy) NamespaceScoped() bool { } func (strategy) PrepareForCreate(ctx genericapirequest.Context, obj runtime.Object) { + crd := obj.(*apiextensions.CustomResourceDefinition) + crd.Status = apiextensions.CustomResourceDefinitionStatus{} + crd.Generation = 1 } func (strategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old runtime.Object) { + newCRD := obj.(*apiextensions.CustomResourceDefinition) + oldCRD := old.(*apiextensions.CustomResourceDefinition) + newCRD.Status = oldCRD.Status + + // Any changes to the spec increment the generation number, any changes to the + // status should reflect the generation number of the corresponding object. We push + // the burden of managing the status onto the clients because we can't (in general) + // know here what version of spec the writer of the status has seen. It may seem like + // we can at first -- since obj contains spec -- but in the future we will probably make + // status its own object, and even if we don't, writes may be the result of a + // read-update-write loop, so the contents of spec may not actually be the spec that + // the controller has *seen*. + if !apiequality.Semantic.DeepEqual(oldCRD.Spec, newCRD.Spec) { + newCRD.Generation = oldCRD.Generation + 1 + } } func (strategy) Validate(ctx genericapirequest.Context, obj runtime.Object) field.ErrorList { @@ -87,9 +106,14 @@ func (statusStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old r newObj := obj.(*apiextensions.CustomResourceDefinition) oldObj := old.(*apiextensions.CustomResourceDefinition) newObj.Spec = oldObj.Spec + + // Status updates are for only for updating status, not objectmeta. + // TODO: Update after ResetObjectMetaForStatus is added to meta/v1. newObj.Labels = oldObj.Labels newObj.Annotations = oldObj.Annotations newObj.OwnerReferences = oldObj.OwnerReferences + newObj.Generation = oldObj.Generation + newObj.SelfLink = oldObj.SelfLink } func (statusStrategy) AllowCreateOnUpdate() bool { diff --git a/staging/src/k8s.io/apimachinery/Godeps/Godeps.json b/staging/src/k8s.io/apimachinery/Godeps/Godeps.json index 8793842a07cce..50b644b46b787 100644 --- a/staging/src/k8s.io/apimachinery/Godeps/Godeps.json +++ b/staging/src/k8s.io/apimachinery/Godeps/Godeps.json @@ -136,71 +136,75 @@ }, { "ImportPath": "golang.org/x/net/html", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/html/atom", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/text/cases", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" + }, + { + "ImportPath": "golang.org/x/text/internal", + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/internal/tag", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/language", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/runes", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/bidirule", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/precis", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/transform", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/bidi", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/norm", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/width", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "gopkg.in/inf.v0", diff --git a/staging/src/k8s.io/apimachinery/pkg/api/errors/errors.go b/staging/src/k8s.io/apimachinery/pkg/api/errors/errors.go index 91975be85b316..981602270246b 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/errors/errors.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/errors/errors.go @@ -128,6 +128,14 @@ func NewUnauthorized(reason string) *StatusError { // NewForbidden returns an error indicating the requested action was forbidden func NewForbidden(qualifiedResource schema.GroupResource, name string, err error) *StatusError { + var message string + if qualifiedResource.Empty() { + message = fmt.Sprintf("forbidden: %v", err) + } else if name == "" { + message = fmt.Sprintf("%s is forbidden: %v", qualifiedResource.String(), err) + } else { + message = fmt.Sprintf("%s %q is forbidden: %v", qualifiedResource.String(), name, err) + } return &StatusError{metav1.Status{ Status: metav1.StatusFailure, Code: http.StatusForbidden, @@ -137,7 +145,7 @@ func NewForbidden(qualifiedResource schema.GroupResource, name string, err error Kind: qualifiedResource.Resource, Name: name, }, - Message: fmt.Sprintf("%s %q is forbidden: %v", qualifiedResource.String(), name, err), + Message: message, }} } diff --git a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/roundtrip.go b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/roundtrip.go index 6fec930dc21ed..479a6a50a3e24 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/roundtrip.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/testing/roundtrip/roundtrip.go @@ -270,6 +270,12 @@ func roundTrip(t *testing.T, scheme *runtime.Scheme, codec runtime.Codec, object // deep copy the original object object = object.DeepCopyObject() name := reflect.TypeOf(object).Elem().Name() + if !apiequality.Semantic.DeepEqual(original, object) { + t.Errorf("%v: DeepCopy altered the object, diff: %v", name, diff.ObjectReflectDiff(original, object)) + t.Errorf("%s", spew.Sdump(original)) + t.Errorf("%s", spew.Sdump(object)) + return + } // catch deepcopy errors early if !apiequality.Semantic.DeepEqual(original, object) { @@ -323,7 +329,7 @@ func roundTrip(t *testing.T, scheme *runtime.Scheme, codec runtime.Codec, object // ensure that the object produced from decoding the encoded data is equal // to the original object if !apiequality.Semantic.DeepEqual(original, obj2) { - t.Errorf("%v: diff: %v\nCodec: %#v\nSource:\n\n%#v\n\nEncoded:\n\n%s\n\nFinal:\n\n%#v", name, diff.ObjectReflectDiff(object, obj2), codec, printer.Sprintf("%#v", object), dataAsString(data), printer.Sprintf("%#v", obj2)) + t.Errorf("%v: diff: %v\nCodec: %#v\nSource:\n\n%#v\n\nEncoded:\n\n%s\n\nFinal:\n\n%#v", name, diff.ObjectReflectDiff(original, obj2), codec, printer.Sprintf("%#v", original), dataAsString(data), printer.Sprintf("%#v", obj2)) return } @@ -370,7 +376,7 @@ func roundTrip(t *testing.T, scheme *runtime.Scheme, codec runtime.Codec, object // NOTE: we use the encoding+decoding here as an alternative, guaranteed deep-copy to compare against. fuzzer.ValueFuzz(object) if !apiequality.Semantic.DeepEqual(original, obj3) { - t.Errorf("%v: fuzzing a copy altered the original, diff: %v", name, diff.ObjectReflectDiff(original, object)) + t.Errorf("%v: fuzzing a copy altered the original, diff: %v", name, diff.ObjectReflectDiff(original, obj3)) return } } diff --git a/staging/src/k8s.io/apimachinery/pkg/api/validation/objectmeta.go b/staging/src/k8s.io/apimachinery/pkg/api/validation/objectmeta.go index 84bd9cdedc6b8..5bef9495c60bc 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/validation/objectmeta.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/validation/objectmeta.go @@ -200,9 +200,6 @@ func ValidateInitializers(initializers *metav1.Initializers, fldPath *field.Path } } allErrs = append(allErrs, validateInitializersResult(initializers.Result, fldPath.Child("result"))...) - if len(initializers.Pending) == 0 && initializers.Result == nil { - allErrs = append(allErrs, field.Invalid(fldPath.Child("pending"), nil, "must be non-empty when result is not set")) - } return allErrs } @@ -288,7 +285,7 @@ func ValidateObjectMetaAccessorUpdate(newMeta, oldMeta metav1.Object, fldPath *f if newMeta.GetDeletionGracePeriodSeconds() != nil && (oldMeta.GetDeletionGracePeriodSeconds() == nil || *newMeta.GetDeletionGracePeriodSeconds() != *oldMeta.GetDeletionGracePeriodSeconds()) { allErrs = append(allErrs, field.Invalid(fldPath.Child("deletionGracePeriodSeconds"), newMeta.GetDeletionGracePeriodSeconds(), "field is immutable; may only be changed via deletion")) } - if newMeta.GetDeletionTimestamp() != nil && (oldMeta.GetDeletionTimestamp() == nil || !newMeta.GetDeletionTimestamp().Equal(*oldMeta.GetDeletionTimestamp())) { + if newMeta.GetDeletionTimestamp() != nil && (oldMeta.GetDeletionTimestamp() == nil || !newMeta.GetDeletionTimestamp().Equal(oldMeta.GetDeletionTimestamp())) { allErrs = append(allErrs, field.Invalid(fldPath.Child("deletionTimestamp"), newMeta.GetDeletionTimestamp(), "field is immutable; may only be changed via deletion")) } diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto index 83b9eb14bb22b..c6a9edd380188 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto @@ -250,6 +250,8 @@ message Initializers { // When the last pending initializer is removed, and no failing result is set, the initializers // struct will be set to nil and the object is considered as initialized and visible to all // clients. + // +patchMergeKey=name + // +patchStrategy=merge repeated Initializer pending = 1; // If result is set with the Failure field, the object will be persisted to storage and then deleted, diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time.go index cdfbcb54fda15..dbe206704f891 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time.go @@ -40,8 +40,8 @@ type MicroTime struct { // DeepCopy returns a deep-copy of the MicroTime value. The underlying time.Time // type is effectively immutable in the time API, so it is safe to // copy-by-assign, despite the presence of (unexported) Pointer fields. -func (t MicroTime) DeepCopy() MicroTime { - return t +func (t *MicroTime) DeepCopyInto(out *MicroTime) { + *out = *t } // String returns the representation of the time. @@ -74,22 +74,22 @@ func (t *MicroTime) IsZero() bool { } // Before reports whether the time instant t is before u. -func (t MicroTime) Before(u MicroTime) bool { +func (t *MicroTime) Before(u *MicroTime) bool { return t.Time.Before(u.Time) } // Equal reports whether the time instant t is equal to u. -func (t MicroTime) Equal(u MicroTime) bool { +func (t *MicroTime) Equal(u *MicroTime) bool { return t.Time.Equal(u.Time) } // BeforeTime reports whether the time instant t is before second-lever precision u. -func (t MicroTime) BeforeTime(u Time) bool { +func (t *MicroTime) BeforeTime(u *Time) bool { return t.Time.Before(u.Time) } // EqualTime reports whether the time instant t is equal to second-lever precision u. -func (t MicroTime) EqualTime(u Time) bool { +func (t *MicroTime) EqualTime(u *Time) bool { return t.Time.Equal(u.Time) } diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/time.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/time.go index c73d7ca633549..435f6a8f59946 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/time.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/time.go @@ -74,12 +74,12 @@ func (t *Time) IsZero() bool { } // Before reports whether the time instant t is before u. -func (t Time) Before(u Time) bool { +func (t *Time) Before(u *Time) bool { return t.Time.Before(u.Time) } // Equal reports whether the time instant t is equal to u. -func (t Time) Equal(u Time) bool { +func (t *Time) Equal(u *Time) bool { return t.Time.Equal(u.Time) } diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go index 597e0e80ddc19..315980b8773b8 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package unversioned contains API types that are common to all versions. +// Package v1 contains API types that are common to all versions. // // The package contains two categories of types: // - external (serialized) types that lack their own version (e.g TypeMeta) @@ -248,7 +248,9 @@ type Initializers struct { // When the last pending initializer is removed, and no failing result is set, the initializers // struct will be set to nil and the object is considered as initialized and visible to all // clients. - Pending []Initializer `json:"pending" protobuf:"bytes,1,rep,name=pending"` + // +patchMergeKey=name + // +patchStrategy=merge + Pending []Initializer `json:"pending" protobuf:"bytes,1,rep,name=pending" patchStrategy:"merge" patchMergeKey:"name"` // If result is set with the Failure field, the object will be persisted to storage and then deleted, // ensuring that other clients can observe the deletion. Result *Status `json:"result,omitempty" protobuf:"bytes,2,opt,name=result"` diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go index 20d7ff391116a..c73e777b50b56 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/zz_generated.deepcopy.go @@ -778,10 +778,14 @@ func (in *ListOptions) DeepCopyObject() runtime.Object { } } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MicroTime) DeepCopyInto(out *MicroTime) { - *out = in.DeepCopy() - return +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MicroTime. +func (in *MicroTime) DeepCopy() *MicroTime { + if in == nil { + return nil + } + out := new(MicroTime) + in.DeepCopyInto(out) + return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/v1/zz_generated.conversion.go b/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/v1/zz_generated.conversion.go index bd3dbf1d69465..bc060c309bf18 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/v1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/testapigroup/v1/zz_generated.conversion.go @@ -143,7 +143,7 @@ func autoConvert_testapigroup_CarpList_To_v1_CarpList(in *testapigroup.CarpList, } } } else { - out.Items = make([]Carp, 0) + out.Items = nil } return nil } diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/conversion.go b/staging/src/k8s.io/apimachinery/pkg/runtime/conversion.go index 8eedffc9c348b..afe4fab15e8f4 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/conversion.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/conversion.go @@ -19,6 +19,7 @@ limitations under the License. package runtime import ( + "fmt" "reflect" "strconv" "strings" @@ -26,6 +27,20 @@ import ( "k8s.io/apimachinery/pkg/conversion" ) +// DefaultFieldSelectorConversion auto-accepts metav1 values for name and namespace. +// A cluster scoped resource specifying namespace empty works fine and specifying a particular +// namespace will return no results, as expected. +func DefaultMetaV1FieldSelectorConversion(label, value string) (string, string, error) { + switch label { + case "metadata.name": + return label, value, nil + case "metadata.namespace": + return label, value, nil + default: + return "", "", fmt.Errorf("%q is not a known field selector: only %q, %q", label, "metadata.name", "metadata.namespace") + } +} + // JSONKeyMapper uses the struct tags on a conversion to determine the key value for // the other side. Use when mapping from a map[string]* to a struct or vice versa. func JSONKeyMapper(key string, sourceTag, destTag reflect.StructTag) (string, string) { diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/scheme.go b/staging/src/k8s.io/apimachinery/pkg/runtime/scheme.go index ee78705b79c1f..c3d4b7f5f8eea 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/scheme.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/scheme.go @@ -454,11 +454,11 @@ func (s *Scheme) Convert(in, out interface{}, context interface{}) error { // versioned representation to an unversioned one. func (s *Scheme) ConvertFieldLabel(version, kind, label, value string) (string, string, error) { if s.fieldLabelConversionFuncs[version] == nil { - return "", "", fmt.Errorf("No field label conversion function found for version: %s", version) + return DefaultMetaV1FieldSelectorConversion(label, value) } conversionFunc, ok := s.fieldLabelConversionFuncs[version][kind] if !ok { - return "", "", fmt.Errorf("No field label conversion function found for version %s and kind %s", version, kind) + return DefaultMetaV1FieldSelectorConversion(label, value) } return conversionFunc(label, value) } diff --git a/staging/src/k8s.io/apimachinery/pkg/test/apis_meta_v1_unstructed_unstructure_test.go b/staging/src/k8s.io/apimachinery/pkg/test/apis_meta_v1_unstructed_unstructure_test.go index d6436214c7abd..338206d42c005 100644 --- a/staging/src/k8s.io/apimachinery/pkg/test/apis_meta_v1_unstructed_unstructure_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/test/apis_meta_v1_unstructed_unstructure_test.go @@ -206,11 +206,11 @@ func TestUnstructuredGetters(t *testing.T) { t.Errorf("GetSelfLink() = %s, want %s", got, want) } - if got, want := unstruct.GetCreationTimestamp(), metav1.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC); !got.Equal(want) { + if got, want := unstruct.GetCreationTimestamp(), metav1.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC); !got.Equal(&want) { t.Errorf("GetCreationTimestamp() = %s, want %s", got, want) } - if got, want := unstruct.GetDeletionTimestamp(), metav1.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC); got == nil || !got.Equal(want) { + if got, want := unstruct.GetDeletionTimestamp(), metav1.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC); got == nil || !got.Equal(&want) { t.Errorf("GetDeletionTimestamp() = %s, want %s", got, want) } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/sets/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/sets/BUILD index 831606ed3be91..3bf4ebc13b671 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/sets/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/sets/BUILD @@ -33,7 +33,7 @@ go_genrule( "string.go", ], cmd = """ -$(location //vendor/k8s.io/kube-gen/cmd/set-gen) \ +$(location //vendor/k8s.io/code-generator/cmd/set-gen) \ --input-dirs ./vendor/k8s.io/apimachinery/pkg/util/sets/types \ --output-base $$(dirname $$(dirname $(location :byte.go))) \ --go-header-file $(location //hack/boilerplate:boilerplate.go.txt) \ @@ -43,7 +43,7 @@ $(location //vendor/k8s.io/kube-gen/cmd/set-gen) \ "//vendor/k8s.io/apimachinery/pkg/util/sets/types:go_default_library", ], tools = [ - "//vendor/k8s.io/kube-gen/cmd/set-gen", + "//vendor/k8s.io/code-generator/cmd/set-gen", ], ) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/sets/types/types.go b/staging/src/k8s.io/apimachinery/pkg/util/sets/types/types.go index f7a280ec77f48..801498ad7d8bb 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/sets/types/types.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/sets/types/types.go @@ -16,7 +16,7 @@ limitations under the License. // Package types just provides input types to the set generator. It also // contains a "go generate" block. -// (You must first `go install k8s.io/kube-gen/cmd/set-gen`) +// (You must first `go install k8s.io/code-generator/cmd/set-gen`) package types //go:generate set-gen -i k8s.io/kubernetes/pkg/util/sets/types diff --git a/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/BUILD b/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/BUILD index cf64e1f539a62..9754f28e93219 100644 --- a/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/BUILD +++ b/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/BUILD @@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", + "go_test", ) go_library( @@ -10,6 +11,12 @@ go_library( srcs = ["fields.go"], ) +go_test( + name = "go_default_test", + srcs = ["fields_test.go"], + library = ":go_default_library", +) + filegroup( name = "package-srcs", srcs = glob(["**"]), diff --git a/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/fields.go b/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/fields.go index ac6d9cb96b1d3..006972ecafd1f 100644 --- a/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/fields.go +++ b/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/fields.go @@ -28,6 +28,9 @@ const ( // TODO: fix the returned errors to be introspectable. func LookupPatchMetadata(t reflect.Type, jsonField string) ( elemType reflect.Type, patchStrategies []string, patchMergeKey string, e error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } if t.Kind() == reflect.Map { elemType = t.Elem() return diff --git a/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/fields_test.go b/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/fields_test.go new file mode 100644 index 0000000000000..04d8cebd9b27a --- /dev/null +++ b/staging/src/k8s.io/apimachinery/third_party/forked/golang/json/fields_test.go @@ -0,0 +1,30 @@ +package json + +import ( + "reflect" + "testing" +) + +func TestLookupPtrToStruct(t *testing.T) { + type Elem struct { + Key string + Value string + } + type Outer struct { + Inner []Elem `json:"inner" patchStrategy:"merge" patchMergeKey:"key"` + } + outer := &Outer{} + elemType, patchStrategies, patchMergeKey, err := LookupPatchMetadata(reflect.TypeOf(outer), "inner") + if err != nil { + t.Fatal(err) + } + if elemType != reflect.TypeOf([]Elem{}) { + t.Errorf("elemType = %v, want: %v", elemType, reflect.TypeOf([]Elem{})) + } + if !reflect.DeepEqual(patchStrategies, []string{"merge"}) { + t.Errorf("patchStrategies = %v, want: %v", patchStrategies, []string{"merge"}) + } + if patchMergeKey != "key" { + t.Errorf("patchMergeKey = %v, want: %v", patchMergeKey, "key") + } +} diff --git a/staging/src/k8s.io/apiserver/Godeps/Godeps.json b/staging/src/k8s.io/apiserver/Godeps/Godeps.json index 123f5c9918e80..780cd8721485a 100644 --- a/staging/src/k8s.io/apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/apiserver/Godeps/Godeps.json @@ -326,6 +326,10 @@ "ImportPath": "github.com/evanphx/json-patch", "Rev": "944e07253867aacae43c04b2e6a239005443f33a" }, + { + "ImportPath": "github.com/fatih/structs", + "Rev": "7e5a8eef611ee84dd359503f3969f80df4c50723" + }, { "ImportPath": "github.com/ghodss/yaml", "Rev": "73d445a93680fa1a78ae23a5839bad48f32ba1ee" @@ -382,6 +386,10 @@ "ImportPath": "github.com/golang/protobuf/ptypes/timestamp", "Rev": "4bd1920723d7b7c925de087aa32e2187708897f7" }, + { + "ImportPath": "github.com/golang/snappy", + "Rev": "553a641470496b2327abcac10b36396bd98e45c9" + }, { "ImportPath": "github.com/google/btree", "Rev": "7d79101e329e5a3adf994758c578dab82b90c017" @@ -446,6 +454,22 @@ "ImportPath": "github.com/grpc-ecosystem/grpc-gateway/utilities", "Rev": "84398b94e188ee336f307779b57b3aa91af7063c" }, + { + "ImportPath": "github.com/hashicorp/errwrap", + "Rev": "7554cd9344cec97297fa6649b055a8c98c2a1e55" + }, + { + "ImportPath": "github.com/hashicorp/go-cleanhttp", + "Rev": "3573b8b52aa7b37b9358d966a898feb387f62437" + }, + { + "ImportPath": "github.com/hashicorp/go-multierror", + "Rev": "83588e72410abfbe4df460eeb6f30841ae47d4c4" + }, + { + "ImportPath": "github.com/hashicorp/go-rootcerts", + "Rev": "6bb64b370b90e7ef1fa532be9e591a81c3493e00" + }, { "ImportPath": "github.com/hashicorp/golang-lru", "Rev": "a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4" @@ -454,6 +478,58 @@ "ImportPath": "github.com/hashicorp/golang-lru/simplelru", "Rev": "a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4" }, + { + "ImportPath": "github.com/hashicorp/hcl", + "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" + }, + { + "ImportPath": "github.com/hashicorp/hcl/hcl/ast", + "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" + }, + { + "ImportPath": "github.com/hashicorp/hcl/hcl/parser", + "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" + }, + { + "ImportPath": "github.com/hashicorp/hcl/hcl/scanner", + "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" + }, + { + "ImportPath": "github.com/hashicorp/hcl/hcl/strconv", + "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" + }, + { + "ImportPath": "github.com/hashicorp/hcl/hcl/token", + "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" + }, + { + "ImportPath": "github.com/hashicorp/hcl/json/parser", + "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" + }, + { + "ImportPath": "github.com/hashicorp/hcl/json/scanner", + "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" + }, + { + "ImportPath": "github.com/hashicorp/hcl/json/token", + "Rev": "d8c773c4cba11b11539e3d45f93daeaa5dcf1fa1" + }, + { + "ImportPath": "github.com/hashicorp/vault/api", + "Rev": "52401fd0e5d983e44112c81d933553d2c03f612a" + }, + { + "ImportPath": "github.com/hashicorp/vault/helper/compressutil", + "Rev": "52401fd0e5d983e44112c81d933553d2c03f612a" + }, + { + "ImportPath": "github.com/hashicorp/vault/helper/jsonutil", + "Rev": "52401fd0e5d983e44112c81d933553d2c03f612a" + }, + { + "ImportPath": "github.com/hashicorp/vault/helper/parseutil", + "Rev": "52401fd0e5d983e44112c81d933553d2c03f612a" + }, { "ImportPath": "github.com/howeyc/gopass", "Rev": "bf9dde6d0d2c004a008c27aaee91170c786f6db8" @@ -490,6 +566,14 @@ "ImportPath": "github.com/matttproud/golang_protobuf_extensions/pbutil", "Rev": "fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a" }, + { + "ImportPath": "github.com/mitchellh/go-homedir", + "Rev": "b8bc1bf767474819792c23f32d8286a45736f1c6" + }, + { + "ImportPath": "github.com/mitchellh/mapstructure", + "Rev": "53818660ed4955e899c0bcafa97299a388bd7c8e" + }, { "ImportPath": "github.com/mxk/go-flowrate/flowrate", "Rev": "cca7078d478f8520f85629ad7c68962d31ed7682" @@ -534,6 +618,10 @@ "ImportPath": "github.com/prometheus/procfs/xfs", "Rev": "65c1f6f8f0fc1e2185eb9863a3bc751496404259" }, + { + "ImportPath": "github.com/sethgrid/pester", + "Rev": "a86a2d88f4dc3c7dbf3a6a6bbbfb095690b834b6" + }, { "ImportPath": "github.com/spf13/pflag", "Rev": "9ff6c6923cfffbcd502984b8e0c80539a94968b7" @@ -580,43 +668,43 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/html", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/html/atom", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/idna", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/internal/timeseries", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/lex/httplex", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/trace", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + "Rev": "1c05540f6879653db88113bc4a2b70aec4bd491f" }, { "ImportPath": "golang.org/x/sys/unix", @@ -624,83 +712,111 @@ }, { "ImportPath": "golang.org/x/text/cases", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" + }, + { + "ImportPath": "golang.org/x/text/internal", + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/internal/tag", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/language", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/runes", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/bidirule", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/secure/precis", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/transform", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/bidi", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/unicode/norm", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/text/width", - "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" + "Rev": "b19bf474d317b857955b12035d2c5acb57ce8b01" }, { "ImportPath": "golang.org/x/time/rate", "Rev": "f51c12702a4d776e4c1fa9b0fabab841babae631" }, + { + "ImportPath": "google.golang.org/genproto/googleapis/rpc/status", + "Rev": "09f6ed296fc66555a25fe4ce95173148778dfa85" + }, { "ImportPath": "google.golang.org/grpc", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/codes", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/credentials", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/grpclb/grpc_lb_v1", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/grpclog", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/internal", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/keepalive", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/metadata", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/naming", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/peer", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/stats", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/status", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" + }, + { + "ImportPath": "google.golang.org/grpc/tap", + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "google.golang.org/grpc/transport", - "Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0" + "Rev": "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" }, { "ImportPath": "gopkg.in/inf.v0", diff --git a/staging/src/k8s.io/apiserver/OWNERS b/staging/src/k8s.io/apiserver/OWNERS index e4525fc6b7f96..52f250446a3b7 100644 --- a/staging/src/k8s.io/apiserver/OWNERS +++ b/staging/src/k8s.io/apiserver/OWNERS @@ -15,3 +15,4 @@ reviewers: - ncdc - tallclair - timothysc +- enj diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugins.go b/staging/src/k8s.io/apiserver/pkg/admission/plugins.go index dd1368d4ddd94..5ddfc7e1f84b7 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugins.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugins.go @@ -67,13 +67,15 @@ func (ps *Plugins) Registered() []string { func (ps *Plugins) Register(name string, plugin Factory) { ps.lock.Lock() defer ps.lock.Unlock() - _, found := ps.registry[name] - if found { - glog.Fatalf("Admission plugin %q was registered twice", name) - } - if ps.registry == nil { + if ps.registry != nil { + _, found := ps.registry[name] + if found { + glog.Fatalf("Admission plugin %q was registered twice", name) + } + } else { ps.registry = map[string]Factory{} } + glog.V(1).Infof("Registered admission plugin %q", name) ps.registry[name] = plugin } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go index 750a73cd9e222..8b6a1695634fc 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go @@ -71,7 +71,7 @@ func autoConvert_apiserver_AdmissionConfiguration_To_v1alpha1_AdmissionConfigura } } } else { - out.Plugins = make([]AdmissionPluginConfiguration, 0) + out.Plugins = nil } return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/audit/fuzzer/fuzzer.go b/staging/src/k8s.io/apiserver/pkg/apis/audit/fuzzer/fuzzer.go index 24b06367350aa..1ec4464c5b341 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/audit/fuzzer/fuzzer.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/audit/fuzzer/fuzzer.go @@ -28,6 +28,7 @@ import ( func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { return []interface{}{ func(e *audit.Event, c fuzz.Continue) { + c.FuzzNoCustom(e) switch c.RandBool() { case true: e.RequestObject = nil diff --git a/staging/src/k8s.io/apiserver/pkg/apis/audit/v1alpha1/zz_generated.conversion.go b/staging/src/k8s.io/apiserver/pkg/apis/audit/v1alpha1/zz_generated.conversion.go index 972dca99f0c28..8007dda36e855 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/audit/v1alpha1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/audit/v1alpha1/zz_generated.conversion.go @@ -120,11 +120,7 @@ func Convert_v1alpha1_EventList_To_audit_EventList(in *EventList, out *audit.Eve func autoConvert_audit_EventList_To_v1alpha1_EventList(in *audit.EventList, out *EventList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]Event, 0) - } else { - out.Items = *(*[]Event)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]Event)(unsafe.Pointer(&in.Items)) return nil } @@ -200,11 +196,7 @@ func Convert_v1alpha1_Policy_To_audit_Policy(in *Policy, out *audit.Policy, s co func autoConvert_audit_Policy_To_v1alpha1_Policy(in *audit.Policy, out *Policy, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Rules == nil { - out.Rules = make([]PolicyRule, 0) - } else { - out.Rules = *(*[]PolicyRule)(unsafe.Pointer(&in.Rules)) - } + out.Rules = *(*[]PolicyRule)(unsafe.Pointer(&in.Rules)) return nil } @@ -226,11 +218,7 @@ func Convert_v1alpha1_PolicyList_To_audit_PolicyList(in *PolicyList, out *audit. func autoConvert_audit_PolicyList_To_v1alpha1_PolicyList(in *audit.PolicyList, out *PolicyList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]Policy, 0) - } else { - out.Items = *(*[]Policy)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]Policy)(unsafe.Pointer(&in.Items)) return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1/zz_generated.conversion.go b/staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1/zz_generated.conversion.go index 78617dba62095..905420af9debe 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1/zz_generated.conversion.go @@ -120,11 +120,7 @@ func Convert_v1beta1_EventList_To_audit_EventList(in *EventList, out *audit.Even func autoConvert_audit_EventList_To_v1beta1_EventList(in *audit.EventList, out *EventList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]Event, 0) - } else { - out.Items = *(*[]Event)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]Event)(unsafe.Pointer(&in.Items)) return nil } @@ -200,11 +196,7 @@ func Convert_v1beta1_Policy_To_audit_Policy(in *Policy, out *audit.Policy, s con func autoConvert_audit_Policy_To_v1beta1_Policy(in *audit.Policy, out *Policy, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta - if in.Rules == nil { - out.Rules = make([]PolicyRule, 0) - } else { - out.Rules = *(*[]PolicyRule)(unsafe.Pointer(&in.Rules)) - } + out.Rules = *(*[]PolicyRule)(unsafe.Pointer(&in.Rules)) return nil } @@ -226,11 +218,7 @@ func Convert_v1beta1_PolicyList_To_audit_PolicyList(in *PolicyList, out *audit.P func autoConvert_audit_PolicyList_To_v1beta1_PolicyList(in *audit.PolicyList, out *PolicyList, s conversion.Scope) error { out.ListMeta = in.ListMeta - if in.Items == nil { - out.Items = make([]Policy, 0) - } else { - out.Items = *(*[]Policy)(unsafe.Pointer(&in.Items)) - } + out.Items = *(*[]Policy)(unsafe.Pointer(&in.Items)) return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/example/v1/zz_generated.conversion.go b/staging/src/k8s.io/apiserver/pkg/apis/example/v1/zz_generated.conversion.go index 4785d9da2c9e9..8fa603fde7648 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/example/v1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/example/v1/zz_generated.conversion.go @@ -143,7 +143,7 @@ func autoConvert_example_PodList_To_v1_PodList(in *example.PodList, out *PodList } } } else { - out.Items = make([]Pod, 0) + out.Items = nil } return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount/util.go b/staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount/util.go index ac3c252b7515d..1b7bbc1390d0f 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount/util.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount/util.go @@ -59,8 +59,8 @@ func SplitUsername(username string) (string, string, error) { return namespace, name, nil } -// MakeGroupNames generates service account group names for the given namespace and ServiceAccount name -func MakeGroupNames(namespace, name string) []string { +// MakeGroupNames generates service account group names for the given namespace +func MakeGroupNames(namespace string) []string { return []string{ AllServiceAccountsGroup, MakeNamespaceGroupName(namespace), diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/apiserver_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/apiserver_test.go index af4e119ef191a..4847a59340808 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/apiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/apiserver_test.go @@ -1869,7 +1869,7 @@ func TestGetTable(t *testing.T) { expected: &metav1alpha1.Table{ TypeMeta: metav1.TypeMeta{Kind: "Table", APIVersion: "meta.k8s.io/v1alpha1"}, ColumnDefinitions: []metav1alpha1.TableColumnDefinition{ - {Name: "Name", Type: "string", Description: metaDoc["name"]}, + {Name: "Name", Type: "string", Format: "name", Description: metaDoc["name"]}, {Name: "Created At", Type: "date", Description: metaDoc["creationTimestamp"]}, }, Rows: []metav1alpha1.TableRow{ @@ -1883,7 +1883,7 @@ func TestGetTable(t *testing.T) { expected: &metav1alpha1.Table{ TypeMeta: metav1.TypeMeta{Kind: "Table", APIVersion: "meta.k8s.io/v1alpha1"}, ColumnDefinitions: []metav1alpha1.TableColumnDefinition{ - {Name: "Name", Type: "string", Description: metaDoc["name"]}, + {Name: "Name", Type: "string", Format: "name", Description: metaDoc["name"]}, {Name: "Created At", Type: "date", Description: metaDoc["creationTimestamp"]}, }, Rows: []metav1alpha1.TableRow{ diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go index a31bff125911a..167bc53021368 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go @@ -85,7 +85,7 @@ func WithImpersonation(handler http.Handler, requestContextMapper request.Reques username = serviceaccount.MakeUsername(impersonationRequest.Namespace, impersonationRequest.Name) if !groupsSpecified { // if groups aren't specified for a service account, we know the groups because its a fixed mapping. Add them - groups = serviceaccount.MakeGroupNames(impersonationRequest.Namespace, impersonationRequest.Name) + groups = serviceaccount.MakeGroupNames(impersonationRequest.Namespace) } case v1.SchemeGroupVersion.WithKind("User").GroupKind(): diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/errors_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/errors_test.go index 8ea30401e277a..de5d3a300c357 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/errors_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/errors_test.go @@ -68,13 +68,15 @@ func TestForbidden(t *testing.T) { reason string contentType string }{ - {`{"metadata":{},"status":"Failure","message":" \"\" is forbidden: User \"NAME\" cannot GET path \"/whatever\".","reason":"Forbidden","details":{},"code":403} + {`{"metadata":{},"status":"Failure","message":"forbidden: User \"NAME\" cannot GET path \"/whatever\".","reason":"Forbidden","details":{},"code":403} `, authorizer.AttributesRecord{User: u, Verb: "GET", Path: "/whatever"}, "", "application/json"}, - {`{"metadata":{},"status":"Failure","message":" \"\" is forbidden: User \"NAME\" cannot GET path \"/\u0026lt;script\u0026gt;\".","reason":"Forbidden","details":{},"code":403} + {`{"metadata":{},"status":"Failure","message":"forbidden: User \"NAME\" cannot GET path \"/\u0026lt;script\u0026gt;\".","reason":"Forbidden","details":{},"code":403} `, authorizer.AttributesRecord{User: u, Verb: "GET", Path: "/