Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Add workaround for Minikube storage provisioning bug #752

Merged
merged 2 commits into from
Jun 23, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion src/api/kube.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
* SPDX-License-Identifier: EPL-2.0
**********************************************************************/

import { ApiextensionsV1beta1Api, ApisApi, AppsV1Api, BatchV1Api, CoreV1Api, CustomObjectsApi, ExtensionsV1beta1Api, KubeConfig, Log, PortForward, RbacAuthorizationV1Api, V1beta1CustomResourceDefinition, V1beta1IngressList, V1ClusterRole, V1ClusterRoleBinding, V1ConfigMap, V1ConfigMapEnvSource, V1Container, V1DeleteOptions, V1Deployment, V1DeploymentList, V1DeploymentSpec, V1EnvFromSource, V1Job, V1JobSpec, V1LabelSelector, V1NamespaceList, V1ObjectMeta, V1PersistentVolumeClaimList, V1Pod, V1PodList, V1PodSpec, V1PodTemplateSpec, V1Role, V1RoleBinding, V1RoleRef, V1Secret, V1ServiceAccount, V1ServiceList, V1Subject, Watch } from '@kubernetes/client-node'
import { ApiextensionsV1beta1Api, ApisApi, AppsV1Api, BatchV1Api, CoreV1Api, CustomObjectsApi, ExtensionsV1beta1Api, KubeConfig, Log, PortForward, RbacAuthorizationV1Api, V1beta1CustomResourceDefinition, V1beta1IngressList, V1ClusterRole, V1ClusterRoleBinding, V1ConfigMap, V1ConfigMapEnvSource, V1Container, V1DeleteOptions, V1Deployment, V1DeploymentList, V1DeploymentSpec, V1EnvFromSource, V1Job, V1JobSpec, V1LabelSelector, V1NamespaceList, V1ObjectMeta, V1PersistentVolumeClaimList, V1Pod, V1PodList, V1PodSpec, V1PodTemplateSpec, V1PolicyRule, V1Role, V1RoleBinding, V1RoleRef, V1Secret, V1ServiceAccount, V1ServiceList, V1Subject, Watch } from '@kubernetes/client-node'
import { Cluster, Context } from '@kubernetes/client-node/dist/config_types'
import axios, { AxiosRequestConfig } from 'axios'
import { cli } from 'cli-ux'
Expand Down Expand Up @@ -225,6 +225,16 @@ export class KubeHelper {
}
}

async getClusterRole(name: string): Promise<V1ClusterRole | undefined> {
const k8sRbacAuthApi = KubeHelper.KUBE_CONFIG.makeApiClient(RbacAuthorizationV1Api)
try {
const { body } = await k8sRbacAuthApi.readClusterRole(name)
return body
} catch {
return
}
}

async createRoleFromFile(filePath: string, namespace = '') {
const yamlRole = this.safeLoadFromYamlFile(filePath) as V1Role
const k8sRbacAuthApi = KubeHelper.KUBE_CONFIG.makeApiClient(RbacAuthorizationV1Api)
Expand Down Expand Up @@ -292,6 +302,30 @@ export class KubeHelper {
}
}

async addClusterRoleRule(name: string, apiGroups: string[], resources: string[], verbs: string[]): Promise<V1ClusterRole | undefined> {
const k8sRbacAuthApi = KubeHelper.KUBE_CONFIG.makeApiClient(RbacAuthorizationV1Api)
const clusterRole = await this.getClusterRole(name)
if (clusterRole) {
// Clean up metadata, otherwise replace role call will fail
clusterRole.metadata = {}
clusterRole.metadata.name = name

// Add new policy
const additionaRule = new V1PolicyRule()
additionaRule.apiGroups = apiGroups
additionaRule.resources = resources
additionaRule.verbs = verbs
clusterRole.rules.push(additionaRule)

try {
const { body } = await k8sRbacAuthApi.replaceClusterRole(name, clusterRole)
return body
} catch {
return
}
}
}

async deleteRole(name = '', namespace = '') {
const k8sCoreApi = KubeHelper.KUBE_CONFIG.makeApiClient(RbacAuthorizationV1Api)
try {
Expand Down Expand Up @@ -518,6 +552,26 @@ export class KubeHelper {
}
}

async patchNamespacedPod(name: string, namespace: string, patch: any): Promise<V1Pod | undefined> {
const k8sCoreApi = KubeHelper.KUBE_CONFIG.makeApiClient(CoreV1Api)

// It is required to patch content-type, otherwise request will be rejected with 415 (Unsupported media type) error.
const requestOptions = {
headers: {
'content-type': 'application/strategic-merge-patch+json'
}
}

try {
const res = await k8sCoreApi.patchNamespacedPod(name, namespace, patch, undefined, undefined, requestOptions)
if (res && res.body) {
return res.body
}
} catch {
return
}
}

async podsExistBySelector(selector: string, namespace = ''): Promise<boolean> {
const k8sCoreApi = KubeHelper.KUBE_CONFIG.makeApiClient(CoreV1Api)
let res
Expand Down
52 changes: 52 additions & 0 deletions src/tasks/platforms/minikube.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import * as commandExists from 'command-exists'
import * as execa from 'execa'
import * as Listr from 'listr'

import { KubeHelper } from '../../api/kube'
import { VersionHelper } from '../../api/version'

import { CommonPlatformTasks } from './common-platform-tasks'
Expand All @@ -22,6 +23,7 @@ export class MinikubeTasks {
* Returns tasks list which perform preflight platform checks.
*/
preflightCheckTasks(flags: any, command: Command): Listr {
const kube = new KubeHelper(flags)
return new Listr([
{
title: 'Verify if kubectl is installed',
Expand Down Expand Up @@ -79,6 +81,49 @@ export class MinikubeTasks {
task.title = `${task.title}...${flags.domain}.`
}
},
{
title: 'Checking minikube version',
task: async (ctx: any, task: any) => {
const version = await this.getMinikbeVersion()
const versionComponents = version.split('.')
ctx.minikubeVersionMajor = parseInt(versionComponents[0], 10)
ctx.minikubeVersionMinor = parseInt(versionComponents[1], 10)
ctx.minikubeVersionPatch = parseInt(versionComponents[2], 10)

task.title = `${task.title}... ${version}`
}
},
{
// Starting from Minikube 1.9 there is a bug with storage provisioner which prevents Che from successful deployment.
// For more details see https://github.com/kubernetes/minikube/issues/7218
// To workaround the bug, it is required to patch storage provisioner as well as its permissions.
title: 'Patch minikube storage',
enabled: ctx => ctx.minikubeVersionMajor && ctx.minikubeVersionMinor &&
ctx.minikubeVersionMajor === 1 && ctx.minikubeVersionMinor >= 9 && ctx.minikubeVersionMinor <= 11,
Copy link
Collaborator

@l0rd l0rd Jun 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need to check that the version is <= 11? I mean I would set the upper bound only when we know which version of minikube will contain the fix. Otherwise we will get a blocker as soon as 1.12 is out.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is sort of a template when the bug is fixed. Seems there are some activity in the Minikube issue, so I hoped that it will be fixed soon. But agree, we'll add stop condition when Minikube fix arrives.

task: async (_ctx: any, task: any) => {
// Patch storage provisioner pod to the latest version
const storageProvisionerImagePatch = {
apiVersion: 'v1',
kind: 'Pod',
spec: {
containers: [
{ name: 'storage-provisioner', image: 'gcr.io/k8s-minikube/storage-provisioner:latest' }
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of latest here we should use use the digest (or a version) to avoid that an update to latest breaks things.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, I will use digest as there is no tag (if don't count latest) for the images above v1.8.1 which is 2 years old.

]
}
}
if (! await kube.patchNamespacedPod('storage-provisioner', 'kube-system', storageProvisionerImagePatch)) {
throw new Error('Failed to patch storage provisioner image')
}

// Set required permissions for cluster role of persistent volume provisioner
if (! await kube.addClusterRoleRule('system:persistent-volume-provisioner',
[''], ['endpoints'], ['get', 'list', 'watch', 'create', 'patch', 'update'])) {
throw new Error('Failed to patch permissions for persistent-volume-provisioner')
}

task.title = `${task.title}... done`
}
},
CommonPlatformTasks.getPingClusterTask(flags)
], { renderer: flags['listr-renderer'] as any })
}
Expand Down Expand Up @@ -106,4 +151,11 @@ export class MinikubeTasks {
return stdout
}

async getMinikbeVersion(): Promise<string> {
const { stdout } = await execa('minikube', ['version'], { timeout: 10000 })
const versionLine = stdout.split('\n')[0]
const versionString = versionLine.trim().split(' ')[2].substr(1)
return versionString
}

}