Skip to content

Commit

Permalink
Private Commit for Azure Console Shell
Browse files Browse the repository at this point in the history
Remove SPN secrets from agent node
Remove the Kube Dashboard and Heapster Addons
Add agentpool label on the agent nodes
Use static IP address for system and agentpool1
  • Loading branch information
robbiezhang committed May 9, 2017
1 parent cb47749 commit 87c56c3
Show file tree
Hide file tree
Showing 15 changed files with 225 additions and 218 deletions.
7 changes: 3 additions & 4 deletions parts/kubernetesagentcustomdata.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ write_files:
encoding: gzip
owner: "root"
content: !!binary |
KUBELET_SERVICE_B64_GZIP_STR
KUBELET_SERVICE_AGENT_B64_GZIP_STR

- path: "/opt/azure/containers/kubelet.sh"
permissions: "0755"
Expand All @@ -136,7 +136,7 @@ write_files:
encoding: gzip
owner: "root"
content: !!binary |
{{WrapAsVariable "provisionScript"}}
{{WrapAsVariable "agentProvisionScript"}}

runcmd:
- apt-get update
Expand All @@ -153,5 +153,4 @@ runcmd:
- apt-get install -y docker-engine
- systemctl restart docker
- mkdir -p /etc/kubernetes/manifests
- usermod -aG docker {{WrapAsVariable "username"}}

- usermod -aG docker {{WrapAsVariable "username"}}
98 changes: 98 additions & 0 deletions parts/kubernetesagentcustomscript.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/bin/bash

###########################################################
# START SECRET DATA - ECHO DISABLED
###########################################################

# Fields for `azure.json`
KUBELET_PRIVATE_KEY="${1}"
NETWORK_POLICY="${2}"

KUBELET_PRIVATE_KEY_PATH="/etc/kubernetes/certs/client.key"
touch "${KUBELET_PRIVATE_KEY_PATH}"
chmod 0644 "${KUBELET_PRIVATE_KEY_PATH}"
chown root:root "${KUBELET_PRIVATE_KEY_PATH}"
echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}"

###########################################################
# END OF SECRET DATA
###########################################################

set -x

function ensureDocker() {
systemctl enable docker
systemctl restart docker
dockerStarted=1
for i in {1..600}; do
if ! /usr/bin/docker info; then
echo "status $?"
/bin/systemctl restart docker
else
echo "docker started"
dockerStarted=0
break
fi
sleep 1
done
if [ $dockerStarted -ne 0 ]
then
echo "docker did not start"
exit 1
fi
}

function setAgentPool() {
AGENTPOOL=`hostname | cut -d- -f2`
sed -i "s/^KUBELET_NODE_LABELS=.*/KUBELET_NODE_LABELS=role=agent,agentpool=${AGENTPOOL}/" /etc/default/kubelet
}

function ensureKubelet() {
systemctl enable kubelet
systemctl restart kubelet
}

function setNetworkPlugin () {
sed -i "s/^KUBELET_NETWORK_PLUGIN=.*/KUBELET_NETWORK_PLUGIN=${1}/" /etc/default/kubelet
}

function setDockerOpts () {
sed -i "s#^DOCKER_OPTS=.*#DOCKER_OPTS=${1}#" /etc/default/kubelet
}

function configNetworkPolicy() {
if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then
# on masters
ADDONS="calico-configmap.yaml calico-daemonset.yaml"
ADDONS_PATH=/etc/kubernetes/addons
CALICO_URL="https://github.com/simonswine/calico/raw/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager"
if [[ "${NETWORK_POLICY}" = "calico" ]]; then
# download calico yamls
for addon in ${ADDONS}; do
curl -o "${ADDONS_PATH}/${addon}" -sSL --retry 12 --retry-delay 10 "${CALICO_URL}/${addon}"
done
else
# make sure calico yaml are removed
for addon in ${ADDONS}; do
rm -f "${ADDONS_PATH}/${addon}"
done
fi
else
# on agents
if [[ "${NETWORK_POLICY}" = "calico" ]]; then
setNetworkPlugin cni
setDockerOpts " --volume=/etc/cni/:/etc/cni:ro --volume=/opt/cni/:/opt/cni:ro"
else
setNetworkPlugin kubenet
setDockerOpts ""
fi
fi
}

ensureDocker
configNetworkPolicy
setAgentPool
ensureKubelet

echo "Install complete successfully"

51 changes: 51 additions & 0 deletions parts/kubernetesagentkubelet.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
[Unit]
Description=Kubelet
Requires=docker.service
After=docker.service

[Service]
Restart=always
EnvironmentFile=/etc/default/kubelet
SuccessExitStatus=143
ExecStartPre=/bin/bash /opt/azure/containers/kubelet.sh
ExecStartPre=/bin/mkdir -p /var/lib/kubelet
ExecStartPre=/bin/bash -c "if [ $(mount | grep \"/var/lib/kubelet\" | wc -l) -le 0 ] ; then /bin/mount --bind /var/lib/kubelet /var/lib/kubelet ; fi"
ExecStartPre=/bin/mount --make-shared /var/lib/kubelet
ExecStartPre=-/sbin/ebtables -t nat --list
ExecStartPre=-/sbin/iptables -t nat --list
ExecStart=/usr/bin/docker run \
--net=host \
--pid=host \
--privileged \
--rm \
--volume=/dev:/dev \
--volume=/sys:/sys:ro \
--volume=/var/run:/var/run:rw \
--volume=/var/lib/docker/:/var/lib/docker:rw \
--volume=/var/lib/kubelet/:/var/lib/kubelet:shared \
--volume=/var/log:/var/log:rw \
--volume=/etc/kubernetes/:/etc/kubernetes:ro \
--volume=/srv/kubernetes/:/srv/kubernetes:ro $DOCKER_OPTS \
${KUBELET_IMAGE} \
/hyperkube kubelet \
--kubeconfig=/var/lib/kubelet/kubeconfig \
--require-kubeconfig \
--pod-infra-container-image="${KUBELET_POD_INFRA_CONTAINER_IMAGE}" \
--address=0.0.0.0 \
--allow-privileged=true \
--enable-server \
--enable-debugging-handlers \
--pod-manifest-path=/etc/kubernetes/manifests \
--cluster-dns=${KUBELET_CLUSTER_DNS} \
--cluster-domain=cluster.local \
--register-schedulable=${KUBELET_REGISTER_SCHEDULABLE} \
--node-labels=${KUBELET_NODE_LABELS} \
--cloud-provider= \
--cloud-config= \
--azure-container-registry-config= \
--hairpin-mode=promiscuous-bridge \
--network-plugin=${KUBELET_NETWORK_PLUGIN} \
--v=2

[Install]
WantedBy=multi-user.target
10 changes: 9 additions & 1 deletion parts/kubernetesagentresourcesvmas.t
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,15 @@
{{if eq $seq 1}}
"primary": true,
{{end}}
{{if eq $.Name "system"}}
"privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(50, int(variables('masterFirstAddrOctet4')))))]",
"privateIPAllocationMethod": "Static",
{{else if eq $.Name "agentpool1"}}
"privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(100, int(variables('masterFirstAddrOctet4')))))]",
"privateIPAllocationMethod": "Static",
{{else}}
"privateIPAllocationMethod": "Dynamic",
{{end}}
"subnet": {
"id": "[variables('{{$.Name}}VnetSubnetID')]"
}
Expand Down Expand Up @@ -205,7 +213,7 @@
"autoUpgradeMinorVersion": true,
"settings": {},
"protectedSettings": {
"commandToExecute": "[concat('/usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh ',variables('tenantID'),' ',variables('subscriptionId'),' ',variables('resourceGroup'),' ',variables('location'),' ',variables('subnetName'),' ',variables('nsgName'),' ',variables('virtualNetworkName'),' ',variables('routeTableName'),' ',variables('primaryAvailablitySetName'),' ',variables('servicePrincipalClientId'),' ',variables('servicePrincipalClientSecret'),' ',variables('clientPrivateKey'),' ',variables('targetEnvironment'),' ',variables('networkPolicy'),' >> /var/log/azure/cluster-provision.log 2>&1 &\" &')]"
"commandToExecute": "[concat('/usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh ',variables('clientPrivateKey'),' ',variables('networkPolicy'),'>> /var/log/azure/agent-provision.log 2>&1 &\" &')]"
}
}
}
1 change: 1 addition & 0 deletions parts/kubernetesagentvars.t
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,4 @@
"{{.Name}}VnetSubnetID": "[variables('vnetSubnetID')]",
"{{.Name}}SubnetName": "[variables('subnetName')]",
{{end}}
"agentProvisionScript": "{{GetKubernetesAgentB64Provision}}",
70 changes: 0 additions & 70 deletions parts/kubernetesmasteraddons-heapster-deployment.yaml

This file was deleted.

14 changes: 0 additions & 14 deletions parts/kubernetesmasteraddons-heapster-service.yaml

This file was deleted.

3 changes: 2 additions & 1 deletion parts/kubernetesmasteraddons-kube-dns-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -100,4 +100,5 @@ spec:
memory: 50Mi
dnsPolicy: Default
nodeSelector:
beta.kubernetes.io/os: linux
beta.kubernetes.io/os: linux
agentpool: system
15 changes: 0 additions & 15 deletions parts/kubernetesmasteraddons-kubernetes-dashboard-service.yaml

This file was deleted.

30 changes: 0 additions & 30 deletions parts/kubernetesmastercustomdata.yml
Original file line number Diff line number Diff line change
Expand Up @@ -152,34 +152,6 @@ write_files:
content: !!binary |
MASTER_ADDON_KUBE_PROXY_DAEMONSET_B64_GZIP_STR

- path: /etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml
permissions: "0644"
encoding: gzip
owner: "root"
content: !!binary |
MASTER_ADDON_KUBERNETES_DASHBOARD_DEPLOYMENT_B64_GZIP_STR

- path: /etc/kubernetes/addons/kubernetes-dashboard-service.yaml
permissions: "0644"
encoding: gzip
owner: "root"
content: !!binary |
MASTER_ADDON_KUBERNETES_DASHBOARD_SERVICE_B64_GZIP_STR

- path: /etc/kubernetes/addons/kube-heapster-service.yaml
permissions: "0644"
encoding: gzip
owner: "root"
content: !!binary |
MASTER_ADDON_HEAPSTER_SERVICE_B64_GZIP_STR

- path: /etc/kubernetes/addons/kube-heapster-deployment.yaml
permissions: "0644"
encoding: gzip
owner: "root"
content: !!binary |
MASTER_ADDON_HEAPSTER_DEPLOYMENT_B64_GZIP_STR

- path: /etc/kubernetes/addons/default-storage-class.yaml
permissions: "0644"
encoding: gzip
Expand Down Expand Up @@ -248,8 +220,6 @@ write_files:
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
sed -i "s|<kubernetesKubeDNSSpec>|{{WrapAsVariable "kubernetesKubeDNSSpec"}}|g; s|<kubernetesDNSMasqSpec>|{{WrapAsVariable "kubernetesDNSMasqSpec"}}|g; s|<kubernetesExecHealthzSpec>|{{WrapAsVariable "kubernetesExecHealthzSpec"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
sed -i "s|<kubernetesHeapsterSpec>|{{WrapAsVariable "kubernetesHeapsterSpec"}}|g; s|<kubernetesAddonResizerSpec>|{{WrapAsVariable "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml"
sed -i "s|<kubernetesDashboardSpec>|{{WrapAsVariable "kubernetesDashboardSpec"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
echo $(curl -f --retry 5 http://169.254.169.254/metadata/v1/InstanceInfo 2>/dev/null | jq -r .FD) > /etc/kubernetes/fd
echo "{{WrapAsVariable "masterVMSize"}}" > /etc/kubernetes/vmsize
Expand Down
1 change: 0 additions & 1 deletion parts/kubernetesmastercustomscript.sh
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,5 @@ if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then
ensureApiserver
fi

# If APISERVER_PRIVATE_KEY is empty, then we are not on the master
echo "Install complete successfully"

2 changes: 1 addition & 1 deletion parts/kubernetesmastervars.t
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@
{{end}}
"nsgName": "[concat(variables('masterVMNamePrefix'), 'nsg')]",
"nsgID": "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]",
"primaryAvailablitySetName": "[concat('{{ (index .AgentPoolProfiles 0).Name }}-availabilitySet-',variables('nameSuffix'))]",
"primaryAvailablitySetName": "[concat('{{ (index .AgentPoolProfiles 1).Name }}-availabilitySet-',variables('nameSuffix'))]",
"masterPublicIPAddressName": "[concat(variables('orchestratorName'), '-master-ip-', variables('masterFqdnPrefix'), '-', variables('nameSuffix'))]",
"masterLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterLbName'))]",
"masterLbIPConfigID": "[concat(variables('masterLbID'),'/frontendIPConfigurations/', variables('masterLbIPConfigName'))]",
Expand Down
2 changes: 1 addition & 1 deletion pkg/acsengine/azureconst.go
Original file line number Diff line number Diff line change
Expand Up @@ -1110,4 +1110,4 @@ func GetClassicSizeMap() string {
}
}
`
}
}

0 comments on commit 87c56c3

Please sign in to comment.