Skip to content

Commit

Permalink
some manifest tuning for deployment in azure.
Browse files Browse the repository at this point in the history
  • Loading branch information
@andyliuliming @svrc
andyliuliming authored and svrc committed Aug 25, 2018
1 parent 537d947 commit ec74225
Show file tree
Hide file tree
Showing 24 changed files with 755 additions and 298 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
.DS_Store
bin/go-outline
bin/gopkgs
pkg
!/src/kubo-deployment-tests/resources/environments/test_gcp_with_creds/creds.yml
*.iml
*.terraform
*.tfstate
*.backup
src/github.com/uudashr/gopkgs
5 changes: 4 additions & 1 deletion bin/deploy_bosh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,10 @@ main() {
"$(repo_directory)/bin/set_bosh_alias" "${bosh_env}"

BOSH_ENV="$bosh_env" source "$(repo_directory)/bin/set_bosh_environment"
bosh update-runtime-config -n "$(repo_directory)/bosh-deployment/runtime-configs/dns.yml"

if [[ ! "azure" == "${iaas}" ]]; then
bosh update-runtime-config -n "$(repo_directory)/bosh-deployment/runtime-configs/dns.yml"
fi

export_bosh_environment "${bosh_env}"
set_cloud_config
Expand Down
10 changes: 8 additions & 2 deletions bin/lib/deploy_utils
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,8 @@ generate_manifest() {
if ! bosh int "${director_path}" --path='/vcenter_rp' &>/dev/null; then
interpolate_arguments+=" --ops-file=\"${ops_dir}/iaas/vsphere/set-working-dir-no-rp.yml\""
fi
elif [[ "azure" == "${iaas}" ]]; then
interpolate_arguments+=" --ops-file=\"${ops_dir}/vm-types.yml\""
fi

if [[ -z "$existing_deployment" ]]; then
Expand All @@ -247,15 +249,19 @@ generate_manifest() {
interpolate_arguments+=" --ops-file=\"${ops_dir}/add-oidc-endpoint.yml\""
fi

if [[ ! "azure" == "${iaas}" ]]; then
interpolate_arguments+=" --ops-file=\"${ops_dir}/use-runtime-config-bosh-dns.yml\""
fi

set +x
local manifest=$(bosh int <(echo "${manifest}") --ops-file="${ops_dir}/misc/dev.yml" \
--ops-file="${ops_dir}/misc/bootstrap.yml" \
--ops-file="${ops_dir}/misc/single-master.yml" \
--ops-file="${ops_dir}/use-runtime-config-bosh-dns.yml" \
--vars-file="${director_path}" \
--var deployment_name="${deployment_name}" \
--var director_uuid="${director_uuid}" \
${interpolate_arguments})
${interpolate_arguments} \
${KUBO_EXTRA_OPS})
restore_debug_output

printf "%s" "${manifest}"
Expand Down
6 changes: 5 additions & 1 deletion configurations/azure/project-config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ resource_group_name: # Azure Resource Group for the cluster
vnet_resource_group_name: # Azure Resource Group for the vnet/subnet
vnet_name: # Azure VNet for BOSH
subnet_name: # Azure Subnet name
default_security_group: # Azure NSG attached to the subnet
default_security_group: # Azure NSG attached to the subnet
allow_privileged_containers: # Whether privileged containers allowed.
disable_deny_escalating_exec: # Whether we should disable deny escalating exec.
azure_cloud_name: AzurePublicCloud # can also be AzureUSGovernmentCloud, AzureChinaCloud, or AzureGermanCloud.
dns_recursor_ip: 168.63.129.16
prefix: cfcr # unique prefix for availability sets
Expand All @@ -14,4 +16,6 @@ routing_mode: iaas
kubernetes_master_host: # IP address that can be used to reach the Kubernetes API, usually the address of a load balancer or static IP
kubernetes_master_port: 8443 # Port for the K8S API
master_target_pool: # Azure load balancer name that master nodes will be a part of
master_vm_type: # vm type for master node
worker_vm_type: # vm type for worker node

27 changes: 27 additions & 0 deletions docs/terraform/azure/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
## How to Deploy CFCR

### Step 1: deploy bastion vm.
terraform apply -var prefix="YOUR PREFIX FOR RESOURCES" -var subscription_id="YOUR SUBSCRIPTION ID" -var client_id="YOUR SERVICE PRINCIPAL ID" -var client_secret="YOUR SERVICE PRINCIPAL PASSWORD" -var tenant_id="YOUR TENANT ID" -var ssh_user_username="cfuser" -var ssh_public_key_filename="./ssh_key.pub"

### Step 2: connec to the bastion vm, and run.
a. deploy the bosh director:
cd /share/kubo-deployment
export kubo_envs=~/kubo-env
export kubo_env_name=kubo
export kubo_env_path="${kubo_envs}/${kubo_env_name}"
mkdir -p "${kubo_envs}"
./bin/generate_env_config "${kubo_envs}" "${kubo_env_name}" azure
/usr/bin/update_azure_env "${kubo_env_path}/director.yml"
/usr/bin/update_azure_secrets "${kubo_env_path}/director-secrets.yml"
/usr/bin/set_iaas_routing "${kubo_env_path}/director.yml"
/share/kubo-deployment/bin/deploy_bosh "${kubo_env_path}"

b. deploy the cfcr
BOSH_ENV=${kubo_env_path}
source /share/kubo-deployment/bin/set_bosh_environment
./bin/deploy_k8s ~/kubo-env/kubo my-cluster
./bin/set_kubeconfig ~/kubo-env/kubo my-cluster
kubectl get pods --namespace=kube-system

c. run the test app
kubectl create -f filepath to do the test.
221 changes: 51 additions & 170 deletions ...orm/azure/platform/kubo-infrastructure.tf → docs/terraform/azure/bastion.tf
View file
Original file line number Diff line number Diff line change
@@ -1,185 +1,63 @@


variable "subscription_id" {}

variable "tenant_id" {}

variable "client_id" {}

variable "client_secret" {}

variable "latest_ubuntu" {
type = "map"
default = {
publisher = "Canonical"
offer = "UbuntuServer"
sku = "14.04.5-LTS"
version = "latest"
}
}

variable "ssh_user_username" {
type = "string"
default = "ubuntu"
}

variable "ssh_private_key_filename" {
type = "string"
}

variable "ssh_public_key_filename" {
type = "string"
}

variable "location" {
type = "string"
default = "eastus2"
}

variable "prefix" {
type = "string"
}

variable "network_cidr" {
default = "10.0.0.0/16"
}

provider "azurerm" {
subscription_id = "${var.subscription_id}"
tenant_id = "${var.tenant_id}"
client_id = "${var.client_id}"
client_secret = "${var.client_secret}"
}

resource "azurerm_resource_group" "bosh" {
name = "${var.prefix}-cfcr"
location = "${var.location}"

tags {
environment = "${var.prefix}-cfcr"
}
}

resource "azurerm_public_ip" "bosh-bastion" {
name = "${var.prefix}-cfcr-ip"
resource "azurerm_public_ip" "bastion" {
name = "${var.prefix}-bastion-ip"
location = "${var.location}"
depends_on = ["azurerm_resource_group.bosh"]
resource_group_name = "${azurerm_resource_group.bosh.name}"
depends_on = ["azurerm_resource_group.rg"]
resource_group_name = "${azurerm_resource_group.rg.name}"
public_ip_address_allocation = "static"
sku = "Standard"

tags {
environment = "${var.prefix}-cfcr"
}
}

// Subnet for CFCR
resource "azurerm_virtual_network" "cfcr-vnet" {
name = "${var.prefix}-cfcr-vnet"
location = "${var.location}"
depends_on = ["azurerm_resource_group.bosh"]

resource_group_name = "${azurerm_resource_group.bosh.name}"
address_space = ["${var.network_cidr}"]
dns_servers = ["168.63.129.16"]

}

resource "azurerm_subnet" "cfcr-subnet" {
name = "cfcr-subnet"
depends_on = ["azurerm_virtual_network.cfcr-vnet"]

resource_group_name = "${azurerm_resource_group.bosh.name}"
virtual_network_name = "${azurerm_virtual_network.cfcr-vnet.name}"
address_prefix = "${cidrsubnet(var.network_cidr, 8, 0)}"
}


// Allow SSH to BOSH bastion
resource "azurerm_network_security_group" "bosh-bastion" {
name = "${var.prefix}bosh-bastion"
location = "${var.location}"
resource_group_name = "${azurerm_resource_group.bosh.name}"
depends_on = ["azurerm_resource_group.bosh"]
security_rule {
name = "ssh"
priority = 100
direction = "Inbound"
access = "Allow"
protocol = "Tcp"
source_port_range = "*"
destination_port_range = "22"
source_address_prefix = "*"
destination_address_prefix = "*"
}
}

// Allow port 8443 to master
resource "azurerm_network_security_group" "cfcr-master" {
name = "${var.prefix}cfcr-master"
location = "${var.location}"
resource_group_name = "${azurerm_resource_group.bosh.name}"
depends_on = ["azurerm_resource_group.bosh"]
security_rule {
name = "master"
priority = 100
direction = "Inbound"
access = "Allow"
protocol = "Tcp"
source_port_range = "*"
destination_port_range = "8443"
source_address_prefix = "*"
destination_address_prefix = "*"
environment = "${module.variables.environment-tag}"
}
}


// BOSH bastion host

resource "azurerm_network_interface" "bosh-bastion" {
name = "${var.prefix}bosh-bastion-nic"
depends_on = ["azurerm_public_ip.bosh-bastion", "azurerm_subnet.cfcr-subnet", "azurerm_network_security_group.bosh-bastion"]
resource "azurerm_network_interface" "bastion" {
name = "${var.prefix}-bastion-nic"
depends_on = ["azurerm_public_ip.bastion", "azurerm_subnet.bosh-subnet", "azurerm_network_security_group.bastion"]
location = "${var.location}"
resource_group_name = "${azurerm_resource_group.bosh.name}"
network_security_group_id = "${azurerm_network_security_group.bosh-bastion.id}"
resource_group_name = "${azurerm_resource_group.rg.name}"
network_security_group_id = "${azurerm_network_security_group.bastion.id}"

ip_configuration {
name = "${var.prefix}-bosh-bastion-ip-config"
subnet_id = "${azurerm_subnet.cfcr-subnet.id}"
name = "${var.prefix}-bastion-ip-config"
subnet_id = "${azurerm_subnet.bosh-subnet.id}"
private_ip_address_allocation = "static"
private_ip_address = "${cidrhost(azurerm_subnet.cfcr-subnet.address_prefix,4)}"
public_ip_address_id = "${azurerm_public_ip.bosh-bastion.id}"
private_ip_address = "${cidrhost(azurerm_subnet.bosh-subnet.address_prefix,100)}"
public_ip_address_id = "${azurerm_public_ip.bastion.id}"
}
}


resource "azurerm_virtual_machine" "bosh-bastion" {
name = "${var.prefix}bosh-bastion"
depends_on = ["azurerm_network_interface.bosh-bastion"]
vm_size = "Standard_D2_V2"
location = "${var.location}"
resource_group_name = "${azurerm_resource_group.bosh.name}"
network_interface_ids = ["${azurerm_network_interface.bosh-bastion.id}"]
resource "azurerm_virtual_machine" "bastion" {
name = "${var.prefix}-bastion"
depends_on = ["azurerm_network_interface.bastion"]
vm_size = "Standard_D1_v2"
location = "${var.location}"
resource_group_name = "${azurerm_resource_group.rg.name}"
network_interface_ids = ["${azurerm_network_interface.bastion.id}"]
storage_image_reference = ["${var.latest_ubuntu}"]

storage_os_disk {
name = "osdisk1"
caching = "ReadWrite"
create_option = "FromImage"
managed_disk_type = "Standard_LRS"
disk_size_gb = "100"
disk_size_gb = "50"
}

os_profile_linux_config {
disable_password_authentication = true

ssh_keys = [{
path = "/home/${var.ssh_user_username}/.ssh/authorized_keys"
key_data = "${file(var.ssh_public_key_filename)}"
}]
}

os_profile {
computer_name = "bosh-bastion"
os_profile {
computer_name = "bastion"
admin_username = "${var.ssh_user_username}"

custom_data = <<EOT
#!/bin/bash
cat > /etc/motd <<EOF
Expand All @@ -206,21 +84,25 @@ tar -zxvf /tmp/cf.tgz && mv cf /usr/bin/cf && chmod +x /usr/bin/cf
cat > /etc/profile.d/bosh.sh <<'EOF'
#!/bin/bash
# Misc vars
export prefix=${var.prefix}
export ssh_key_path=$HOME/.ssh/bosh
# Vars from Terraform
export resource_group=${azurerm_resource_group.bosh.name}
export subnetwork=${azurerm_subnet.cfcr-subnet.name}
export network=${azurerm_virtual_network.cfcr-vnet.name}
export subnet_ip_prefix=${azurerm_subnet.cfcr-subnet.address_prefix}
export subscription_id=${var.subscription_id}
export tenant_id=${var.tenant_id}
export client_id=${var.client_id}
export client_secret=${var.client_secret}1
export client_secret=${var.client_secret}
export resource_group_name=${azurerm_resource_group.rg.name}
export vnet_name=${module.variables.vnet-name}
export cfcr_subnet_name=${azurerm_subnet.cfcr-subnet.name}
export cfcr_master_sg_name=${azurerm_network_security_group.cfcr-master.name}
export cfcr_subnet_address_range=${azurerm_subnet.cfcr-subnet.address_prefix}
export cfcr_internal_ip=${cidrhost(azurerm_subnet.cfcr-subnet.address_prefix, 5)}
export cfcr_internal_gw=${cidrhost(azurerm_subnet.cfcr-subnet.address_prefix, 1)}
export location=${var.location}
export bosh_director_name=${var.bosh_director_name}
export subscription_id=${var.subscription_id}
export kubernetes_master_host=${azurerm_public_ip.cfcr-balancer-ip.ip_address}
export kubernetes_master_port=${var.kubernetes_master_port}
export master_target_pool=${azurerm_lb.cfcr-balancer.name}
export allow_privileged_containers=${var.allow_privileged_containers}
export disable_deny_escalating_exec=${var.disable_deny_escalating_exec}
EOF
cat > /usr/bin/update_azure_env <<'EOF'
Expand All @@ -232,12 +114,16 @@ if [[ ! -f "$1" ]] || [[ ! "$1" =~ director.yml$ ]]; then
fi
# Azure specific updates
sed -i -e 's/^\(resource_group_name:\).*\(#.*\)/\1 ${azurerm_resource_group.bosh.name} \2/' "$1"
sed -i -e 's/^\(vnet_resource_group_name:\).*\(#.*\)/\1 ${azurerm_resource_group.bosh.name} \2/' "$1"
sed -i -e 's/^\(vnet_name:\).*\(#.*\)/\1 ${azurerm_virtual_network.cfcr-vnet.name} \2/' "$1"
sed -i -e 's/^\(resource_group_name:\).*\(#.*\)/\1 ${azurerm_resource_group.rg.name} \2/' "$1"
sed -i -e 's/^\(vnet_resource_group_name:\).*\(#.*\)/\1 ${azurerm_resource_group.rg.name} \2/' "$1"
sed -i -e 's/^\(vnet_name:\).*\(#.*\)/\1 ${azurerm_virtual_network.vnet.name} \2/' "$1"
sed -i -e 's/^\(subnet_name:\).*\(#.*\)/\1 ${azurerm_subnet.cfcr-subnet.name} \2/' "$1"
sed -i -e 's/^\(location:\).*\(#.*\)/\1 ${var.location} \2/' "$1"
sed -i -e 's/^\(default_security_group:\).*\(#.*\)/\1 ${azurerm_network_security_group.cfcr-master.name} \2/' "$1"
sed -i -e 's/^\(master_vm_type:\).*\(#.*\)/\1 'master' \2/' "$1"
sed -i -e 's/^\(worker_vm_type:\).*\(#.*\)/\1 'worker' \2/' "$1"
sed -i -e 's/^\(allow_privileged_containers:\).*\(#.*\)/\1 ${var.allow_privileged_containers} \2/' "$1"
sed -i -e 's/^\(disable_deny_escalating_exec:\).*\(#.*\)/\1 ${var.disable_deny_escalating_exec} \2/' "$1"
# Generic updates
sed -i -e 's/^\(internal_ip:\).*\(#.*\)/\1 ${cidrhost(azurerm_subnet.cfcr-subnet.address_prefix, 5)} \2/' "$1"
Expand Down Expand Up @@ -285,7 +171,7 @@ EOF
chmod a+x /usr/bin/set_iaas_routing
# Get kubo-deployment
wget https://s3.amazonaws.com/scharlton-piv/kubo-deployment-latest.tgz
wget https://opensourcerelease.blob.core.windows.net/alphareleases/kubo-deployment-latest.tgz
mkdir /share
tar -xvf kubo-deployment-latest.tgz -C /share
chmod -R 777 /share
Expand All @@ -305,10 +191,5 @@ chmod a+x credhub
sudo mv credhub /usr/bin
EOT
}
}

output "kubo_subnet" {
value = "${azurerm_subnet.cfcr-subnet.name}"
}
}

Loading

0 comments on commit ec74225

Please sign in to comment.