diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..7108a8a --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,199 @@ +# Kubernetes 集群模块 + +本模块用于在七牛云上一键部署 Kubernetes 集群。 + +## 模块说明 + +### simple - 简单集群 + +单 Master 节点 + 多 Worker 节点的 K8s 集群配置，适用于开发、测试和小型生产环境。 + +## 功能特性 + +- ✅ 自动安装和配置 Kubernetes（支持指定版本） +- ✅ 自动配置容器运行时（containerd） +- ✅ 支持多种 CNI 插件（Flannel、Calico、Weave） +- ✅ 自动生成 bootstrap token +- ✅ Worker 节点自动加入集群 +- ✅ 使用置放组确保节点分散部署 + +## 使用示例 + +### 基本用法 + +```hcl +module "k8s_cluster" { + source = "./k8s/simple" + + # 实例配置 + instance_type = "ecs.t1.c2m4" # 2核4G + instance_system_disk_size = 40 # 40GB 系统盘 + + # K8s 配置 + k8s_version = "1.28.0" # K8s 版本 + worker_count = 2 # Worker 节点数量 + pod_network_cidr = "10.244.0.0/16" # Pod 网络 CIDR + service_cidr = "10.96.0.0/12" # Service 网络 CIDR + cni_plugin = "flannel" # CNI 插件 +} + +# 输出集群信息 +output "master_ip" { + value = module.k8s_cluster.k8s_master_ip +} + +output "master_endpoint" { + value = module.k8s_cluster.k8s_master_endpoint +} + +output "worker_ips" { + value = module.k8s_cluster.k8s_worker_ips +} + +output "cluster_info" { + value = module.k8s_cluster.cluster_info +} +``` + +### 高级配置 + +```hcl +module "k8s_cluster_prod" { + source = "./k8s/simple" + + # 使用更高配置的实例 + instance_type = "ecs.t1.c4m8" # 4核8G + instance_system_disk_size = 80 # 80GB 系统盘 + + # 更多 Worker 节点 + worker_count = 5 + + # 使用 Calico CNI + cni_plugin = "calico" +} +``` + +## 变量说明 + +### 通用变量（common_variables.tf） + +| 变量名 | 类型 | 默认值 | 说明 | +|--------|------|--------|------| +| `instance_type` | string | `ecs.t1.c2m4` | 实例规格 | +| `instance_system_disk_size` | number | `40` | 系统盘大小（GiB），最小 20GB | + +### K8s 特定变量（k8s_variables.tf） + +| 变量名 | 类型 | 默认值 | 说明 | +|--------|------|--------|------| +| `k8s_version` | string | `1.28.0` | Kubernetes 版本 | +| `worker_count` | number | `2` | Worker 节点数量（1-10） | +| `pod_network_cidr` | string | `10.244.0.0/16` | Pod 网络 CIDR | +| `service_cidr` | string | `10.96.0.0/12` | Service 网络 CIDR | +| `cni_plugin` | string | `flannel` | CNI 插件（flannel/calico/weave） | + +## 输出说明 + +| 输出名 | 说明 | +|--------|------| +| `k8s_master_endpoint` | Kubernetes API Server 地址 | +| `k8s_master_ip` | Master 节点 IP | +| `k8s_master_password` | Master 节点 SSH 密码（敏感） | +| `k8s_worker_ips` | Worker 节点 IP 列表 | +| `k8s_worker_passwords` | Worker 节点 SSH 密码映射（敏感） | +| `k8s_bootstrap_token` | K8s bootstrap token（敏感） | +| `cluster_info` | 集群信息汇总 | +| `kubeconfig_command` | 获取 kubeconfig 的命令 | + +## 获取 kubeconfig + +集群创建完成后，使用以下方法获取 kubeconfig： + +```bash +# 方法 1：使用输出的命令 +terraform output -raw kubeconfig_command | bash + +# 方法 2：直接 SSH 到 master 节点 +ssh root@ 'cat /etc/kubernetes/admin.conf' > kubeconfig.yaml + +# 方法 3：在 master 节点上查看 +ssh root@ +cat /etc/kubernetes/admin.conf +``` + +然后设置 KUBECONFIG 环境变量： + +```bash +export KUBECONFIG=./kubeconfig.yaml +kubectl get nodes +``` + +## 最小配置要求 + +- **Master 节点**: 至少 2C4G（ecs.t1.c2m4） +- **Worker 节点**: 至少 2C4G（ecs.t1.c2m4） +- **系统盘**: 至少 20GB（推荐 40GB） + +## 网络要求 + +- Master 和 Worker 节点必须在同一 VPC/子网 +- 需要开放以下端口： + - **Master**: 6443 (API Server), 2379-2380 (etcd), 10250-10252 + - **Worker**: 10250 (kubelet), 30000-32767 (NodePort) + +## 注意事项 + +1. **初始化时间**: 集群初始化大约需要 5-10 分钟 +2. **网络连接**: 需要稳定的外网连接下载 K8s 组件和镜像 +3. **资源清理**: 删除集群前，请先删除所有 K8s 资源（PV、LoadBalancer 等） +4. **安全性**: 生产环境建议修改默认配置，加强安全防护 +5. **证书管理**: K8s 证书默认 1 年有效期，注意续期 + +## 故障排查 + +### 查看初始化日志 + +```bash +# Master 节点 +ssh root@ +journalctl -u kubelet -f + +# Worker 节点 +ssh root@ +journalctl -u kubelet -f +``` + +### 检查集群状态 + +```bash +# 在 master 节点上 +kubectl get nodes +kubectl get pods -A +``` + +### 重新加入 Worker 节点 + +如果 Worker 节点加入失败，可以手动重新加入： + +```bash +# 在 master 节点上生成 join 命令 +kubeadm token create --print-join-command + +# 在 worker 节点上执行该命令 +``` + +## 支持的 CNI 插件 + +- **Flannel**: 简单易用，默认选项，适合大多数场景 +- **Calico**: 功能强大，支持网络策略，适合安全要求高的场景 +- **Weave**: 轻量级，支持加密，适合跨云部署 + +## 版本兼容性 + +- Terraform: >= 0.12.0 +- Qiniu Provider: ~> 1.0.0 +- Kubernetes: 1.28.x（可配置其他版本） + +## 许可证 + +本模块遵循 MIT 许可证。 diff --git a/k8s/common/common_variables.tf b/k8s/common/common_variables.tf new file mode 100644 index 0000000..b510334 --- /dev/null +++ b/k8s/common/common_variables.tf @@ -0,0 +1,39 @@ +variable "instance_type" { + type = string + description = "K8s instance type" + default = "ecs.t1.c2m4" + validation { + condition = var.instance_type != "" && contains([ + "ecs.t1.c1m2", + "ecs.t1.c2m4", + "ecs.t1.c4m8", + "ecs.t1.c12m24", + "ecs.t1.c32m64", + "ecs.t1.c24m48", + "ecs.t1.c8m16", + "ecs.t1.c16m32", + "ecs.g1.c16m120", + "ecs.g1.c32m240", + "ecs.c1.c1m2", + "ecs.c1.c2m4", + "ecs.c1.c4m8", + "ecs.c1.c8m16", + "ecs.c1.c16m32", + "ecs.c1.c24m48", + "ecs.c1.c12m24", + "ecs.c1.c32m64", + ], var.instance_type) + error_message = "instance_type parameter must be one of the allowed instance types" + } +} + +variable "instance_system_disk_size" { + type = number + description = "System disk size in GiB" + default = 40 + + validation { + condition = var.instance_system_disk_size >= 20 + error_message = "instance_system_disk_size parameter must be at least 20 GiB for K8s" + } +} diff --git a/k8s/common/common_versions.tf b/k8s/common/common_versions.tf new file mode 100644 index 0000000..5ad6e7b --- /dev/null +++ b/k8s/common/common_versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = "> 0.12.0" + + required_providers { + qiniu = { + source = "hashicorp/qiniu" + version = "~> 1.0.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} + +provider "qiniu" {} + +provider "random" {} diff --git a/k8s/common/image_data.tf b/k8s/common/image_data.tf new file mode 100644 index 0000000..6c37d2e --- /dev/null +++ b/k8s/common/image_data.tf @@ -0,0 +1,7 @@ +data "qiniu_compute_image" "ubuntu" { + name = "Ubuntu" +} + +locals { + ubuntu_image_id = data.qiniu_compute_image.ubuntu.id +} diff --git a/k8s/simple/data.tf b/k8s/simple/data.tf new file mode 100644 index 0000000..6483163 --- /dev/null +++ b/k8s/simple/data.tf @@ -0,0 +1,52 @@ +# Generate random suffix for resource names +resource "random_string" "random_suffix" { + length = 6 + upper = false + lower = true + special = false +} + +locals { + # Cluster suffix for resource naming + cluster_suffix = random_string.random_suffix.result + + # CNI manifest URLs + cni_manifests = { + flannel = "https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml" + calico = "https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/calico.yaml" + weave = "https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s.yaml" + } + + cni_manifest_url = local.cni_manifests[var.cni_plugin] +} + +# Generate K8s bootstrap token (format: [a-z0-9]{6}.[a-z0-9]{16}) +resource "random_string" "k8s_token_part1" { + length = 6 + upper = false + lower = true + numeric = true + special = false +} + +resource "random_string" "k8s_token_part2" { + length = 16 + upper = false + lower = true + numeric = true + special = false +} + +locals { + k8s_bootstrap_token = "${random_string.k8s_token_part1.result}.${random_string.k8s_token_part2.result}" +} + +# Generate random passwords for instance access +resource "random_password" "instance_passwords" { + count = var.worker_count + 1 # master + workers + length = 16 + special = true + lower = true + upper = true + numeric = true +} diff --git a/k8s/simple/image_data.tf b/k8s/simple/image_data.tf new file mode 120000 index 0000000..cb4b77d --- /dev/null +++ b/k8s/simple/image_data.tf @@ -0,0 +1 @@ +../common/image_data.tf \ No newline at end of file diff --git a/k8s/simple/init_master.sh b/k8s/simple/init_master.sh new file mode 100644 index 0000000..524da5f --- /dev/null +++ b/k8s/simple/init_master.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -e + +# K8s Master Node Initialization Script +# This script installs and configures Kubernetes master node + +echo "=== Starting K8s Master Node Initialization ===" + +# Disable swap (required for K8s) +swapoff -a +sed -i '/ swap / s/^/#/' /etc/fstab + +# Load required kernel modules +cat < /dev/null; do + echo "Waiting for API server..." + sleep 5 +done + +# Install CNI plugin +echo "Installing CNI plugin: ${cni_plugin}..." +kubectl apply -f ${cni_manifest_url} + +# Generate and save join command +echo "Generating worker join command..." +kubeadm token create ${k8s_token} --print-join-command --ttl=0 > /tmp/k8s_join_command.txt + +# Get CA cert hash for join command +CA_CERT_HASH=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | \ + openssl rsa -pubin -outform der 2>/dev/null | \ + openssl dgst -sha256 -hex | sed 's/^.* //') + +echo "$CA_CERT_HASH" > /tmp/k8s_ca_cert_hash.txt + +# Make master node schedulable (for small clusters) +kubectl taint nodes --all node-role.kubernetes.io/control-plane- || true + +echo "=== K8s Master Node Initialization Complete ===" +echo "Cluster endpoint: https://$(hostname -I | awk '{print $1}'):6443" +echo "Token: ${k8s_token}" +echo "CA cert hash: sha256:$CA_CERT_HASH" diff --git a/k8s/simple/init_worker.sh b/k8s/simple/init_worker.sh new file mode 100644 index 0000000..9f26c3b --- /dev/null +++ b/k8s/simple/init_worker.sh @@ -0,0 +1,90 @@ +#!/bin/bash +set -e + +# K8s Worker Node Initialization Script +# This script installs and configures Kubernetes worker node + +echo "=== Starting K8s Worker Node Initialization ===" + +# Wait for master to be fully initialized (simple delay) +echo "Waiting for master node to initialize..." +sleep 120 + +# Disable swap (required for K8s) +swapoff -a +sed -i '/ swap / s/^/#/' /etc/fstab + +# Load required kernel modules +cat </dev/null; then + echo "Master API server is reachable" + break + fi + echo "Waiting for master API server... (attempt $((RETRY_COUNT+1))/$MAX_RETRIES)" + sleep 10 + RETRY_COUNT=$((RETRY_COUNT+1)) +done + +if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then + echo "ERROR: Could not reach master API server after $MAX_RETRIES attempts" + exit 1 +fi + +# Additional wait to ensure master is fully ready +sleep 30 + +# Join the cluster with discovery-token-unsafe-skip-ca-verification +echo "Joining K8s cluster..." +kubeadm join ${master_ip}:6443 \ + --token=${k8s_token} \ + --discovery-token-unsafe-skip-ca-verification \ + --ignore-preflight-errors=NumCPU,Mem + +echo "=== K8s Worker Node Initialization Complete ===" +echo "Joined cluster at: ${master_ip}:6443" diff --git a/k8s/simple/k8s_variables.tf b/k8s/simple/k8s_variables.tf new file mode 100644 index 0000000..f776001 --- /dev/null +++ b/k8s/simple/k8s_variables.tf @@ -0,0 +1,54 @@ +variable "k8s_version" { + type = string + description = "Kubernetes version" + default = "1.28.0" + + validation { + condition = can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", var.k8s_version)) + error_message = "k8s_version must be in format X.Y.Z (e.g., 1.28.0)" + } +} + +variable "worker_count" { + type = number + description = "Number of K8s worker nodes" + default = 2 + + validation { + condition = var.worker_count >= 1 && var.worker_count <= 10 + error_message = "worker_count must be between 1 and 10" + } +} + +variable "pod_network_cidr" { + type = string + description = "Pod network CIDR" + default = "10.244.0.0/16" + + validation { + condition = can(cidrhost(var.pod_network_cidr, 0)) + error_message = "pod_network_cidr must be a valid CIDR block" + } +} + +variable "service_cidr" { + type = string + description = "Service network CIDR" + default = "10.96.0.0/12" + + validation { + condition = can(cidrhost(var.service_cidr, 0)) + error_message = "service_cidr must be a valid CIDR block" + } +} + +variable "cni_plugin" { + type = string + description = "CNI plugin (flannel, calico, or weave)" + default = "flannel" + + validation { + condition = contains(["flannel", "calico", "weave"], var.cni_plugin) + error_message = "cni_plugin must be one of: flannel, calico, weave" + } +} diff --git a/k8s/simple/main.tf b/k8s/simple/main.tf new file mode 100644 index 0000000..5f1a31b --- /dev/null +++ b/k8s/simple/main.tf @@ -0,0 +1,48 @@ +# K8s Cluster Configuration + +# Create placement group for K8s nodes +resource "qiniu_compute_placement_group" "k8s_pg" { + name = format("k8s-cluster-%s", local.cluster_suffix) + description = format("Placement group for K8s cluster %s", local.cluster_suffix) + strategy = "Spread" +} + +# Create K8s master node +resource "qiniu_compute_instance" "k8s_master" { + instance_type = var.instance_type + placement_group_id = qiniu_compute_placement_group.k8s_pg.id + name = format("k8s-master-%s", local.cluster_suffix) + description = format("Master node for K8s cluster %s", local.cluster_suffix) + image_id = local.ubuntu_image_id + system_disk_size = var.instance_system_disk_size + password = random_password.instance_passwords[0].result + + user_data = base64encode(templatefile("${path.module}/init_master.sh", { + k8s_version = var.k8s_version + pod_network_cidr = var.pod_network_cidr + service_cidr = var.service_cidr + k8s_token = local.k8s_bootstrap_token + cni_manifest_url = local.cni_manifest_url + cni_plugin = var.cni_plugin + })) +} + +# Create K8s worker nodes +resource "qiniu_compute_instance" "k8s_workers" { + depends_on = [qiniu_compute_instance.k8s_master] + + count = var.worker_count + instance_type = var.instance_type + placement_group_id = qiniu_compute_placement_group.k8s_pg.id + name = format("k8s-worker-%02d-%s", count.index + 1, local.cluster_suffix) + description = format("Worker node %02d for K8s cluster %s", count.index + 1, local.cluster_suffix) + image_id = local.ubuntu_image_id + system_disk_size = var.instance_system_disk_size + password = random_password.instance_passwords[count.index + 1].result + + user_data = base64encode(templatefile("${path.module}/init_worker.sh", { + k8s_version = var.k8s_version + master_ip = qiniu_compute_instance.k8s_master.private_ip_addresses[0].ipv4 + k8s_token = local.k8s_bootstrap_token + })) +} diff --git a/k8s/simple/outputs.tf b/k8s/simple/outputs.tf new file mode 100644 index 0000000..7a596cc --- /dev/null +++ b/k8s/simple/outputs.tf @@ -0,0 +1,56 @@ +output "k8s_master_endpoint" { + value = format("https://%s:6443", qiniu_compute_instance.k8s_master.private_ip_addresses[0].ipv4) + description = "Kubernetes API Server endpoint" +} + +output "k8s_master_ip" { + value = qiniu_compute_instance.k8s_master.private_ip_addresses[0].ipv4 + description = "K8s master node IP address" +} + +output "k8s_master_password" { + value = random_password.instance_passwords[0].result + description = "K8s master node SSH password" + sensitive = true +} + +output "k8s_worker_ips" { + value = [ + for instance in qiniu_compute_instance.k8s_workers : + instance.private_ip_addresses[0].ipv4 + ] + description = "List of K8s worker node IP addresses" +} + +output "k8s_worker_passwords" { + value = { + for idx, instance in qiniu_compute_instance.k8s_workers : + instance.name => random_password.instance_passwords[idx + 1].result + } + description = "Map of worker node names to SSH passwords" + sensitive = true +} + +output "k8s_bootstrap_token" { + value = local.k8s_bootstrap_token + description = "K8s bootstrap token for joining nodes" + sensitive = true +} + +output "cluster_info" { + value = { + cluster_name = format("k8s-cluster-%s", local.cluster_suffix) + k8s_version = var.k8s_version + master_endpoint = format("https://%s:6443", qiniu_compute_instance.k8s_master.private_ip_addresses[0].ipv4) + pod_network_cidr = var.pod_network_cidr + service_cidr = var.service_cidr + cni_plugin = var.cni_plugin + worker_count = var.worker_count + } + description = "K8s cluster information" +} + +output "kubeconfig_command" { + value = format("ssh root@%s 'cat /etc/kubernetes/admin.conf' > kubeconfig.yaml", qiniu_compute_instance.k8s_master.private_ip_addresses[0].ipv4) + description = "Command to retrieve kubeconfig file from master node" +} diff --git a/k8s/simple/variables.tf b/k8s/simple/variables.tf new file mode 120000 index 0000000..5884570 --- /dev/null +++ b/k8s/simple/variables.tf @@ -0,0 +1 @@ +../common/common_variables.tf \ No newline at end of file diff --git a/k8s/simple/versions.tf b/k8s/simple/versions.tf new file mode 120000 index 0000000..1b092e6 --- /dev/null +++ b/k8s/simple/versions.tf @@ -0,0 +1 @@ +../common/common_versions.tf \ No newline at end of file