From 70e1823e677f4e02803df1152a3a4bb7850b57b0 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Wed, 12 Nov 2025 10:10:26 +0530 Subject: [PATCH 1/8] Minor change to notes Signed-off-by: Madhav Kandukuri --- charts/mcp-stack/templates/NOTES.txt | 10 ++++++++-- charts/mcp-stack/values.yaml | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/charts/mcp-stack/templates/NOTES.txt b/charts/mcp-stack/templates/NOTES.txt index e1f51c61c..ca4a48043 100644 --- a/charts/mcp-stack/templates/NOTES.txt +++ b/charts/mcp-stack/templates/NOTES.txt @@ -279,8 +279,14 @@ export GW_TOKEN=$(curl -s -u '{{ .Values.mcpContextForge.secret.BASIC_AUTH_USER -X POST http://localhost:4444/auth/login | jq -r '.access_token') {{- else }} export GW_PASS=$(kubectl -n {{ $ns }} get secret {{ $gwSecret }} -o jsonpath="{.data.BASIC_AUTH_PASSWORD}" | base64 -d) -export GW_TOKEN=$(curl -s -u '{{ .Values.mcpContextForge.secret.BASIC_AUTH_USER }}:$GW_PASS' \ - -X POST http://localhost:4444/auth/login | jq -r '.access_token') +export GW_TOKEN=$(curl --request POST \ + --url http://localhost:4444/auth/login \ + --header 'Content-Type: application/json' \ + --data "{ + \"email\": \"admin@example.com\", + \"password\": \"$GW_PASS\" + }" | jq -r '.access_token' +) {{- end }} # 4) Test the gateway health diff --git a/charts/mcp-stack/values.yaml b/charts/mcp-stack/values.yaml index 29a041a90..cd2d9cd7f 100644 --- a/charts/mcp-stack/values.yaml +++ b/charts/mcp-stack/values.yaml @@ -808,7 +808,7 @@ mcpFastTimeServer: replicaCount: 2 image: repository: ghcr.io/ibm/fast-time-server - tag: "0.9.0" + tag: "latest" pullPolicy: IfNotPresent port: 8080 From cb5e6c1302e2466cc453b9e38159195401c78e60 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Wed, 12 Nov 2025 11:26:27 +0530 Subject: [PATCH 2/8] add minio Signed-off-by: Madhav Kandukuri --- .../mcp-stack/templates/minio-deployment.yaml | 76 +++++++++++++++++++ charts/mcp-stack/templates/minio-pvc.yaml | 18 +++++ charts/mcp-stack/templates/minio-secret.yaml | 12 +++ charts/mcp-stack/templates/minio-service.yaml | 23 ++++++ charts/mcp-stack/values.schema.json | 51 +++++++++++++ charts/mcp-stack/values.yaml | 40 ++++++++++ my-values.yaml | 29 +++++++ 7 files changed, 249 insertions(+) create mode 100644 charts/mcp-stack/templates/minio-deployment.yaml create mode 100644 charts/mcp-stack/templates/minio-pvc.yaml create mode 100644 charts/mcp-stack/templates/minio-secret.yaml create mode 100644 charts/mcp-stack/templates/minio-service.yaml create mode 100644 my-values.yaml diff --git a/charts/mcp-stack/templates/minio-deployment.yaml b/charts/mcp-stack/templates/minio-deployment.yaml new file mode 100644 index 000000000..33d59156d --- /dev/null +++ b/charts/mcp-stack/templates/minio-deployment.yaml @@ -0,0 +1,76 @@ +{{- if .Values.minio.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mcp-stack.fullname" . }}-minio + labels: + {{- include "mcp-stack.labels" . | nindent 4 }} + app.kubernetes.io/component: minio +spec: + replicas: 1 + selector: + matchLabels: + {{- include "mcp-stack.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: minio + template: + metadata: + labels: + {{- include "mcp-stack.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: minio + spec: + containers: + - name: minio + image: "{{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}" + imagePullPolicy: {{ .Values.minio.image.pullPolicy }} + command: + - /bin/sh + - -c + - "minio server /data --console-address :{{ .Values.minio.service.consolePort }}" + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} + key: MINIO_ROOT_USER + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} + key: MINIO_ROOT_PASSWORD + ports: + - name: api + containerPort: {{ .Values.minio.service.apiPort }} + protocol: TCP + - name: console + containerPort: {{ .Values.minio.service.consolePort }} + protocol: TCP + livenessProbe: + httpGet: + path: /minio/health/live + port: api + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /minio/health/ready + port: api + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + resources: + {{- toYaml .Values.minio.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + {{- if .Values.minio.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "mcp-stack.fullname" . }}-minio + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} diff --git a/charts/mcp-stack/templates/minio-pvc.yaml b/charts/mcp-stack/templates/minio-pvc.yaml new file mode 100644 index 000000000..18d339de9 --- /dev/null +++ b/charts/mcp-stack/templates/minio-pvc.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.minio.enabled .Values.minio.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "mcp-stack.fullname" . }}-minio + labels: + {{- include "mcp-stack.labels" . | nindent 4 }} + app.kubernetes.io/component: minio +spec: + accessModes: + {{- toYaml .Values.minio.persistence.accessModes | nindent 4 }} + resources: + requests: + storage: {{ .Values.minio.persistence.size | quote }} + {{- if .Values.minio.persistence.storageClassName }} + storageClassName: {{ .Values.minio.persistence.storageClassName }} + {{- end }} +{{- end }} diff --git a/charts/mcp-stack/templates/minio-secret.yaml b/charts/mcp-stack/templates/minio-secret.yaml new file mode 100644 index 000000000..5cba794ae --- /dev/null +++ b/charts/mcp-stack/templates/minio-secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.minio.enabled (not .Values.minio.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "mcp-stack.fullname" . }}-minio + labels: + {{- include "mcp-stack.labels" . | nindent 4 }} +type: Opaque +data: + MINIO_ROOT_USER: {{ .Values.minio.credentials.rootUser | b64enc | quote }} + MINIO_ROOT_PASSWORD: {{ .Values.minio.credentials.rootPassword | b64enc | quote }} +{{- end }} diff --git a/charts/mcp-stack/templates/minio-service.yaml b/charts/mcp-stack/templates/minio-service.yaml new file mode 100644 index 000000000..d65c297c8 --- /dev/null +++ b/charts/mcp-stack/templates/minio-service.yaml @@ -0,0 +1,23 @@ +{{- if .Values.minio.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mcp-stack.fullname" . }}-minio + labels: + {{- include "mcp-stack.labels" . | nindent 4 }} + app.kubernetes.io/component: minio +spec: + type: {{ .Values.minio.service.type }} + ports: + - name: api + port: {{ .Values.minio.service.apiPort }} + targetPort: api + protocol: TCP + - name: console + port: {{ .Values.minio.service.consolePort }} + targetPort: console + protocol: TCP + selector: + {{- include "mcp-stack.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: minio +{{- end }} diff --git a/charts/mcp-stack/values.schema.json b/charts/mcp-stack/values.schema.json index abae9b19e..fb1fafce4 100644 --- a/charts/mcp-stack/values.schema.json +++ b/charts/mcp-stack/values.schema.json @@ -1311,6 +1311,57 @@ }, "additionalProperties": false }, + "minio": { + "type": "object", + "description": "Configuration for the MinIO S3-compatible object storage.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable to deploy MinIO." + }, + "image": { + "type": "object", + "properties": { + "repository": { "type": "string" }, + "tag": { "type": "string" }, + "pullPolicy": { "type": "string" } + } + }, + "existingSecret": { + "type": "string", + "description": "Name of an existing secret for MinIO credentials." + }, + "credentials": { + "type": "object", + "properties": { + "rootUser": { "type": "string" }, + "rootPassword": { "type": "string" } + } + }, + "service": { + "type": "object", + "properties": { + "type": { "type": "string" }, + "apiPort": { "type": "integer" }, + "consolePort": { "type": "integer" } + } + }, + "persistence": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "storageClassName": { "type": "string" }, + "accessModes": { "type": "array", "items": { "type": "string" } }, + "size": { "type": "string" }, + "reclaimPolicy": { "type": "string" } + } + }, + "resources": { + "$ref": "#/$defs/resources", + "description": "CPU/Memory resource requests/limits." + } + } + }, "mcpFastTimeServer": { "type": "object", "description": "MCP Fast Time Server configuration", diff --git a/charts/mcp-stack/values.yaml b/charts/mcp-stack/values.yaml index cd2d9cd7f..f55866947 100644 --- a/charts/mcp-stack/values.yaml +++ b/charts/mcp-stack/values.yaml @@ -799,6 +799,46 @@ redisCommander: successThreshold: 1 failureThreshold: 5 +######################################################################## +# MINIO - S3 Compatible Object Storage +######################################################################## +minio: + enabled: false # Set to true to deploy MinIO + + image: + repository: minio/minio + tag: "RELEASE.2025-09-07T16-13-09Z-cpuv1" # Use a specific stable tag + pullPolicy: IfNotPresent + + # Credentials for the MinIO root user + # It's recommended to use existingSecret in production + existingSecret: "" + credentials: + rootUser: minioadmin + rootPassword: minioadminchangeme # CHANGE IN PRODUCTION! + + service: + type: ClusterIP + apiPort: 9000 # S3 API + consolePort: 9001 # Web UI + + # PersistentVolumeClaim for data durability + persistence: + enabled: true + storageClassName: "" # Use default StorageClass + accessModes: [ReadWriteOnce] + size: 10Gi + reclaimPolicy: Retain + + # Resource limits & requests + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 100m + memory: 256Mi + ######################################################################## # MCP-FAST-TIME-SERVER - optional high-performance time server for MCP (go) # Provides a fast implementation including SSE and Streamable HTTP diff --git a/my-values.yaml b/my-values.yaml new file mode 100644 index 000000000..a24613ec6 --- /dev/null +++ b/my-values.yaml @@ -0,0 +1,29 @@ +# my-values.yaml +# Configuration to minimize disk space usage + +# Disable optional UI tools +pgadmin: + enabled: false + +redisCommander: + enabled: false + +# Disable persistence for Redis (use as in-memory cache only) +redis: + persistence: + enabled: false + +# Reduce storage size for PostgreSQL +postgres: + persistence: + size: 1Gi # Default is 5Gi + +# Reduce replica counts for smaller footprint +mcpContextForge: + replicaCount: 1 + hpa: + enabled: false # HPA is disabled as we are setting a fixed replica count + +mcpFastTimeServer: + replicaCount: 1 + From 5750a2efced6ecd3c89aed82a2136047a228ed96 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Wed, 12 Nov 2025 11:26:58 +0530 Subject: [PATCH 3/8] remove my-values Signed-off-by: Madhav Kandukuri --- my-values.yaml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 my-values.yaml diff --git a/my-values.yaml b/my-values.yaml deleted file mode 100644 index a24613ec6..000000000 --- a/my-values.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# my-values.yaml -# Configuration to minimize disk space usage - -# Disable optional UI tools -pgadmin: - enabled: false - -redisCommander: - enabled: false - -# Disable persistence for Redis (use as in-memory cache only) -redis: - persistence: - enabled: false - -# Reduce storage size for PostgreSQL -postgres: - persistence: - size: 1Gi # Default is 5Gi - -# Reduce replica counts for smaller footprint -mcpContextForge: - replicaCount: 1 - hpa: - enabled: false # HPA is disabled as we are setting a fixed replica count - -mcpFastTimeServer: - replicaCount: 1 - From 0cf00d5f388eeb29d878be73c9762ca82777ed6f Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Wed, 12 Nov 2025 17:40:05 +0530 Subject: [PATCH 4/8] Backup to minio and restore from there Signed-off-by: Madhav Kandukuri --- .../configmap-postgres-upgrade-status.yaml | 13 ++ .../templates/deployment-postgres.yaml | 65 +++++++- .../templates/job-postgres-backup.yaml | 86 ++++++++++ .../job-postgres-migration-check.yaml | 64 ++++++++ .../templates/job-postgres-restore.yaml | 88 +++++++++++ .../mcp-stack/templates/minio-deployment.yaml | 5 +- charts/mcp-stack/templates/minio-service.yaml | 3 +- charts/mcp-stack/values.schema.json | 22 +++ charts/mcp-stack/values.yaml | 8 +- docs/postgres-upgrade-guide.md | 130 +++++++++++++++ docs/postgres-upgrade-process.md | 149 ++++++++++++++++++ scripts/set-backup-completed.sh | 20 +++ 12 files changed, 648 insertions(+), 5 deletions(-) create mode 100644 charts/mcp-stack/templates/configmap-postgres-upgrade-status.yaml create mode 100644 charts/mcp-stack/templates/job-postgres-backup.yaml create mode 100644 charts/mcp-stack/templates/job-postgres-migration-check.yaml create mode 100644 charts/mcp-stack/templates/job-postgres-restore.yaml create mode 100644 docs/postgres-upgrade-guide.md create mode 100644 docs/postgres-upgrade-process.md create mode 100644 scripts/set-backup-completed.sh diff --git a/charts/mcp-stack/templates/configmap-postgres-upgrade-status.yaml b/charts/mcp-stack/templates/configmap-postgres-upgrade-status.yaml new file mode 100644 index 000000000..ddf65ab29 --- /dev/null +++ b/charts/mcp-stack/templates/configmap-postgres-upgrade-status.yaml @@ -0,0 +1,13 @@ +{{- if .Values.postgres.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "mcp-stack.fullname" . }}-postgres-upgrade-status + labels: + {{- include "mcp-stack.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres-upgrade-status +data: + backupCompleted: {{ .Values.postgres.upgrade.backupCompleted | quote }} + targetVersion: {{ .Values.postgres.upgrade.targetVersion | quote }} + lastUpdated: {{ now | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/mcp-stack/templates/deployment-postgres.yaml b/charts/mcp-stack/templates/deployment-postgres.yaml index d9de6810c..381c8858a 100644 --- a/charts/mcp-stack/templates/deployment-postgres.yaml +++ b/charts/mcp-stack/templates/deployment-postgres.yaml @@ -7,6 +7,14 @@ - Adds resource requests / limits pulled from values.yaml. ------------------------------------------------------------------- */}} +{{- $targetVersion := .Values.postgres.upgrade.targetVersion | toString -}} +{{- $postgresImage := "" -}} +{{- if and .Values.postgres.upgrade.enabled (eq $targetVersion "18") -}} +{{- $postgresImage = printf "%s:18" .Values.postgres.image.repository -}} +{{- else -}} +{{- $postgresImage = printf "%s:%s" .Values.postgres.image.repository .Values.postgres.image.tag -}} +{{- end -}} + {{- if .Values.postgres.enabled }} apiVersion: apps/v1 kind: Deployment @@ -21,6 +29,11 @@ spec: selector: matchLabels: app: {{ include "mcp-stack.fullname" . }}-postgres + {{- if and .Values.postgres.upgrade.enabled (eq $targetVersion "18") }} + # For PostgreSQL 18 upgrade, we need to perform a rolling update + strategy: + type: Recreate # Recreate strategy to ensure clean transition during upgrade + {{- end }} template: metadata: labels: @@ -33,9 +46,59 @@ spec: - name: {{ . }} {{- end }} {{- end }} + initContainers: + {{- if and .Values.postgres.upgrade.enabled (eq $targetVersion "18") .Values.postgres.upgrade.backupCompleted }} + # Init container to restore data from backup if this is a PostgreSQL 18 upgrade + - name: postgres-restore + image: "{{ .Values.postgres.image.repository }}:18" # Using PostgreSQL 18 for restore + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -e + + # Install MinIO client and jq + apt-get update && apt-get install -y wget jq + wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /tmp/mc + chmod +x /tmp/mc + + # Configure MinIO client + /tmp/mc alias set minio http://{{ include "mcp-stack.fullname" . }}-minio:{{ .Values.minio.service.apiPort }} $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD + + # List and find the latest backup file - mc ls --json returns individual JSON lines + # Collect the keys and timestamps, sort by timestamp, get the most recent + LATEST_BACKUP=$(/tmp/mc ls --json minio/postgres-backups/ | jq -r '"\(.time) \(.key)"' | sort -r | head -n1 | cut -d' ' -f2-) + + if [ -z "$LATEST_BACKUP" ]; then + echo "No backup file found in MinIO, assuming fresh install..." + exit 0 + fi + + echo "Found backup file: $LATEST_BACKUP" + + # Download the backup file from MinIO + /tmp/mc cp minio/postgres-backups/$LATEST_BACKUP /docker-entrypoint-initdb.d/ + + echo "Backup downloaded to initdb directory: $LATEST_BACKUP" + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} + key: MINIO_ROOT_USER + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} + key: MINIO_ROOT_PASSWORD + volumeMounts: + - name: postgredb + mountPath: /docker-entrypoint-initdb.d + {{- end }} containers: - name: postgres - image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}" + image: "{{ $postgresImage }}" imagePullPolicy: "{{ .Values.postgres.image.pullPolicy }}" # Expose Postgres TCP port inside the pod diff --git a/charts/mcp-stack/templates/job-postgres-backup.yaml b/charts/mcp-stack/templates/job-postgres-backup.yaml new file mode 100644 index 000000000..159239f25 --- /dev/null +++ b/charts/mcp-stack/templates/job-postgres-backup.yaml @@ -0,0 +1,86 @@ +{{- $targetVersion := .Values.postgres.upgrade.targetVersion | toString -}} +{{- if and .Values.postgres.upgrade.enabled (eq $targetVersion "18") (not .Values.postgres.upgrade.backupCompleted) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "mcp-stack.fullname" . }}-postgres-backup + labels: + {{- include "mcp-stack.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres-backup + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +spec: + template: + spec: + restartPolicy: Never + containers: + - name: postgres-backup + image: "{{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }}" + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -e + + echo "Starting PostgreSQL backup..." + + # Wait for PostgreSQL to be ready + echo "Waiting for PostgreSQL to be ready..." + until pg_isready -h {{ include "mcp-stack.fullname" . }}-postgres -U {{ .Values.postgres.credentials.user }}; do + sleep 2 + done + + # Create backup file + BACKUP_FILE="/tmp/postgres-dump-$(date +%Y%m%d-%H%M%S).sql" + pg_dump -h {{ include "mcp-stack.fullname" . }}-postgres \ + -U {{ .Values.postgres.credentials.user }} \ + -d {{ .Values.postgres.credentials.database }} \ + --no-password > $BACKUP_FILE + + echo "Database dumped to $BACKUP_FILE" + + # Install MinIO client and jq + apt-get update && apt-get install -y wget jq + wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /tmp/mc + chmod +x /tmp/mc + + # Configure MinIO client + /tmp/mc alias set minio http://{{ include "mcp-stack.fullname" . }}-minio:{{ .Values.minio.service.apiPort }} $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD + + # Create bucket if it doesn't exist + /tmp/mc mb minio/postgres-backups --ignore-existing + + # Upload the backup file to MinIO + /tmp/mc cp $BACKUP_FILE minio/postgres-backups/ + + echo "Backup uploaded to MinIO successfully" + + # Clean up local file + rm $BACKUP_FILE + + echo "PostgreSQL backup completed and stored in MinIO" + env: + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ include "mcp-stack.postgresSecretName" . | trim }} + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ include "mcp-stack.postgresSecretName" . | trim }} + key: POSTGRES_PASSWORD + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} + key: MINIO_ROOT_USER + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} + key: MINIO_ROOT_PASSWORD +{{- end }} \ No newline at end of file diff --git a/charts/mcp-stack/templates/job-postgres-migration-check.yaml b/charts/mcp-stack/templates/job-postgres-migration-check.yaml new file mode 100644 index 000000000..bf785616d --- /dev/null +++ b/charts/mcp-stack/templates/job-postgres-migration-check.yaml @@ -0,0 +1,64 @@ +{{- $targetVersion := .Values.postgres.upgrade.targetVersion | toString -}} +{{- if and .Values.postgres.upgrade.enabled (eq $targetVersion "18") }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "mcp-stack.fullname" . }}-postgres-migration-check + labels: + {{- include "mcp-stack.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres-migration-check + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "10" + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +spec: + template: + spec: + restartPolicy: Never + containers: + - name: postgres-migration-check + image: "{{ .Values.postgres.image.repository }}:18" + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -e + + echo "Checking PostgreSQL 18 migration status..." + + # Wait for PostgreSQL to be ready + echo "Waiting for PostgreSQL 18 to be ready..." + until pg_isready -h {{ include "mcp-stack.fullname" . }}-postgres -U {{ .Values.postgres.credentials.user }}; do + sleep 2 + done + + # Run a simple query to verify the database is working + RESULT=$(psql -h {{ include "mcp-stack.fullname" . }}-postgres \ + -U {{ .Values.postgres.credentials.user }} \ + -d {{ .Values.postgres.credentials.database }} \ + -t -c "SELECT version();" 2>/dev/null || echo "ERROR") + + if [[ $RESULT == *"ERROR"* ]] || [[ -z "$RESULT" ]]; then + echo "❌ Migration check failed - PostgreSQL 18 is not working properly" + exit 1 + else + echo "✅ PostgreSQL 18 is working properly" + echo "PostgreSQL version: $RESULT" + + # Update the upgrade status in a ConfigMap or similar + # For now, we'll just log that the migration was successful + echo "Migration to PostgreSQL 18 completed successfully" + fi + env: + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ include "mcp-stack.postgresSecretName" . | trim }} + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ include "mcp-stack.postgresSecretName" . | trim }} + key: POSTGRES_PASSWORD +{{- end }} \ No newline at end of file diff --git a/charts/mcp-stack/templates/job-postgres-restore.yaml b/charts/mcp-stack/templates/job-postgres-restore.yaml new file mode 100644 index 000000000..1a594ad44 --- /dev/null +++ b/charts/mcp-stack/templates/job-postgres-restore.yaml @@ -0,0 +1,88 @@ +{{- $targetVersion := .Values.postgres.upgrade.targetVersion | toString -}} +{{- if and .Values.postgres.upgrade.enabled (eq $targetVersion "18") }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "mcp-stack.fullname" . }}-postgres-restore + labels: + {{- include "mcp-stack.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres-restore + annotations: + "helm.sh/hook": post-upgrade + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +spec: + template: + spec: + restartPolicy: Never + containers: + - name: postgres-restore + image: "{{ .Values.postgres.image.repository }}:{{- if eq $targetVersion "18" -}}18{{- else -}}{{ .Values.postgres.image.tag }}{{- end -}}" # Using target version PostgreSQL + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -e + + echo "Starting PostgreSQL restore..." + + # Install MinIO client and jq + apt-get update && apt-get install -y wget jq + wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /tmp/mc + chmod +x /tmp/mc + + # Configure MinIO client + /tmp/mc alias set minio http://{{ include "mcp-stack.fullname" . }}-minio:{{ .Values.minio.service.apiPort }} $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD + + # List and find the latest backup file - mc ls --json returns individual JSON lines + # Collect the keys and timestamps, sort by timestamp, get the most recent + LATEST_BACKUP=$(/tmp/mc ls --json minio/postgres-backups/ | jq -r '"\(.time) \(.key)"' | sort -r | head -n1 | cut -d' ' -f2-) + + if [ -z "$LATEST_BACKUP" ]; then + echo "No backup file found in MinIO!" + exit 1 + fi + + echo "Found backup file: $LATEST_BACKUP" + + # Download the backup file from MinIO + /tmp/mc cp minio/postgres-backups/$LATEST_BACKUP /tmp/ + + # Wait for PostgreSQL to be ready + echo "Waiting for PostgreSQL 18 to be ready..." + until pg_isready -h localhost -U {{ .Values.postgres.credentials.user }}; do + sleep 2 + done + + # Import the backup + psql -h localhost -U {{ .Values.postgres.credentials.user }} -d {{ .Values.postgres.credentials.database }} -f /tmp/$LATEST_BACKUP + + echo "Database restored successfully from backup: $LATEST_BACKUP" + + # Clean up + rm /tmp/$LATEST_BACKUP + + echo "PostgreSQL 18 restore completed" + env: + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ include "mcp-stack.postgresSecretName" . | trim }} + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ include "mcp-stack.postgresSecretName" . | trim }} + key: POSTGRES_PASSWORD + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} + key: MINIO_ROOT_USER + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} + key: MINIO_ROOT_PASSWORD +{{- end }} \ No newline at end of file diff --git a/charts/mcp-stack/templates/minio-deployment.yaml b/charts/mcp-stack/templates/minio-deployment.yaml index 33d59156d..5aed0f236 100644 --- a/charts/mcp-stack/templates/minio-deployment.yaml +++ b/charts/mcp-stack/templates/minio-deployment.yaml @@ -10,13 +10,14 @@ spec: replicas: 1 selector: matchLabels: - {{- include "mcp-stack.selectorLabels" . | nindent 6 }} + {{- include "mcp-stack.labels" . | nindent 6 }} app.kubernetes.io/component: minio template: metadata: labels: - {{- include "mcp-stack.selectorLabels" . | nindent 8 }} + {{- include "mcp-stack.labels" . | nindent 8 }} app.kubernetes.io/component: minio + app: {{ include "mcp-stack.fullname" . }}-minio spec: containers: - name: minio diff --git a/charts/mcp-stack/templates/minio-service.yaml b/charts/mcp-stack/templates/minio-service.yaml index d65c297c8..5aab6044e 100644 --- a/charts/mcp-stack/templates/minio-service.yaml +++ b/charts/mcp-stack/templates/minio-service.yaml @@ -18,6 +18,7 @@ spec: targetPort: console protocol: TCP selector: - {{- include "mcp-stack.selectorLabels" . | nindent 4 }} + {{- include "mcp-stack.labels" . | nindent 4 }} app.kubernetes.io/component: minio + app: {{ include "mcp-stack.fullname" . }}-minio {{- end }} diff --git a/charts/mcp-stack/values.schema.json b/charts/mcp-stack/values.schema.json index fb1fafce4..ea55df735 100644 --- a/charts/mcp-stack/values.schema.json +++ b/charts/mcp-stack/values.schema.json @@ -1021,6 +1021,28 @@ } }, "additionalProperties": false + }, + "upgrade": { + "type": "object", + "description": "PostgreSQL upgrade configuration", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable PostgreSQL upgrade process", + "default": false + }, + "targetVersion": { + "type": ["string", "number"], + "description": "Target PostgreSQL version for upgrade", + "default": "18" + }, + "backupCompleted": { + "type": "boolean", + "description": "Flag to indicate if backup has been completed to prevent re-running", + "default": false + } + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/charts/mcp-stack/values.yaml b/charts/mcp-stack/values.yaml index f55866947..3c4deb2e3 100644 --- a/charts/mcp-stack/values.yaml +++ b/charts/mcp-stack/values.yaml @@ -643,6 +643,12 @@ postgres: successThreshold: 1 failureThreshold: 5 + # ─── PostgreSQL Upgrade Configuration ─── + upgrade: + enabled: false # Set to true to enable the upgrade process + targetVersion: "18" # Target PostgreSQL version (e.g., "18") + backupCompleted: false # Set to true after successful backup (to prevent re-running backup job) + ######################################################################## # REDIS CACHE ######################################################################## @@ -803,7 +809,7 @@ redisCommander: # MINIO - S3 Compatible Object Storage ######################################################################## minio: - enabled: false # Set to true to deploy MinIO + enabled: true # Set to true to deploy MinIO (required for PostgreSQL backups during upgrade) image: repository: minio/minio diff --git a/docs/postgres-upgrade-guide.md b/docs/postgres-upgrade-guide.md new file mode 100644 index 000000000..b9e8bd071 --- /dev/null +++ b/docs/postgres-upgrade-guide.md @@ -0,0 +1,130 @@ +# PostgreSQL 17 to 18 Upgrade Guide + +This document describes the process for upgrading PostgreSQL from version 17 to 18 in the MCP Context Forge Helm chart with automated backup and restore functionality. + +## Overview + +The upgrade process involves: + +1. **Backup Phase**: Taking a backup of PostgreSQL 17 data and storing it in MinIO +2. **Upgrade Phase**: Deploying PostgreSQL 18 with data restored from the backup +3. **Verification Phase**: Checking that PostgreSQL 18 is working correctly + +## Configuration + +To enable the PostgreSQL upgrade process, set the following values in your `values.yaml` or `my-values.yaml`: + +```yaml +postgres: + upgrade: + enabled: true # Enable the upgrade process + targetVersion: "18" # Target version for upgrade + backupCompleted: false # Set to true after initial backup (prevents re-running backup) + +minio: + enabled: true # Required for backup storage +``` + +## Process Flow + +### 1. Pre-Upgrade Hook: Backup Job + +When `postgres.upgrade.enabled` is `true` and `postgres.upgrade.targetVersion` is "18", the following happens during upgrade: + +- A pre-upgrade hook job named `-postgres-backup` is executed +- This job connects to the existing PostgreSQL 17 database +- It performs a `pg_dump` to create a SQL backup +- The backup is uploaded to MinIO at the path `postgres-backups/` +- The job runs with Helm hook annotations: + - `helm.sh/hook: pre-upgrade` + - `helm.sh/hook-weight: "-5"` + - `helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded` + +### 2. PostgreSQL Deployment Update + +- The PostgreSQL deployment is updated to use PostgreSQL 18 image +- An init container is added that downloads the backup from MinIO +- The init container places the backup in `/docker-entrypoint-initdb.d/` +- PostgreSQL 18 starts and automatically applies the SQL dump from initdb directory + +### 3. Post-Upgrade Verification + +- A post-upgrade hook job named `-postgres-migration-check` verifies the upgrade +- This job connects to PostgreSQL 18 and runs a simple query to ensure functionality +- Success is logged if the database is accessible + +## Files Created + +The following files were added/modified to support PostgreSQL upgrade: + +- `templates/job-postgres-backup.yaml` - Backup job +- `templates/job-postgres-restore.yaml` - Restore job (for alternative approach) +- `templates/job-postgres-migration-check.yaml` - Verification job +- `templates/configmap-postgres-upgrade-status.yaml` - Status tracking +- `values.yaml` - Added upgrade configuration section + +## Upgrade Steps + +To perform the PostgreSQL 17 to 18 upgrade: + +1. **Prepare Configuration**: + - Ensure `postgres.upgrade.enabled: true` in your values + - Ensure `postgres.upgrade.targetVersion: "18"` in your values + - Ensure `minio.enabled: true` in your values + +2. **Run Helm Upgrade**: + ```bash + helm upgrade --install mcp-stack ./charts/mcp-stack \ + --namespace mcp \ + -f my-values.yaml \ + --wait --timeout 30m + ``` + +3. **Monitor the Process**: + ```bash + kubectl get pods -n mcp + kubectl logs -l app.kubernetes.io/component=postgres-backup -n mcp + kubectl logs -l app.kubernetes.io/component=postgres -n mcp + ``` + +4. **Verify Completion**: + - Check that the PostgreSQL pod is running with PostgreSQL 18 + - Verify the data has been restored by connecting to the database + - Set `postgres.upgrade.backupCompleted: true` in your values for future upgrades + +## Important Notes + +- **Backup Safety**: The backup job will only run if `postgres.upgrade.backupCompleted` is false +- **Data Preservation**: The existing PVC is preserved during the upgrade process +- **Rollback**: To rollback, set `postgres.upgrade.targetVersion` back to "17" +- **Testing**: Always test the upgrade process in a non-production environment first +- **Downtime**: Expect brief downtime during the upgrade process + +## Troubleshooting + +### Backup Job Fails +Check the logs: +```bash +kubectl logs -l app.kubernetes.io/component=postgres-backup -n mcp +``` + +### PostgreSQL 18 Doesn't Start +Check the PostgreSQL logs: +```bash +kubectl logs -l app=RELEASE_NAME-mcp-stack-postgres -n mcp +``` + +### Data Not Restored +Verify the MinIO backup exists and is accessible: +```bash +kubectl port-forward svc/RELEASE_NAME-mcp-stack-minio 9001:9001 -n mcp +``` +Then access the MinIO UI at http://localhost:9001 with the MinIO credentials. + +## Rollback Procedure + +If the upgrade fails and you need to rollback: + +1. Set `postgres.image.tag: "17"` and `postgres.upgrade.enabled: false` in your values +2. Run the Helm upgrade command again +3. The deployment will revert to PostgreSQL 17 \ No newline at end of file diff --git a/docs/postgres-upgrade-process.md b/docs/postgres-upgrade-process.md new file mode 100644 index 000000000..ae8a44ab5 --- /dev/null +++ b/docs/postgres-upgrade-process.md @@ -0,0 +1,149 @@ +# PostgreSQL 17 to 18 Upgrade Guide + +This guide explains how to upgrade PostgreSQL from version 17 to 18 in the MCP Context Forge Helm chart with automated backup and restore to MinIO. + +## Prerequisites + +Before proceeding with the upgrade, ensure: +- You have a running installation with PostgreSQL 17 +- You have `kubectl` and `helm` access to your cluster +- You have sufficient disk space for backup operations +- You can accept brief downtime during the upgrade process + +## Upgrade Process + +The upgrade process occurs in stages and requires two separate Helm operations: + +### Stage 1: Enable MinIO for Backup Storage + +First, you need to ensure MinIO is deployed and running to store the database backup: + +```bash +# Update your my-values.yaml to enable MinIO +helm upgrade --install mcp-stack ./charts/mcp-stack \ + --namespace mcp \ + --create-namespace \ + -f my-values.yaml \ + --wait --timeout 30m +``` + +Or directly set MinIO to enabled: + +```bash +helm upgrade --install mcp-stack ./charts/mcp-stack \ + --namespace mcp \ + --create-namespace \ + -f my-values.yaml \ + --set minio.enabled=true \ + --wait --timeout 30m +``` + +### Stage 2: Perform the PostgreSQL Upgrade with Backup + +Once MinIO is running, proceed with the actual PostgreSQL upgrade: + +1. **Update your values file** to configure the upgrade: + +```yaml +# In your my-values.yaml +postgres: + upgrade: + enabled: true # Enable the PostgreSQL upgrade process + targetVersion: "18" # Target PostgreSQL version (18) + backupCompleted: false # Set to false to initiate backup process +``` + +2. **Run the upgrade command**: + +```bash +helm upgrade --install mcp-stack ./charts/mcp-stack \ + --namespace mcp \ + -f my-values.yaml \ + --wait --timeout 30m +``` + +This will: +- Run a pre-upgrade hook to backup PostgreSQL 17 data to MinIO +- Upgrade the PostgreSQL deployment to version 18 +- Restore data from the backup during PostgreSQL 18 initialization + +### Stage 3: Verify the Upgrade + +After the upgrade completes, verify that everything is working: + +```bash +# Check that all pods are running +kubectl get pods -n mcp + +# Check PostgreSQL logs +kubectl logs -n mcp deployment/mcp-stack-postgres + +# Verify the PostgreSQL version +kubectl exec -n mcp deployment/mcp-stack-postgres -- psql -U admin -c "SELECT version();" +``` + +## Rollback Process + +If something goes wrong and you need to rollback: + +1. Set `postgres.upgrade.enabled: false` and `postgres.upgrade.targetVersion: "17"` in your values file +2. Run `helm upgrade` with the changes +3. The deployment will revert to PostgreSQL 17 + +## Troubleshooting + +### Backup Job Fails +If the backup job fails, check the logs: +```bash +kubectl logs -n mcp -l app.kubernetes.io/component=postgres-backup +``` + +### PostgreSQL Pod Stuck in CrashLoopBackOff +This usually indicates the data directory compatibility issue. Make sure: +- MinIO is accessible and running +- The backup file exists in MinIO +- The PVC has compatible ownership/perms + +### MinIO Not Starting +Ensure your storage class settings match existing PVCs: +```bash +kubectl describe pvc -n mcp +``` + +## Configuration Reference + +### Upgrade Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `postgres.upgrade.enabled` | Enable the PostgreSQL upgrade process | `false` | +| `postgres.upgrade.targetVersion` | Target PostgreSQL version (currently supports "18") | `"18"` | +| `postgres.upgrade.backupCompleted` | Internal flag - set to false to trigger backup | `false` | +| `minio.enabled` | Enable MinIO for backup storage | `true` (recommended for upgrades) | + +### Storage Configuration + +When upgrading with existing PVCs, make sure to maintain the same storage class: + +```yaml +postgres: + persistence: + storageClassName: "same-as-existing-pvc" # Match existing PVC + size: "same-as-existing-pvc" # Match existing PVC +``` + +## Important Notes + +- **Backup Required**: The upgrade process automatically creates a backup before upgrading +- **Data Safety**: Data is preserved during the upgrade process via the backup/restore mechanism +- **Downtime**: Expect brief downtime during the upgrade as PostgreSQL restarts +- **PVC Compatibility**: The PVC will be reused but the data will be migrated through the backup/restore process +- **MinIO Required**: MinIO must be enabled and operational for the upgrade to work + +## Cleanup After Successful Upgrade + +Once you've verified the upgrade was successful: + +1. Optionally set `postgres.upgrade.backupCompleted: true` to prevent the backup job from running in future upgrades +2. Clean up old backup files from MinIO if needed +3. Update your documentation to reflect the new PostgreSQL version \ No newline at end of file diff --git a/scripts/set-backup-completed.sh b/scripts/set-backup-completed.sh new file mode 100644 index 000000000..35bb1959c --- /dev/null +++ b/scripts/set-backup-completed.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Script to update the backupCompleted flag in my-values.yaml after successful backup + +set -e + +echo "This script will update the postgres.upgrade.backupCompleted flag to true in my-values.yaml" +echo "This will prevent the backup job from running again on subsequent upgrades." +echo + +read -p "Do you want to proceed? (y/N) " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]]; then + # Update the my-values.yaml file to set backupCompleted to true + sed -i 's/backupCompleted: false/backupCompleted: true/' my-values.yaml + + echo "Updated postgres.upgrade.backupCompleted to true in my-values.yaml" + echo "You can now safely upgrade/redeploy without running the backup job again." +else + echo "Operation cancelled." +fi \ No newline at end of file From 612d28fa4fed584f76cbe1a2302d917bba35b888 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Wed, 12 Nov 2025 18:51:06 +0530 Subject: [PATCH 5/8] Almost there Signed-off-by: Madhav Kandukuri --- .../templates/deployment-postgres.yaml | 101 ++++++++++++++---- 1 file changed, 81 insertions(+), 20 deletions(-) diff --git a/charts/mcp-stack/templates/deployment-postgres.yaml b/charts/mcp-stack/templates/deployment-postgres.yaml index 381c8858a..5a314f331 100644 --- a/charts/mcp-stack/templates/deployment-postgres.yaml +++ b/charts/mcp-stack/templates/deployment-postgres.yaml @@ -48,9 +48,9 @@ spec: {{- end }} initContainers: {{- if and .Values.postgres.upgrade.enabled (eq $targetVersion "18") .Values.postgres.upgrade.backupCompleted }} - # Init container to restore data from backup if this is a PostgreSQL 18 upgrade + # Init container to upgrade PostgreSQL data from version 17 to 18 - name: postgres-restore - image: "{{ .Values.postgres.image.repository }}:18" # Using PostgreSQL 18 for restore + image: "{{ .Values.postgres.image.repository }}:17" # Use PostgreSQL 17 image to access the old data command: - /bin/bash - -c @@ -58,29 +58,72 @@ spec: #!/bin/bash set -e - # Install MinIO client and jq - apt-get update && apt-get install -y wget jq - wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /tmp/mc - chmod +x /tmp/mc + PGDATA="/var/lib/postgresql/data/pgdata" + INITDB_DONE_FILE="$PGDATA/.initdb_done" - # Configure MinIO client - /tmp/mc alias set minio http://{{ include "mcp-stack.fullname" . }}-minio:{{ .Values.minio.service.apiPort }} $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD - - # List and find the latest backup file - mc ls --json returns individual JSON lines - # Collect the keys and timestamps, sort by timestamp, get the most recent - LATEST_BACKUP=$(/tmp/mc ls --json minio/postgres-backups/ | jq -r '"\(.time) \(.key)"' | sort -r | head -n1 | cut -d' ' -f2-) - - if [ -z "$LATEST_BACKUP" ]; then - echo "No backup file found in MinIO, assuming fresh install..." + echo "Checking PostgreSQL data directory..." + if [ -d "$PGDATA" ]; then + ls -la "$PGDATA" | head -20 + else + echo "Data directory does not exist yet" exit 0 fi - echo "Found backup file: $LATEST_BACKUP" - - # Download the backup file from MinIO - /tmp/mc cp minio/postgres-backups/$LATEST_BACKUP /docker-entrypoint-initdb.d/ + # Check if this is a data upgrade scenario (PG_VERSION file exists with version 17) + if [ -f "$PGDATA/PG_VERSION" ]; then + CURRENT_VERSION=$(cat "$PGDATA/PG_VERSION" 2>/dev/null || echo "") + echo "Found PG_VERSION file with version: $CURRENT_VERSION" + + if [ "$CURRENT_VERSION" = "17" ]; then + echo "PostgreSQL 17 data detected - this should be removed to allow PostgreSQL 18 to initialize fresh and reload from dump" + + # Install MinIO client and jq + apt-get update && apt-get install -y wget jq + wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /tmp/mc + chmod +x /tmp/mc + + # Configure MinIO client + /tmp/mc alias set minio http://{{ include "mcp-stack.fullname" . }}-minio:{{ .Values.minio.service.apiPort }} $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD + + # List and find the latest backup file + LATEST_BACKUP=$(/tmp/mc ls --json minio/postgres-backups/ | jq -r '"\(.time) \(.key)"' | sort -r | head -n1 | cut -d' ' -f2-) + + if [ -z "$LATEST_BACKUP" ]; then + echo "No backup file found in MinIO - cannot perform upgrade" + exit 0 # Allow to continue but no dump will be loaded + fi + + echo "Found backup file: $LATEST_BACKUP" + + # Download the backup file to the initdb directory + /tmp/mc cp minio/postgres-backups/$LATEST_BACKUP /docker-entrypoint-initdb.d/ + + # Remove the old data directory to allow PostgreSQL 18 to initialize fresh + echo "Removing old PostgreSQL 17 data to allow fresh initialization with PG18..." + # Move current data to backup directory first + mv "$PGDATA" "/tmp/postgres-data-old" + + # Create new data directory + mkdir -p "$PGDATA" + + # Set proper ownership + chown -R postgres:postgres /var/lib/postgresql/data + chown -R postgres:postgres /docker-entrypoint-initdb.d + + echo "Old data backed up and directory prepared for PostgreSQL 18 initialization" + else + echo "Data version ($CURRENT_VERSION) is not 17, skipping removal" + # Still touch the initdb_done file to indicate this is not a fresh install + touch "$PGDATA/.initdb_done" + fi + else + echo "No PG_VERSION file found - this is likely a fresh install" + # Create the directory if doesn't exist + mkdir -p "$PGDATA" + touch "$PGDATA/.initdb_done" + fi - echo "Backup downloaded to initdb directory: $LATEST_BACKUP" + echo "Init container completed successfully" env: - name: MINIO_ROOT_USER valueFrom: @@ -92,9 +135,23 @@ spec: secretKeyRef: name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} key: MINIO_ROOT_PASSWORD + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ include "mcp-stack.postgresSecretName" . | trim }} + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ include "mcp-stack.postgresSecretName" . | trim }} + key: POSTGRES_PASSWORD volumeMounts: - name: postgredb + mountPath: /var/lib/postgresql/data + - name: postgres-initdb mountPath: /docker-entrypoint-initdb.d + securityContext: + runAsUser: 0 # Run as root to have permission to move files {{- end }} containers: - name: postgres @@ -134,6 +191,8 @@ spec: volumeMounts: - name: postgredb mountPath: /var/lib/postgresql/data + - name: postgres-initdb + mountPath: /docker-entrypoint-initdb.d # ─── Resource limits & requests ─── resources: @@ -147,4 +206,6 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: postgres-initdb + emptyDir: {} {{- end }} From 44e978ccecb4c750dd121306c62f710eca7143fd Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Wed, 12 Nov 2025 22:57:39 +0530 Subject: [PATCH 6/8] Removed redundant job Signed-off-by: Madhav Kandukuri --- .../templates/job-postgres-restore.yaml | 92 +------------------ docs/postgres-upgrade-guide.md | 2 +- 2 files changed, 5 insertions(+), 89 deletions(-) diff --git a/charts/mcp-stack/templates/job-postgres-restore.yaml b/charts/mcp-stack/templates/job-postgres-restore.yaml index 1a594ad44..9c2995dcc 100644 --- a/charts/mcp-stack/templates/job-postgres-restore.yaml +++ b/charts/mcp-stack/templates/job-postgres-restore.yaml @@ -1,88 +1,4 @@ -{{- $targetVersion := .Values.postgres.upgrade.targetVersion | toString -}} -{{- if and .Values.postgres.upgrade.enabled (eq $targetVersion "18") }} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "mcp-stack.fullname" . }}-postgres-restore - labels: - {{- include "mcp-stack.labels" . | nindent 4 }} - app.kubernetes.io/component: postgres-restore - annotations: - "helm.sh/hook": post-upgrade - "helm.sh/hook-weight": "5" - "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" -spec: - template: - spec: - restartPolicy: Never - containers: - - name: postgres-restore - image: "{{ .Values.postgres.image.repository }}:{{- if eq $targetVersion "18" -}}18{{- else -}}{{ .Values.postgres.image.tag }}{{- end -}}" # Using target version PostgreSQL - command: - - /bin/bash - - -c - - | - #!/bin/bash - set -e - - echo "Starting PostgreSQL restore..." - - # Install MinIO client and jq - apt-get update && apt-get install -y wget jq - wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /tmp/mc - chmod +x /tmp/mc - - # Configure MinIO client - /tmp/mc alias set minio http://{{ include "mcp-stack.fullname" . }}-minio:{{ .Values.minio.service.apiPort }} $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD - - # List and find the latest backup file - mc ls --json returns individual JSON lines - # Collect the keys and timestamps, sort by timestamp, get the most recent - LATEST_BACKUP=$(/tmp/mc ls --json minio/postgres-backups/ | jq -r '"\(.time) \(.key)"' | sort -r | head -n1 | cut -d' ' -f2-) - - if [ -z "$LATEST_BACKUP" ]; then - echo "No backup file found in MinIO!" - exit 1 - fi - - echo "Found backup file: $LATEST_BACKUP" - - # Download the backup file from MinIO - /tmp/mc cp minio/postgres-backups/$LATEST_BACKUP /tmp/ - - # Wait for PostgreSQL to be ready - echo "Waiting for PostgreSQL 18 to be ready..." - until pg_isready -h localhost -U {{ .Values.postgres.credentials.user }}; do - sleep 2 - done - - # Import the backup - psql -h localhost -U {{ .Values.postgres.credentials.user }} -d {{ .Values.postgres.credentials.database }} -f /tmp/$LATEST_BACKUP - - echo "Database restored successfully from backup: $LATEST_BACKUP" - - # Clean up - rm /tmp/$LATEST_BACKUP - - echo "PostgreSQL 18 restore completed" - env: - - name: PGUSER - valueFrom: - secretKeyRef: - name: {{ include "mcp-stack.postgresSecretName" . | trim }} - key: POSTGRES_USER - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: {{ include "mcp-stack.postgresSecretName" . | trim }} - key: POSTGRES_PASSWORD - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} - key: MINIO_ROOT_USER - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ include "mcp-stack.fullname" . }}-minio{{ end }} - key: MINIO_ROOT_PASSWORD -{{- end }} \ No newline at end of file +{{- /* removed: job-postgres-restore is deprecated and intentionally disabled. + Restores are handled by the postgres initContainer during the upgrade flow. + Keeping this template present would be confusing; it is left inert on purpose. +*/ -}} \ No newline at end of file diff --git a/docs/postgres-upgrade-guide.md b/docs/postgres-upgrade-guide.md index b9e8bd071..d0826a45c 100644 --- a/docs/postgres-upgrade-guide.md +++ b/docs/postgres-upgrade-guide.md @@ -58,7 +58,7 @@ When `postgres.upgrade.enabled` is `true` and `postgres.upgrade.targetVersion` i The following files were added/modified to support PostgreSQL upgrade: - `templates/job-postgres-backup.yaml` - Backup job -- `templates/job-postgres-restore.yaml` - Restore job (for alternative approach) +- `templates/job-postgres-backup.yaml` - Backup job - `templates/job-postgres-migration-check.yaml` - Verification job - `templates/configmap-postgres-upgrade-status.yaml` - Status tracking - `values.yaml` - Added upgrade configuration section From 7dad60505fe6e64cc9ba8c6ddcfb88d9891fbd43 Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Thu, 13 Nov 2025 12:50:29 +0530 Subject: [PATCH 7/8] Update postgres version in docker compose Signed-off-by: Madhav Kandukuri --- docker-compose.yml | 7 ++++--- uv.lock | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8cc97274a..8c3475245 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ networks: volumes: # Named volumes survive podman-compose down/up pgdata: - pgdata18: + # pgdata18: # Enable for postgres 18+ mariadbdata: mysqldata: mongodata: @@ -149,13 +149,14 @@ services: ############################################################################### postgres: - image: postgres:18 + image: postgres:17 environment: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=mysecretpassword - POSTGRES_DB=mcp volumes: - - pgdata18:/var/lib/postgresql + - pgdata:/var/lib/postgresql/data + # - pgdata18:/var/lib/postgresql # Enable for postgres 18+ networks: [mcpnet] healthcheck: test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER"] diff --git a/uv.lock b/uv.lock index 249c2a239..ead203ae2 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11, <3.14" resolution-markers = [ "python_full_version >= '3.13' and platform_machine == 'x86_64' and sys_platform == 'darwin'", @@ -3131,8 +3131,8 @@ requires-dist = [ { name = "langchain-openai", marker = "extra == 'llmchat'", specifier = ">=1.0.2" }, { name = "langgraph", marker = "extra == 'llmchat'", specifier = ">=1.0.2" }, { name = "mcp", specifier = ">=1.21.0" }, - { name = "mcp-contextforge-gateway", extras = ["redis"], marker = "extra == 'all'", specifier = ">=0.8.0" }, - { name = "mcp-contextforge-gateway", extras = ["redis", "dev"], marker = "extra == 'dev-all'", specifier = ">=0.8.0" }, + { name = "mcp-contextforge-gateway", extras = ["redis"], marker = "extra == 'all'", specifier = ">=0.9.0" }, + { name = "mcp-contextforge-gateway", extras = ["redis", "dev"], marker = "extra == 'dev-all'", specifier = ">=0.9.0" }, { name = "oauthlib", specifier = ">=3.3.1" }, { name = "opentelemetry-api", marker = "extra == 'observability'", specifier = ">=1.38.0" }, { name = "opentelemetry-sdk", marker = "extra == 'observability'", specifier = ">=1.38.0" }, From c6dbb418d3067e4528e29b16732d149c421349ac Mon Sep 17 00:00:00 2001 From: Madhav Kandukuri Date: Thu, 13 Nov 2025 13:05:19 +0530 Subject: [PATCH 8/8] Move file to docs Signed-off-by: Madhav Kandukuri --- .../manage}/postgres-upgrade-process.md | 0 docs/postgres-upgrade-guide.md | 130 ------------------ 2 files changed, 130 deletions(-) rename docs/{ => docs/manage}/postgres-upgrade-process.md (100%) delete mode 100644 docs/postgres-upgrade-guide.md diff --git a/docs/postgres-upgrade-process.md b/docs/docs/manage/postgres-upgrade-process.md similarity index 100% rename from docs/postgres-upgrade-process.md rename to docs/docs/manage/postgres-upgrade-process.md diff --git a/docs/postgres-upgrade-guide.md b/docs/postgres-upgrade-guide.md deleted file mode 100644 index d0826a45c..000000000 --- a/docs/postgres-upgrade-guide.md +++ /dev/null @@ -1,130 +0,0 @@ -# PostgreSQL 17 to 18 Upgrade Guide - -This document describes the process for upgrading PostgreSQL from version 17 to 18 in the MCP Context Forge Helm chart with automated backup and restore functionality. - -## Overview - -The upgrade process involves: - -1. **Backup Phase**: Taking a backup of PostgreSQL 17 data and storing it in MinIO -2. **Upgrade Phase**: Deploying PostgreSQL 18 with data restored from the backup -3. **Verification Phase**: Checking that PostgreSQL 18 is working correctly - -## Configuration - -To enable the PostgreSQL upgrade process, set the following values in your `values.yaml` or `my-values.yaml`: - -```yaml -postgres: - upgrade: - enabled: true # Enable the upgrade process - targetVersion: "18" # Target version for upgrade - backupCompleted: false # Set to true after initial backup (prevents re-running backup) - -minio: - enabled: true # Required for backup storage -``` - -## Process Flow - -### 1. Pre-Upgrade Hook: Backup Job - -When `postgres.upgrade.enabled` is `true` and `postgres.upgrade.targetVersion` is "18", the following happens during upgrade: - -- A pre-upgrade hook job named `-postgres-backup` is executed -- This job connects to the existing PostgreSQL 17 database -- It performs a `pg_dump` to create a SQL backup -- The backup is uploaded to MinIO at the path `postgres-backups/` -- The job runs with Helm hook annotations: - - `helm.sh/hook: pre-upgrade` - - `helm.sh/hook-weight: "-5"` - - `helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded` - -### 2. PostgreSQL Deployment Update - -- The PostgreSQL deployment is updated to use PostgreSQL 18 image -- An init container is added that downloads the backup from MinIO -- The init container places the backup in `/docker-entrypoint-initdb.d/` -- PostgreSQL 18 starts and automatically applies the SQL dump from initdb directory - -### 3. Post-Upgrade Verification - -- A post-upgrade hook job named `-postgres-migration-check` verifies the upgrade -- This job connects to PostgreSQL 18 and runs a simple query to ensure functionality -- Success is logged if the database is accessible - -## Files Created - -The following files were added/modified to support PostgreSQL upgrade: - -- `templates/job-postgres-backup.yaml` - Backup job -- `templates/job-postgres-backup.yaml` - Backup job -- `templates/job-postgres-migration-check.yaml` - Verification job -- `templates/configmap-postgres-upgrade-status.yaml` - Status tracking -- `values.yaml` - Added upgrade configuration section - -## Upgrade Steps - -To perform the PostgreSQL 17 to 18 upgrade: - -1. **Prepare Configuration**: - - Ensure `postgres.upgrade.enabled: true` in your values - - Ensure `postgres.upgrade.targetVersion: "18"` in your values - - Ensure `minio.enabled: true` in your values - -2. **Run Helm Upgrade**: - ```bash - helm upgrade --install mcp-stack ./charts/mcp-stack \ - --namespace mcp \ - -f my-values.yaml \ - --wait --timeout 30m - ``` - -3. **Monitor the Process**: - ```bash - kubectl get pods -n mcp - kubectl logs -l app.kubernetes.io/component=postgres-backup -n mcp - kubectl logs -l app.kubernetes.io/component=postgres -n mcp - ``` - -4. **Verify Completion**: - - Check that the PostgreSQL pod is running with PostgreSQL 18 - - Verify the data has been restored by connecting to the database - - Set `postgres.upgrade.backupCompleted: true` in your values for future upgrades - -## Important Notes - -- **Backup Safety**: The backup job will only run if `postgres.upgrade.backupCompleted` is false -- **Data Preservation**: The existing PVC is preserved during the upgrade process -- **Rollback**: To rollback, set `postgres.upgrade.targetVersion` back to "17" -- **Testing**: Always test the upgrade process in a non-production environment first -- **Downtime**: Expect brief downtime during the upgrade process - -## Troubleshooting - -### Backup Job Fails -Check the logs: -```bash -kubectl logs -l app.kubernetes.io/component=postgres-backup -n mcp -``` - -### PostgreSQL 18 Doesn't Start -Check the PostgreSQL logs: -```bash -kubectl logs -l app=RELEASE_NAME-mcp-stack-postgres -n mcp -``` - -### Data Not Restored -Verify the MinIO backup exists and is accessible: -```bash -kubectl port-forward svc/RELEASE_NAME-mcp-stack-minio 9001:9001 -n mcp -``` -Then access the MinIO UI at http://localhost:9001 with the MinIO credentials. - -## Rollback Procedure - -If the upgrade fails and you need to rollback: - -1. Set `postgres.image.tag: "17"` and `postgres.upgrade.enabled: false` in your values -2. Run the Helm upgrade command again -3. The deployment will revert to PostgreSQL 17 \ No newline at end of file