From 04434ff2f78f9408433eb205e9a8ee0d3172312e Mon Sep 17 00:00:00 2001 From: esph Date: Mon, 30 Jun 2025 13:22:04 +0200 Subject: [PATCH 1/2] Deploying the k8s files for the pipeline-api repo. deplyment of this repo to berget --- .github/workflows/ci.yml | 68 ++++++++++++++++++++ k8s/base/api.yaml | 74 ++++++++++++++++++++++ k8s/base/hpa.yaml | 30 +++++++++ k8s/base/ingress.yaml | 22 +++++++ k8s/base/kustomization.yaml | 9 +++ k8s/base/service.yaml | 12 ++++ k8s/base/worker.yaml | 47 ++++++++++++++ k8s/overlays/production/ingress.yaml | 20 ++++++ k8s/overlays/production/kustomization.yaml | 10 +++ k8s/overlays/staging/ingress-patch.yaml | 21 ++++++ k8s/overlays/staging/kustomization.yaml | 10 +++ k8s/overlays/staging/worker-patch.yaml | 39 ++++++++++++ 12 files changed, 362 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 k8s/base/api.yaml create mode 100644 k8s/base/hpa.yaml create mode 100644 k8s/base/ingress.yaml create mode 100644 k8s/base/kustomization.yaml create mode 100644 k8s/base/service.yaml create mode 100644 k8s/base/worker.yaml create mode 100644 k8s/overlays/production/ingress.yaml create mode 100644 k8s/overlays/production/kustomization.yaml create mode 100644 k8s/overlays/staging/ingress-patch.yaml create mode 100644 k8s/overlays/staging/kustomization.yaml create mode 100644 k8s/overlays/staging/worker-patch.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a2df998 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,68 @@ +name: Continuous Integration + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +on: + push: + branches: + - main + paths-ignore: + - 'k8s/**' # Avoid CI runs for updates inside k8s folder + +jobs: + ci: + runs-on: ubuntu-latest + + steps: + # Step 1: Checkout the repository code + - name: 🛎️ Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Step 2: Extract and normalize repository name into lowercase (e.g., pipeline-api) + - id: imagename + uses: ASzc/change-string-case-action@v6 + with: + string: ${{ github.repository }} + + # Step 3: Get the current version of the project (assumes `package.json` exists) + - name: 📝 Get Current Version + id: package-version + uses: martinbeentjes/npm-get-version-action@main + + # Step 4: Login to Docker Registry (GHCR authentication) + - name: 🔐 Login to Docker Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Step 5: Set up Docker Buildx (required for advanced Docker build features) + - name: 🏗️ Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # Step 6: Build and push Docker image + - name: 🔧 Build and push Docker Image + uses: docker/build-push-action@v6 + with: + push: true + tags: | + ${{ env.REGISTRY }}/${{ steps.imagename.outputs.lowercase }}:${{ steps.package-version.outputs.current-version }} + ${{ env.REGISTRY }}/${{ steps.imagename.outputs.lowercase }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + # Step 7: Update version in package.json to prepare for next development cycle + - name: 🎫 Update patch version + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git pull --rebase origin main + npm version prerelease --preid=rc # Increment version (e.g., 1.0.0 -> 1.0.1-rc.0) + git push origin main \ No newline at end of file diff --git a/k8s/base/api.yaml b/k8s/base/api.yaml new file mode 100644 index 0000000..8db11f8 --- /dev/null +++ b/k8s/base/api.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pipeline-api + namespace: default + labels: + app: pipeline-api +spec: + replicas: 3 + selector: + matchLabels: + app: pipeline-api + template: + metadata: + labels: + app: pipeline-api + spec: + terminationGracePeriodSeconds: 60 + initContainers: + - name: db-migration + image: ghcr.io/klimatbyran/pipeline-api + command: ['npm', 'run', 'migrate'] + resources: + requests: + memory: '64Mi' + cpu: '100m' + limits: + memory: '128Mi' + cpu: '200m' + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: postgres-password + - name: DATABASE_URL + value: postgresql://postgres:$(POSTGRES_PASSWORD)@postgresql:5432/pipeline-api + containers: + - name: pipeline-api + image: ghcr.io/klimatbyran/pipeline-api + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1024Mi" + cpu: "500m" + ports: + - containerPort: 4000 + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: postgres-password + - name: DATABASE_URL + value: postgresql://postgres:$(POSTGRES_PASSWORD)@postgresql:5432/pipeline-api + - name: API_SECRET + valueFrom: + secretKeyRef: + name: env + key: API_SECRET + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis + key: redis-password + - name: REDIS_HOST + value: redis-master + - name: CHROMA_HOST + value: http://chromadb:8000 + - name: FRONTEND_URL + value: https://frontend.myapp.com +imagePullPolicy: Always \ No newline at end of file diff --git a/k8s/base/hpa.yaml b/k8s/base/hpa.yaml new file mode 100644 index 0000000..4071d50 --- /dev/null +++ b/k8s/base/hpa.yaml @@ -0,0 +1,30 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: pipeline-api-hpa + namespace: default +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: pipeline-api + minReplicas: 2 + maxReplicas: 12 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 60 \ No newline at end of file diff --git a/k8s/base/ingress.yaml b/k8s/base/ingress.yaml new file mode 100644 index 0000000..5498b37 --- /dev/null +++ b/k8s/base/ingress.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: pipeline-api + annotations: + cert-manager.io/cluster-issuer: 'letsencrypt-prod' +spec: + tls: + - hosts: + - api.pipeline.myapp.com + secretName: pipeline-api-tls + rules: + - host: api.pipeline.myapp.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: pipeline-api + port: + number: 80 \ No newline at end of file diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml new file mode 100644 index 0000000..a41def9 --- /dev/null +++ b/k8s/base/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - api.yaml + - service.yaml + - ingress.yaml + - worker.yaml + - hpa.yaml \ No newline at end of file diff --git a/k8s/base/service.yaml b/k8s/base/service.yaml new file mode 100644 index 0000000..d261d92 --- /dev/null +++ b/k8s/base/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: pipeline-api + namespace: default +spec: + selector: + app: pipeline-api + ports: + - protocol: TCP + port: 80 + targetPort: 4000 \ No newline at end of file diff --git a/k8s/base/worker.yaml b/k8s/base/worker.yaml new file mode 100644 index 0000000..20fc590 --- /dev/null +++ b/k8s/base/worker.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pipeline-api-worker + namespace: default + labels: + app: pipeline-api-worker +spec: + replicas: 1 + selector: + matchLabels: + app: pipeline-api-worker + template: + metadata: + labels: + app: pipeline-api-worker + spec: + containers: + - name: pipeline-api-worker + image: ghcr.io/klimatbyran/pipeline-api + command: ['npm', 'run', 'worker'] + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1024Mi" + cpu: "500m" + env: + - name: REDIS_HOST + value: redis-master + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis + key: redis-password + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: postgresql + key: DATABASE_URL + - name: API_SECRET + valueFrom: + secretKeyRef: + name: env + key: API_SECRET + imagePullPolicy: Always \ No newline at end of file diff --git a/k8s/overlays/production/ingress.yaml b/k8s/overlays/production/ingress.yaml new file mode 100644 index 0000000..f233368 --- /dev/null +++ b/k8s/overlays/production/ingress.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: pipeline-api +spec: + tls: + - hosts: + - api.pipeline.myapp.com + secretName: api-pipeline-myapp-com-tls + rules: + - host: api.pipeline.myapp.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: pipeline-api + port: + number: 80 \ No newline at end of file diff --git a/k8s/overlays/production/kustomization.yaml b/k8s/overlays/production/kustomization.yaml new file mode 100644 index 0000000..1e73c43 --- /dev/null +++ b/k8s/overlays/production/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: pipeline-api +resources: + - ../../base +patches: + - path: ingress-patch.yaml +images: + - name: ghcr.io/klimatbyran/pipeline-api + newTag: '1.0.0' # {"$imagepolicy": "flux-system:pipeline-api:tag"} \ No newline at end of file diff --git a/k8s/overlays/staging/ingress-patch.yaml b/k8s/overlays/staging/ingress-patch.yaml new file mode 100644 index 0000000..16acf86 --- /dev/null +++ b/k8s/overlays/staging/ingress-patch.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: pipeline-api + namespace: pipeline-api-stage +spec: + tls: + - secretName: stage-api-pipeline-myapp-com-tls + hosts: + - stage-api.pipeline.myapp.com + rules: + - host: stage-api.pipeline.myapp.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: pipeline-api + port: + number: 80 \ No newline at end of file diff --git a/k8s/overlays/staging/kustomization.yaml b/k8s/overlays/staging/kustomization.yaml new file mode 100644 index 0000000..52f9ee4 --- /dev/null +++ b/k8s/overlays/staging/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: pipeline-api-stage +resources: + - ../../base +patches: + - path: ingress-patch.yaml +images: + - name: ghcr.io/klimatbyran/pipeline-api + newTag: '1.0.1-rc.0' # {"$imagepolicy": "flux-system:pipeline-api-stage:tag"} \ No newline at end of file diff --git a/k8s/overlays/staging/worker-patch.yaml b/k8s/overlays/staging/worker-patch.yaml new file mode 100644 index 0000000..bc3fbdd --- /dev/null +++ b/k8s/overlays/staging/worker-patch.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pipeline-api-worker +spec: + replicas: 1 # Reduced replicas for staging + selector: + matchLabels: + app: pipeline-api-worker + template: + metadata: + labels: + app: pipeline-api-worker + spec: + containers: + - name: pipeline-api-worker + image: ghcr.io/klimatbyran/pipeline-api + command: ['npm', 'run', 'worker'] + resources: + requests: + memory: "256Mi" + cpu: "125m" + limits: + memory: "512Mi" + cpu: "250m" + env: + - name: REDIS_HOST + value: redis-master + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis + key: redis-password + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: postgresql + key: DATABASE_URL + imagePullPolicy: Always \ No newline at end of file From 993e37bff2610ff7bca8998863529e8c0464213f Mon Sep 17 00:00:00 2001 From: esph Date: Tue, 1 Jul 2025 15:43:57 +0200 Subject: [PATCH 2/2] Implementing docker file --- Dockerfile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d665e26 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +# Use Node.js with alpine for a lightweight environment +FROM node:lts-alpine3.20 AS base +# Install system dependencies required for the application +RUN apk update && apk add --no-cache \ + python3 \ + make \ + g++ \ + ca-certificates +# Set working directory +WORKDIR /app +# Copy `package.json` and `package-lock.json` to the container +COPY package*.json ./ +# Copy the rest of the application source code +COPY . . +# Expose API port +EXPOSE 4000 + +FROM base AS prod +# Install only production dependencies +RUN npm ci --omit=dev +# Build the TypeScript application +RUN npm run build +# Start the Node.js application in prod mode +CMD ["npm", "start"] + +FROM base AS dev +# Install dependencies (including dev) +RUN npm ci +# Start the Node.js application in dev mode +CMD ["npm", "dev"] \ No newline at end of file