From e4c9b344bc5510de40f52e91727dd166846016b8 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Thu, 24 Jul 2025 14:44:29 +0200
Subject: [PATCH 01/24] added docker componse and dockerfile, fixed e2e test,
 added e2e yaml file for github stage

---
 .github/workflows/e2e_tests.yaml    | 211 ++++++++++++++++++++++++++++
 docker-compose.yaml                 |  32 +++++
 test.containerfile                  |  37 +++++
 tests/e2e/features/rest_api.feature |   5 +-
 4 files changed, 283 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/e2e_tests.yaml
 create mode 100644 docker-compose.yaml
 create mode 100644 test.containerfile

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
new file mode 100644
index 00000000..2422299e
--- /dev/null
+++ b/.github/workflows/e2e_tests.yaml
@@ -0,0 +1,211 @@
+# .github/workflows/e2e_test.yml
+name: E2E Tests
+
+on: [push, pull_request]
+
+jobs:
+  e2e_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: 1arp/create-a-file-action@0.4.5
+        env: 
+            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        with:
+          path: '.'
+          isAbsolutePath: false
+          file: 'lightspeed-stack.yaml'
+          content: |
+            name: foo bar baz
+            service:
+              host: 0.0.0.0
+              port: 8080
+              auth_enabled: false
+              workers: 1
+              color_log: true
+              access_log: true
+            llama_stack:
+              # Uses a remote llama-stack service
+              # The instance would have already been started with a llama-stack-run.yaml file
+              use_as_library_client: false
+              # Alternative for "as library use"
+              # use_as_library_client: true
+              # library_client_config_path: <path-to-llama-stack-run.yaml-file>
+              url: http://llama-stack:8321
+              api_key: xyzzy
+            user_data_collection:
+              feedback_disabled: false
+              feedback_storage: "/tmp/data/feedback"
+              transcripts_disabled: false
+              transcripts_storage: "/tmp/data/transcripts"
+              data_collector:
+                enabled: false
+                ingress_server_url: null
+                ingress_server_auth_token: null
+                ingress_content_service_name: null
+                collection_interval: 7200  # 2 hours in seconds
+                cleanup_after_send: true
+                connection_timeout_seconds: 30
+            authentication:
+              module: "noop"
+      
+      - uses: 1arp/create-a-file-action@0.4.5
+        env: 
+            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        with:
+          path: '.'
+          isAbsolutePath: false
+          file: 'run.yaml'
+          content: |
+            version: '2'
+            image_name: simplest-llamastack-app
+
+            apis:
+              - agents
+              - datasetio
+              - eval
+              - inference
+              - post_training
+              - safety
+              - scoring
+              - telemetry
+              - tool_runtime
+              - vector_io
+            benchmarks: []
+            container_image: null
+            datasets: []
+            external_providers_dir: null
+            inference_store:
+              db_path: /app-root/.llama/distributions/ollama/inference_store.db
+              type: sqlite
+            logging: null
+            metadata_store:
+              db_path: /app-root/.llama/distributions/ollama/registry.db
+              namespace: null
+              type: sqlite
+            providers:
+              agents:
+              - config:
+                  persistence_store:
+                    db_path: /app-root/.llama/distributions/ollama/agents_store.db
+                    namespace: null
+                    type: sqlite
+                  responses_store:
+                    db_path: /app-root/.llama/distributions/ollama/responses_store.db
+                    type: sqlite
+                provider_id: meta-reference
+                provider_type: inline::meta-reference
+              datasetio:
+              - config:
+                  kvstore:
+                    db_path: /app-root/.llama/distributions/ollama/huggingface_datasetio.db
+                    namespace: null
+                    type: sqlite
+                provider_id: huggingface
+                provider_type: remote::huggingface
+              - config:
+                  kvstore:
+                    db_path: /app-root/.llama/distributions/ollama/localfs_datasetio.db
+                    namespace: null
+                    type: sqlite
+                provider_id: localfs
+                provider_type: inline::localfs
+              eval:
+              - config:
+                  kvstore:
+                    db_path: /app-root/.llama/distributions/ollama/meta_reference_eval.db
+                    namespace: null
+                    type: sqlite
+                provider_id: meta-reference
+                provider_type: inline::meta-reference
+              inference:
+                - provider_id: openai
+                  provider_type: remote::openai
+                  config:
+                    api_key: ${env.OPENAI_API_KEY}
+              post_training:
+              - config:
+                  checkpoint_format: huggingface
+                  device: cpu
+                  distributed_backend: null
+                provider_id: huggingface
+                provider_type: inline::huggingface
+              safety:
+              - config:
+                  excluded_categories: []
+                provider_id: llama-guard
+                provider_type: inline::llama-guard
+              scoring:
+              - config: {}
+                provider_id: basic
+                provider_type: inline::basic
+              - config: {}
+                provider_id: llm-as-judge
+                provider_type: inline::llm-as-judge
+              - config:
+                  openai_api_key: '******'
+                provider_id: braintrust
+                provider_type: inline::braintrust
+              telemetry:
+              - config:
+                  service_name: ''
+                  sinks: sqlite
+                  sqlite_db_path: /app-root/.llama/distributions/ollama/trace_store.db
+                provider_id: meta-reference
+                provider_type: inline::meta-reference
+              tool_runtime:
+                - provider_id: model-context-protocol
+                  provider_type: remote::model-context-protocol
+                  config: {}
+              vector_io:
+              - config:
+                  kvstore:
+                    db_path: /app-root/.llama/distributions/ollama/faiss_store.db
+                    namespace: null
+                    type: sqlite
+                provider_id: faiss
+                provider_type: inline::faiss
+            scoring_fns: []
+            server:
+              auth: null
+              host: null
+              port: 8321
+              quota: null
+              tls_cafile: null
+              tls_certfile: null
+              tls_keyfile: null
+            shields: []
+            vector_dbs: []
+
+            models:
+              - model_id: gpt-4o-mini
+                provider_id: openai
+                model_type: llm
+                provider_model_id: gpt-4o-mini
+
+      - name: list files
+        run: |
+          ls
+          cat lightspeed-stack.yaml
+          cat run.yaml
+
+      - name: Run service manually
+        env: 
+            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          chmod +x /usr/local/bin/docker-compose
+          docker-compose --version
+          docker-compose up -d
+
+      - name: Wait for services
+        run: |
+          echo "Waiting for services to be healthy..."
+          sleep 20  # adjust depending on boot time
+
+      - name: Run tests in test container
+        run: |
+          echo "We got here and the setup is done, yey"
+          curl http://localhost:8080/v1/models
diff --git a/docker-compose.yaml b/docker-compose.yaml
new file mode 100644
index 00000000..ee0c2cd7
--- /dev/null
+++ b/docker-compose.yaml
@@ -0,0 +1,32 @@
+services:
+  llama-stack:
+    build:
+      context: .
+      dockerfile: test.containerfile
+    container_name: llama-stack
+    ports:
+      - "8321:8321"  # Expose llama-stack on 8321 (adjust if needed)
+    volumes:
+      - ./run.yaml:/app-root/run.yaml:Z
+    environment:
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+    networks:
+      - lightspeednet
+
+  lightspeed-stack:
+    image: quay.io/lightspeed-core/lightspeed-stack:latest
+    container_name: lightspeed-stack
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./lightspeed-stack.yaml:/app-root/lightspeed-stack.yaml:Z
+    environment:
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+    depends_on:
+      - llama-stack
+    networks:
+      - lightspeednet
+
+networks:
+  lightspeednet:
+    driver: bridge
\ No newline at end of file
diff --git a/test.containerfile b/test.containerfile
new file mode 100644
index 00000000..c9c56190
--- /dev/null
+++ b/test.containerfile
@@ -0,0 +1,37 @@
+# vim: set filetype=dockerfile
+FROM registry.access.redhat.com/ubi9/ubi-minimal
+
+ARG APP_ROOT=/app-root
+
+ENV PATH="$PATH:/root/.local/bin"
+
+ADD run.yaml ./
+
+RUN microdnf install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs \
+    python3.11 python3.11-devel python3.11-pip git tar
+
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+
+RUN uv -h
+
+RUN uv venv && \
+    uv pip install llama-stack \
+    fastapi \
+    opentelemetry-sdk \
+    opentelemetry-exporter-otlp \
+    opentelemetry-instrumentation \
+    aiosqlite \
+    litellm \
+    uvicorn \
+    blobfile \
+    datasets \
+    sqlalchemy \
+    faiss-cpu \
+    mcp \
+    autoevals \
+    psutil \
+    torch \
+    peft \
+    trl
+
+CMD ["uv", "run", "llama", "stack", "run", "run.yaml"]
\ No newline at end of file
diff --git a/tests/e2e/features/rest_api.feature b/tests/e2e/features/rest_api.feature
index 4c41ec4b..7735ee3f 100644
--- a/tests/e2e/features/rest_api.feature
+++ b/tests/e2e/features/rest_api.feature
@@ -16,12 +16,13 @@ Feature: REST API tests
           """
           {
               "ready": "bool",
-              "reason": "str"
+              "reason": "str",
+              "providers": "list"
           }
           """
       And The body of the response is the following
           """
-          {"ready": true, "reason": "service is ready"}
+          {"ready": true, "reason": "All providers are healthy", "providers": []}
           """
 
 

From c6faa0d544bd641dd7dfb5bacc725e148cb313cd Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 08:45:40 +0200
Subject: [PATCH 02/24] run e2e tests on pipeline

---
 .github/workflows/e2e_tests.yaml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 2422299e..5e3e0b44 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -6,13 +6,13 @@ on: [push, pull_request]
 jobs:
   e2e_tests:
     runs-on: ubuntu-latest
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
     steps:
       - uses: actions/checkout@v4
 
       - uses: 1arp/create-a-file-action@0.4.5
-        env: 
-            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         with:
           path: '.'
           isAbsolutePath: false
@@ -52,8 +52,6 @@ jobs:
               module: "noop"
       
       - uses: 1arp/create-a-file-action@0.4.5
-        env: 
-            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         with:
           path: '.'
           isAbsolutePath: false
@@ -192,8 +190,6 @@ jobs:
           cat run.yaml
 
       - name: Run service manually
-        env: 
-            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
           curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
           chmod +x /usr/local/bin/docker-compose
@@ -208,4 +204,4 @@ jobs:
       - name: Run tests in test container
         run: |
           echo "We got here and the setup is done, yey"
-          curl http://localhost:8080/v1/models
+          make test-e2e

From 37c1397f44f42702655f6d30ca5976fd27c8e29b Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 09:22:13 +0200
Subject: [PATCH 03/24] install uv on pipeline

---
 .github/workflows/e2e_tests.yaml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 5e3e0b44..94f4d26b 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -201,7 +201,16 @@ jobs:
           echo "Waiting for services to be healthy..."
           sleep 20  # adjust depending on boot time
 
-      - name: Run tests in test container
+      - name: Install uv
         run: |
-          echo "We got here and the setup is done, yey"
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Install Python dependencies
+        run: |
+          uv sync
+
+      - name: Run e2e tests
+        run: |
+          echo "Running e2e tests against running services..."
           make test-e2e

From 1353edcb175fa46c1535f7ab5fedc8d6f84a98bd Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 10:08:27 +0200
Subject: [PATCH 04/24] bumped python version in dockefile

---
 test.containerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test.containerfile b/test.containerfile
index c9c56190..6dfc5d2c 100644
--- a/test.containerfile
+++ b/test.containerfile
@@ -8,7 +8,7 @@ ENV PATH="$PATH:/root/.local/bin"
 ADD run.yaml ./
 
 RUN microdnf install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs \
-    python3.11 python3.11-devel python3.11-pip git tar
+    python3.12 python3.12-devel python3.12-pip git tar
 
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 

From 6827cd6e942d4a440f3148db025f7870fc06689a Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 10:31:29 +0200
Subject: [PATCH 05/24] using latest docker compose

---
 .github/workflows/e2e_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 94f4d26b..10cc50c1 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -191,7 +191,7 @@ jobs:
 
       - name: Run service manually
         run: |
-          curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
           chmod +x /usr/local/bin/docker-compose
           docker-compose --version
           docker-compose up -d

From 214b769d32f50ee01a8805592c0c74a50d4dde85 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 10:51:13 +0200
Subject: [PATCH 06/24] install uv using pip

---
 .github/workflows/e2e_tests.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 10cc50c1..9a32e89a 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -203,8 +203,7 @@ jobs:
 
       - name: Install uv
         run: |
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+          pip install uv
 
       - name: Install Python dependencies
         run: |

From 27df6770ff00a2398bd6f91cb1581a3847ac27cb Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 11:05:07 +0200
Subject: [PATCH 07/24] use preinstalled docker compose

---
 .github/workflows/e2e_tests.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 9a32e89a..17077520 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -191,10 +191,8 @@ jobs:
 
       - name: Run service manually
         run: |
-          curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-          chmod +x /usr/local/bin/docker-compose
-          docker-compose --version
-          docker-compose up -d
+          docker compose --version
+          docker compose up -d
 
       - name: Wait for services
         run: |

From 4b1166a6836497e2e02e5fe687201593ccb23e48 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 11:38:33 +0200
Subject: [PATCH 08/24] add healthcheck for lightspeed stack

---
 .github/workflows/e2e_tests.yaml | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 17077520..e9f55d18 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -197,7 +197,22 @@ jobs:
       - name: Wait for services
         run: |
           echo "Waiting for services to be healthy..."
-          sleep 20  # adjust depending on boot time
+          # Wait for services to be ready with timeout
+          timeout=300
+          while [ $timeout -gt 0 ]; do
+            if curl -X 'GET' 'http://localhost:8080/v1/readiness' -H 'accept: application/json'then
+              echo "Services are ready!"
+              break
+            fi
+            echo "Waiting for services... ($timeout seconds remaining)"
+            sleep 5
+            timeout=$((timeout-5))
+          done
+          if [ $timeout -le 0 ]; then
+            echo "Services failed to become ready within timeout"
+            docker-compose logs
+            exit 1
+          fi
 
       - name: Install uv
         run: |

From 17c14536b5582da00a5589e67ab9b009768e3b87 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 12:10:23 +0200
Subject: [PATCH 09/24] fix errors

---
 .github/workflows/e2e_tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index e9f55d18..776a7631 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -200,7 +200,7 @@ jobs:
           # Wait for services to be ready with timeout
           timeout=300
           while [ $timeout -gt 0 ]; do
-            if curl -X 'GET' 'http://localhost:8080/v1/readiness' -H 'accept: application/json'then
+            if curl -X 'GET' 'http://localhost:8080/v1/readiness' -H 'accept: application/json'; then
               echo "Services are ready!"
               break
             fi
@@ -210,7 +210,7 @@ jobs:
           done
           if [ $timeout -le 0 ]; then
             echo "Services failed to become ready within timeout"
-            docker-compose logs
+            docker compose logs
             exit 1
           fi
 

From edc70e395ba98d869ab03ce7a75c88712e8ffa44 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 12:54:17 +0200
Subject: [PATCH 10/24] add -f to the healthcheck curl

---
 .github/workflows/e2e_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 776a7631..7a3ff108 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -200,7 +200,7 @@ jobs:
           # Wait for services to be ready with timeout
           timeout=300
           while [ $timeout -gt 0 ]; do
-            if curl -X 'GET' 'http://localhost:8080/v1/readiness' -H 'accept: application/json'; then
+            if curl -f -X 'GET' 'http://localhost:8080/v1/readiness' -H 'accept: application/json'; then
               echo "Services are ready!"
               break
             fi

From df0fac3b9b8aaec1994a4ea93882b577fd4177dd Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 13:44:10 +0200
Subject: [PATCH 11/24] add env var to the commands

---
 .github/workflows/e2e_tests.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 7a3ff108..8be5715a 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -190,12 +190,21 @@ jobs:
           cat run.yaml
 
       - name: Run service manually
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
+          # Debug: Verify the environment variable is available
+          echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"
+          
           docker compose --version
           docker compose up -d
 
       - name: Wait for services
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
+          echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"
+          
           echo "Waiting for services to be healthy..."
           # Wait for services to be ready with timeout
           timeout=300

From 9fa890feb310cd3c305b7b0a36728bc8447cb753 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 13:55:01 +0200
Subject: [PATCH 12/24] double curly brackets for env var

---
 .github/workflows/e2e_tests.yaml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 8be5715a..09043d07 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -122,7 +122,7 @@ jobs:
                 - provider_id: openai
                   provider_type: remote::openai
                   config:
-                    api_key: ${env.OPENAI_API_KEY}
+                    api_key: ${{ env.OPENAI_API_KEY }}
               post_training:
               - config:
                   checkpoint_format: huggingface
@@ -193,18 +193,11 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
-          # Debug: Verify the environment variable is available
-          echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"
-          
           docker compose --version
           docker compose up -d
 
       - name: Wait for services
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
-          echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"
-          
           echo "Waiting for services to be healthy..."
           # Wait for services to be ready with timeout
           timeout=300

From a7d112646d01ac0adb01969195580b13f2f47b1e Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 15:42:56 +0200
Subject: [PATCH 13/24] echo if the variable is actually set

---
 .github/workflows/e2e_tests.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 09043d07..5c81ac7a 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -193,6 +193,9 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
+          # Debug: Verify the environment variable is available for file creation
+          echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"
+          
           docker compose --version
           docker compose up -d
 

From 8782827836f8dc45cc20da393999f168e867050e Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 15:43:43 +0200
Subject: [PATCH 14/24] reduce the time to curl

---
 .github/workflows/e2e_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 5c81ac7a..a6c56876 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -203,7 +203,7 @@ jobs:
         run: |
           echo "Waiting for services to be healthy..."
           # Wait for services to be ready with timeout
-          timeout=300
+          timeout=30
           while [ $timeout -gt 0 ]; do
             if curl -f -X 'GET' 'http://localhost:8080/v1/readiness' -H 'accept: application/json'; then
               echo "Services are ready!"

From b6a3625f8af4c0ce13c4d605cb5c7053167930b9 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 15:47:45 +0200
Subject: [PATCH 15/24] remove redundant set

---
 .github/workflows/e2e_tests.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index a6c56876..eb3e79d6 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -190,8 +190,6 @@ jobs:
           cat run.yaml
 
       - name: Run service manually
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
           # Debug: Verify the environment variable is available for file creation
           echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"

From 01fddd004b48a06fad9d1072e26710554b09a64a Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 16:10:30 +0200
Subject: [PATCH 16/24] fix env var

---
 .github/workflows/e2e_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index eb3e79d6..19a697f4 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -122,7 +122,7 @@ jobs:
                 - provider_id: openai
                   provider_type: remote::openai
                   config:
-                    api_key: ${{ env.OPENAI_API_KEY }}
+                    api_key: ${OPENAI_API_KEY}
               post_training:
               - config:
                   checkpoint_format: huggingface

From 3c126146086d9764f6ea7711f7c85214adb5e675 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 16:13:55 +0200
Subject: [PATCH 17/24] set env var

---
 .github/workflows/e2e_tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 19a697f4..3cee33e6 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -190,6 +190,8 @@ jobs:
           cat run.yaml
 
       - name: Run service manually
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
           # Debug: Verify the environment variable is available for file creation
           echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"

From 86148a59caa32de583dea95e5fbe9109d967fddd Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 16:29:34 +0200
Subject: [PATCH 18/24] reset e2e tests yaml

---
 .github/workflows/e2e_tests.yaml | 79 ++++++++------------------------
 1 file changed, 20 insertions(+), 59 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 3cee33e6..8e6980fb 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -6,54 +6,42 @@ on: [push, pull_request]
 jobs:
   e2e_tests:
     runs-on: ubuntu-latest
-    env:
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
     steps:
       - uses: actions/checkout@v4
 
       - uses: 1arp/create-a-file-action@0.4.5
+        env: 
+            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         with:
-          path: '.'
+          path: 'src'
           isAbsolutePath: false
           file: 'lightspeed-stack.yaml'
           content: |
-            name: foo bar baz
+            name: lcore
             service:
-              host: 0.0.0.0
+              host: localhost
               port: 8080
               auth_enabled: false
               workers: 1
               color_log: true
               access_log: true
             llama_stack:
-              # Uses a remote llama-stack service
-              # The instance would have already been started with a llama-stack-run.yaml file
-              use_as_library_client: false
-              # Alternative for "as library use"
               # use_as_library_client: true
-              # library_client_config_path: <path-to-llama-stack-run.yaml-file>
-              url: http://llama-stack:8321
-              api_key: xyzzy
+              # library_client_config_path: /app-root/run.yaml
+              url: http://lightspeednet:8321
+              api_key: "$OPENAI_API_KEY"
             user_data_collection:
               feedback_disabled: false
               feedback_storage: "/tmp/data/feedback"
               transcripts_disabled: false
               transcripts_storage: "/tmp/data/transcripts"
-              data_collector:
-                enabled: false
-                ingress_server_url: null
-                ingress_server_auth_token: null
-                ingress_content_service_name: null
-                collection_interval: 7200  # 2 hours in seconds
-                cleanup_after_send: true
-                connection_timeout_seconds: 30
-            authentication:
-              module: "noop"
       
       - uses: 1arp/create-a-file-action@0.4.5
+        env: 
+            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         with:
-          path: '.'
+          path: 'src'
           isAbsolutePath: false
           file: 'run.yaml'
           content: |
@@ -122,7 +110,7 @@ jobs:
                 - provider_id: openai
                   provider_type: remote::openai
                   config:
-                    api_key: ${OPENAI_API_KEY}
+                    api_key: ${{ env.OPENAI_API_KEY }}
               post_training:
               - config:
                   checkpoint_format: huggingface
@@ -176,12 +164,11 @@ jobs:
               tls_keyfile: null
             shields: []
             vector_dbs: []
-
             models:
-              - model_id: gpt-4o-mini
+              - model_id: gpt-4-turbo
                 provider_id: openai
                 model_type: llm
-                provider_model_id: gpt-4o-mini
+                provider_model_id: gpt-4-turbo
 
       - name: list files
         run: |
@@ -190,44 +177,18 @@ jobs:
           cat run.yaml
 
       - name: Run service manually
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        env: 
+            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
-          # Debug: Verify the environment variable is available for file creation
-          echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"
-          
           docker compose --version
           docker compose up -d
 
       - name: Wait for services
         run: |
           echo "Waiting for services to be healthy..."
-          # Wait for services to be ready with timeout
-          timeout=30
-          while [ $timeout -gt 0 ]; do
-            if curl -f -X 'GET' 'http://localhost:8080/v1/readiness' -H 'accept: application/json'; then
-              echo "Services are ready!"
-              break
-            fi
-            echo "Waiting for services... ($timeout seconds remaining)"
-            sleep 5
-            timeout=$((timeout-5))
-          done
-          if [ $timeout -le 0 ]; then
-            echo "Services failed to become ready within timeout"
-            docker compose logs
-            exit 1
-          fi
-
-      - name: Install uv
-        run: |
-          pip install uv
-
-      - name: Install Python dependencies
-        run: |
-          uv sync
+          sleep 20  # adjust depending on boot time
 
-      - name: Run e2e tests
+      - name: Run tests in test container
         run: |
-          echo "Running e2e tests against running services..."
-          make test-e2e
+          echo "We got here and the setup is done, yey"
+          curl http://localhost:8080/v1/models
\ No newline at end of file

From 2b910277ece63ae5d24c68b828bbc408787a453f Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 16:30:37 +0200
Subject: [PATCH 19/24] fix paths

---
 .github/workflows/e2e_tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 8e6980fb..b5665d87 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -14,7 +14,7 @@ jobs:
         env: 
             OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         with:
-          path: 'src'
+          path: '.'
           isAbsolutePath: false
           file: 'lightspeed-stack.yaml'
           content: |
@@ -41,7 +41,7 @@ jobs:
         env: 
             OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         with:
-          path: 'src'
+          path: '.'
           isAbsolutePath: false
           file: 'run.yaml'
           content: |

From 116c27905a1326bcacf87fd421f87bbfb8193854 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 16:35:54 +0200
Subject: [PATCH 20/24] fix url

---
 .github/workflows/e2e_tests.yaml | 47 ++++++++++++++++++++++++++------
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index b5665d87..a55963c3 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -27,10 +27,10 @@ jobs:
               color_log: true
               access_log: true
             llama_stack:
-              # use_as_library_client: true
-              # library_client_config_path: /app-root/run.yaml
-              url: http://lightspeednet:8321
-              api_key: "$OPENAI_API_KEY"
+                # use_as_library_client: true
+                # library_client_config_path: /app-root/run.yaml
+                url: http://llama-stack:8321
+                api_key: ${{ env.OPENAI_API_KEY }}
             user_data_collection:
               feedback_disabled: false
               feedback_storage: "/tmp/data/feedback"
@@ -186,9 +186,40 @@ jobs:
       - name: Wait for services
         run: |
           echo "Waiting for services to be healthy..."
-          sleep 20  # adjust depending on boot time
+          
+          # Check container status
+          echo "=== Container Status ==="
+          docker compose ps
+          
+          # Show logs to check for startup errors
+          echo "=== Initial Logs ==="
+          docker compose logs --tail=20
+          
+          # Wait longer for services to fully initialize
+          sleep 40
+          
+          # Check final status
+          echo "=== Final Container Status ==="
+          docker compose ps
+          
+          # Show any error logs
+          echo "=== Recent Logs ==="
+          docker compose logs --tail=30
 
-      - name: Run tests in test container
+      - name: Test services
         run: |
-          echo "We got here and the setup is done, yey"
-          curl http://localhost:8080/v1/models
\ No newline at end of file
+          echo "Testing service connectivity..."
+          
+          # Test basic connectivity first
+          echo "Testing basic connectivity to port 8080:"
+          curl -v --connect-timeout 10 http://localhost:8080/ || echo "❌ Basic connection failed"
+          
+          echo "Testing /v1/models endpoint:"
+          curl -v --connect-timeout 10 http://localhost:8080/v1/models || echo "❌ Models endpoint failed"
+          
+          # If that fails, show logs again
+          if ! curl -s --connect-timeout 5 http://localhost:8080/v1/models > /dev/null; then
+            echo "=== Service appears to be failing - showing logs ==="
+            docker compose logs lightspeed-stack --tail=50
+            docker compose logs llama-stack --tail=50
+          fi
\ No newline at end of file

From c3bcc5712cf120b48051061ee31b14460227d705 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 16:45:45 +0200
Subject: [PATCH 21/24] fix lightspeed stack yaml file

---
 .github/workflows/e2e_tests.yaml | 48 +++++++-------------------------
 1 file changed, 10 insertions(+), 38 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index a55963c3..35b039e7 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -18,7 +18,7 @@ jobs:
           isAbsolutePath: false
           file: 'lightspeed-stack.yaml'
           content: |
-            name: lcore
+            name: foo bar baz
             service:
               host: localhost
               port: 8080
@@ -27,10 +27,11 @@ jobs:
               color_log: true
               access_log: true
             llama_stack:
-                # use_as_library_client: true
-                # library_client_config_path: /app-root/run.yaml
-                url: http://llama-stack:8321
-                api_key: ${{ env.OPENAI_API_KEY }}
+              use_as_library_client: false
+              # use_as_library_client: true
+              # library_client_config_path: /app-root/run.yaml
+              url: http://llama-stack:8321
+              api_key: xyzzy
             user_data_collection:
               feedback_disabled: false
               feedback_storage: "/tmp/data/feedback"
@@ -110,7 +111,7 @@ jobs:
                 - provider_id: openai
                   provider_type: remote::openai
                   config:
-                    api_key: ${{ env.OPENAI_API_KEY }}
+                    api_key: ${{ secrets.OPENAI_API_KEY }}
               post_training:
               - config:
                   checkpoint_format: huggingface
@@ -186,40 +187,11 @@ jobs:
       - name: Wait for services
         run: |
           echo "Waiting for services to be healthy..."
-          
-          # Check container status
-          echo "=== Container Status ==="
-          docker compose ps
-          
-          # Show logs to check for startup errors
-          echo "=== Initial Logs ==="
-          docker compose logs --tail=20
-          
-          # Wait longer for services to fully initialize
-          sleep 40
-          
-          # Check final status
-          echo "=== Final Container Status ==="
-          docker compose ps
-          
-          # Show any error logs
-          echo "=== Recent Logs ==="
-          docker compose logs --tail=30
+          sleep 20  # adjust depending on boot time
 
       - name: Test services
         run: |
           echo "Testing service connectivity..."
+          curl http://localhost:8080/v1/models
           
-          # Test basic connectivity first
-          echo "Testing basic connectivity to port 8080:"
-          curl -v --connect-timeout 10 http://localhost:8080/ || echo "❌ Basic connection failed"
-          
-          echo "Testing /v1/models endpoint:"
-          curl -v --connect-timeout 10 http://localhost:8080/v1/models || echo "❌ Models endpoint failed"
-          
-          # If that fails, show logs again
-          if ! curl -s --connect-timeout 5 http://localhost:8080/v1/models > /dev/null; then
-            echo "=== Service appears to be failing - showing logs ==="
-            docker compose logs lightspeed-stack --tail=50
-            docker compose logs llama-stack --tail=50
-          fi
\ No newline at end of file
+          
\ No newline at end of file

From d872884d2a607f0d1b669b1827801a83e684240e Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 17:02:10 +0200
Subject: [PATCH 22/24] revert

---
 .github/workflows/e2e_tests.yaml | 35 +++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index 35b039e7..ef0bc45b 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -20,16 +20,19 @@ jobs:
           content: |
             name: foo bar baz
             service:
-              host: localhost
+              host: 0.0.0.0
               port: 8080
               auth_enabled: false
               workers: 1
               color_log: true
               access_log: true
             llama_stack:
+              # Uses a remote llama-stack service
+              # The instance would have already been started with a llama-stack-run.yaml file
               use_as_library_client: false
+              # Alternative for "as library use"
               # use_as_library_client: true
-              # library_client_config_path: /app-root/run.yaml
+              # library_client_config_path: <path-to-llama-stack-run.yaml-file>
               url: http://llama-stack:8321
               api_key: xyzzy
             user_data_collection:
@@ -37,7 +40,17 @@ jobs:
               feedback_storage: "/tmp/data/feedback"
               transcripts_disabled: false
               transcripts_storage: "/tmp/data/transcripts"
-      
+              data_collector:
+                enabled: false
+                ingress_server_url: null
+                ingress_server_auth_token: null
+                ingress_content_service_name: null
+                collection_interval: 7200  # 2 hours in seconds
+                cleanup_after_send: true
+                connection_timeout_seconds: 30
+            authentication:
+              module: "noop"
+
       - uses: 1arp/create-a-file-action@0.4.5
         env: 
             OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -48,7 +61,6 @@ jobs:
           content: |
             version: '2'
             image_name: simplest-llamastack-app
-
             apis:
               - agents
               - datasetio
@@ -111,7 +123,7 @@ jobs:
                 - provider_id: openai
                   provider_type: remote::openai
                   config:
-                    api_key: ${{ secrets.OPENAI_API_KEY }}
+                    api_key: ${env.OPENAI_API_KEY}
               post_training:
               - config:
                   checkpoint_format: huggingface
@@ -165,11 +177,12 @@ jobs:
               tls_keyfile: null
             shields: []
             vector_dbs: []
+
             models:
-              - model_id: gpt-4-turbo
+              - model_id: gpt-4o-mini
                 provider_id: openai
                 model_type: llm
-                provider_model_id: gpt-4-turbo
+                provider_model_id: gpt-4o-mini
 
       - name: list files
         run: |
@@ -189,9 +202,7 @@ jobs:
           echo "Waiting for services to be healthy..."
           sleep 20  # adjust depending on boot time
 
-      - name: Test services
+      - name: Run tests in test container
         run: |
-          echo "Testing service connectivity..."
-          curl http://localhost:8080/v1/models
-          
-          
\ No newline at end of file
+          echo "We got here and the setup is done, yey"
+          curl http://localhost:8080/v1/models
\ No newline at end of file

From 84c61ad481cac97d2b31f44a4e6db76abf934d87 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Fri, 25 Jul 2025 17:12:14 +0200
Subject: [PATCH 23/24] debug env key

---
 .github/workflows/e2e_tests.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index ef0bc45b..bd2eaed7 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -122,8 +122,8 @@ jobs:
               inference:
                 - provider_id: openai
                   provider_type: remote::openai
-                  config:
-                    api_key: ${env.OPENAI_API_KEY}
+                config:
+                  api_key: ${{ secrets.OPENAI_API_KEY }}
               post_training:
               - config:
                   checkpoint_format: huggingface
@@ -194,6 +194,10 @@ jobs:
         env: 
             OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
+          # Debug: Check if environment variable is available for docker-compose
+          echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"
+          echo "OPENAI_API_KEY length: ${#OPENAI_API_KEY}"
+          
           docker compose --version
           docker compose up -d
 

From 11568d5424bb942af3c9f5cc62a4907348421b35 Mon Sep 17 00:00:00 2001
From: Radovan Fuchs <rfuchs@rfuchs-thinkpadp1gen7.tpb.csb>
Date: Mon, 28 Jul 2025 08:38:29 +0200
Subject: [PATCH 24/24] use pull request target

---
 .github/workflows/e2e_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
index bd2eaed7..00db84f1 100644
--- a/.github/workflows/e2e_tests.yaml
+++ b/.github/workflows/e2e_tests.yaml
@@ -1,7 +1,7 @@
 # .github/workflows/e2e_test.yml
 name: E2E Tests
 
-on: [push, pull_request]
+on: [push, pull_request_target]
 
 jobs:
   e2e_tests: