run loadtest for changed samples

.github/workflows/deploy-changed-samples.yml

@@ -0,0 +1,103 @@
+name: Deploy Changed Samples
+
+on:
+  pull_request:
+    paths:
+      - 'samples/**'
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  deploy_changed_samples:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: deploy_samples-group-${{ github.head_ref }}
+      cancel-in-progress: true
+    environment: deploy-changed-samples
+    timeout-minutes: 90
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Fetch main branch
+        run: git fetch origin main:main # Fetch the main branch reference for comparison
+
+      - name: Identify changed samples
+        run: |
+          git diff --name-only HEAD main | grep '^samples/' | awk -F'/' '{print $1"/"$2}' | sort | uniq > changed_samples.txt
+          if [ ! -s changed_samples.txt ]; then
+            echo "No samples have changed. Exiting..."
+            exit 0
+          fi
+
+      - name: Install Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: tools/testing/go.mod
+          cache-dependency-path: |
+            tools/testing/go.sum
+
+      - name: Build the test tool using Go
+        run: |
+          go mod tidy
+          go build ./cmd/loadtest
+        working-directory: tools/testing/
+
+      - name: Install Defang
+        run: |
+          . <(curl -Ls https://s.defang.io/install)
+
+      - name: Run tests
+        id: run-tests
+        shell: bash # implies set -o pipefail, so pipe below will keep the exit code from loadtest
+        env:
+          FIXED_VERIFIER_PK: ${{ secrets.FIXED_VERIFIER_PK }}
+          TEST_AWS_ACCESS_KEY: ${{ secrets.TEST_AWS_ACCESS_KEY }}
+          TEST_AWS_SECRET_KEY: ${{ secrets.TEST_AWS_SECRET_KEY }}
+          TEST_BOARD_PASSWORD: ${{ secrets.TEST_BOARD_PASSWORD }}
+          TEST_DATABASE_HOST: ${{ secrets.TEST_DATABASE_HOST }}
+          TEST_DATABASE_NAME: ${{ secrets.TEST_DATABASE_NAME }}
+          TEST_DATABASE_PASSWORD: ${{ secrets.TEST_DATABASE_PASSWORD }}
+          TEST_DATABASE_URL: ${{ secrets.TEST_DATABASE_URL }}
+          TEST_DATABASE_USERNAME: ${{ secrets.TEST_DATABASE_USERNAME }}
+          TEST_HASURA_GRAPHQL_ADMIN_SECRET: ${{ secrets.TEST_HASURA_GRAPHQL_ADMIN_SECRET }}
+          TEST_HASURA_GRAPHQL_DATABASE_URL: ${{ secrets.TEST_HASURA_GRAPHQL_DATABASE_URL }}
+          TEST_HF_TOKEN: ${{ secrets.TEST_HF_TOKEN }}
+          TEST_MB_DB_DBNAME: ${{ secrets.TEST_MB_DB_DBNAME }}
+          TEST_MB_DB_HOST: ${{ secrets.TEST_MB_DB_HOST }}
+          TEST_MB_DB_PASS: ${{ secrets.TEST_MB_DB_PASS }}
+          TEST_MB_DB_PORT: ${{ secrets.TEST_MB_DB_PORT }}
+          TEST_MB_DB_USER: ${{ secrets.TEST_MB_DB_USER }}
+          TEST_NC_DB: ${{ secrets.TEST_NC_DB }}
+          TEST_NC_S3_ENDPOINT: ${{ secrets.TEST_NC_S3_ENDPOINT }}
+          TEST_NC_S3_BUCKET_NAME: ${{ secrets.TEST_NC_S3_BUCKET_NAME }}
+          TEST_NC_S3_REGION: ${{ secrets.TEST_NC_S3_REGION }}
+          TEST_NC_S3_ACCESS_KEY: ${{ secrets.TEST_NC_S3_ACCESS_KEY }}
+          TEST_NC_S3_ACCESS_SECRET: ${{ secrets.TEST_NC_S3_ACCESS_SECRET }}
+          TEST_OPENAI_KEY: ${{ secrets.TEST_OPENAI_KEY }}
+          TEST_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
+          TEST_QUEUE: ${{ secrets.TEST_QUEUE }}
+          TEST_SECRET_KEY_BASE: ${{ secrets.TEST_SECRET_KEY_BASE }}
+          TEST_SESSION_SECRET: ${{ secrets.TEST_SESSION_SECRET }}
+          TEST_SLACK_CHANNEL_ID: ${{ secrets.TEST_SLACK_CHANNEL_ID }}
+          TEST_SLACK_TOKEN: ${{ secrets.TEST_SLACK_TOKEN }}
+        run: |
+          SAMPLES=$(sed 's|^samples/||' changed_samples.txt | paste -s -d ',' -)
+          echo "Running tests for samples: $SAMPLES"
+
+          mkdir output
+          # concurrency is set to 10 to avoid exhausting our fargate vCPU limits when running
+          # kaniko builds, which consume 4 vCPUs each. the limit is currently set to 60--6.5 of
+          # which are used to run the staging stack. So floor((60-6.5)/4) = 13 is our upper limit
+          ./tools/testing/loadtest -c fabric-staging.defang.dev:443 --timeout=15m --concurrency=10 -s $SAMPLES -o output --markdown=true | tee output/summary.log | grep -v '^\s*[-*]' # removes load sample log lines
+
+      - name: Upload Output as Artifact
+        uses: actions/upload-artifact@v4
+        if: success() || steps.run-tests.outcome == 'failure' # Always upload result unless cancelled
+        with:
+          name: program-output
+          path: output/**

samples/nextjs-blog/compose.yaml

@@ -7,6 +7,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     deploy:
       resources:
         reservations:

samples/nextjs-documentation/compose.yaml

@@ -7,6 +7,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     deploy:
       resources:
         reservations:

samples/nodejs-chatroom/compose.yaml

@@ -7,6 +7,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     deploy:
       resources:
         reservations:

samples/nodejs-express/compose.yaml

@@ -6,6 +6,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     deploy:
       resources:
         reservations:

samples/nodejs-form/compose.yaml

@@ -7,6 +7,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     deploy:
       resources:
         reservations:

samples/nodejs-http/compose.yaml

@@ -7,6 +7,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     deploy:
       resources:
         reservations:

samples/nodejs-openai/compose.yaml

@@ -7,6 +7,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     environment:
       - OPENAI_KEY
     deploy:

samples/nodejs-rest-api/compose.yaml

@@ -7,6 +7,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     deploy:
       resources:
         reservations:

samples/nodejs-s3/compose.yaml

@@ -7,6 +7,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     environment:
       - AWS_ACCESS_KEY
       - AWS_SECRET_KEY

samples/rails/app/Dockerfile

@@ -28,8 +28,8 @@ RUN bundle config set force_ruby_platform true
 # Install application gems
 COPY Gemfile Gemfile.lock ./
 RUN bundle install && \
-    rm -rf ~/.bundle/ "${BUNDLE_PATH}"/ruby/*/cache "${BUNDLE_PATH}"/ruby/*/bundler/gems/*/.git && \
-    bundle exec bootsnap precompile --gemfile
+    rm -rf ~/.bundle/ "${BUNDLE_PATH}"/ruby/*/cache "${BUNDLE_PATH}"/ruby/*/bundler/gems/*/.git \
+    && bundle exec bootsnap precompile --gemfile
 
 # Copy application code
 COPY . .
@@ -45,8 +45,8 @@ RUN SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile
 FROM base
 
 # Install packages needed for deployment
-RUN apt-get update -qq && \
-    apt-get install --no-install-recommends -y curl libvips libpq-dev && \
+RUN apt-get update -qq \
+    && apt-get install --no-install-recommends -y curl libvips libpq-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -55,8 +55,8 @@ COPY --from=build /usr/local/bundle /usr/local/bundle
 COPY --from=build /rails /rails
 
 # Run and own only the runtime files as a non-root user for security
-RUN useradd rails --create-home --shell /bin/bash && \
-    chown -R rails:rails db log storage tmp
+RUN useradd rails --create-home --shell /bin/bash \
+    && chown -R rails:rails db log storage tmp
 USER rails:rails
 
 # Entrypoint prepares the database.

samples/rails/compose.yaml

@@ -13,6 +13,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     deploy:
       resources:
         reservations:

samples/vllm/compose.yaml

@@ -13,6 +13,7 @@ services:
           memory: 8192M
           devices:
             - capabilities: ["gpu"]
+              count: 1
     healthcheck:
       test: ["CMD", "python3", "-c", "import sys, urllib.request;urllib.request.urlopen(sys.argv[1]).read()", "http://localhost:8000/health"]
       interval: 1m
@@ -26,6 +27,7 @@ services:
     ports:
       - mode: ingress
         target: 3000
+        published: 3000
     deploy:
       resources:
         reservations:

tools/testing/.gitignore

@@ -0,0 +1,3 @@
+/loadtest
+/runtest
+/out

tools/testing/cleanup/cleanup.go

@@ -0,0 +1,42 @@
+package cleanup
+
+import (
+	"errors"
+	"sync"
+)
+
+type CleanUpFunc func() error
+
+var defaultCleanUp = &CleanUp{}
+
+func Add(fn CleanUpFunc) {
+	defaultCleanUp.Add(fn)
+}
+
+func Run() error {
+	return defaultCleanUp.Run()
+}
+
+type CleanUp struct {
+	cleanups []CleanUpFunc
+	lock     sync.Mutex
+}
+
+func (c *CleanUp) Add(fn CleanUpFunc) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.cleanups = append(c.cleanups, fn)
+}
+
+func (c *CleanUp) Run() error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	var errs []error
+	// Run the clean up functions in reverse order
+	for i := len(c.cleanups) - 1; i >= 0; i-- {
+		if err := c.cleanups[i](); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	return errors.Join(errs...)
+}