Dockerizing

.dockerignore

@@ -0,0 +1,4 @@
+.venv
+.git
+__pycache__/
+datasets/

.env.template

@@ -1,2 +1,13 @@
-HF_TOKEN=
-HF_ORGANIZATION=
+OPENROUTER_API_KEY=
+
+BENCHMARK_NAME="test"
+BENCHMARK_SYSTEM_PROMPT="test prompt"
+INPUT_S3_BUCKET="layerlens-private-test-organization"
+INPUT_S3_KEY="benchmarks/test-project/benchmark-name/data.zip"
+OUTPUT_S3_BUCKET="layerlens-private-test-organization"
+OUTPUT_S3_KEY="benchmarks/test-project/benchmark-name/"
+
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+HF_HUB_OFFLINE=1

.github/workflows/ci.yaml

@@ -0,0 +1,47 @@
+name: YourBench CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.12]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install uv
+      run: pip install uv
+
+    - name: Create virtual environment
+      run: uv venv
+
+    - name: Install dependencies
+      run: |
+        . .venv/bin/activate
+        uv pip install -e .
+        uv pip install pytest pytest-cov
+
+    - name: Run tests
+      run: |
+        . .venv/bin/activate
+        python -m pytest tests/ --cov=yourbench --cov-report=xml
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: false

.github/workflows/quality.yaml

@@ -1,5 +1,8 @@
 name: Quality
 
+permissions:
+  contents: read
+
 on:
   push:
     branches:

Dockerfile

@@ -0,0 +1,43 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy all yourbench files
+COPY . .
+
+# Install dependencies and yourbench in editable mode
+RUN pip install --upgrade pip && \
+    pip install boto3 pyyaml awscli && \
+    pip install -e .
+
+# Verify installation
+RUN yourbench --version || echo "Yourbench installation verification failed but continuing build"
+
+# Environment variables (will be overridden at runtime)
+ENV BENCHMARK_NAME=""
+ENV BENCHMARK_SYSTEM_PROMPT=""
+ENV INPUT_S3_BUCKET=""
+ENV INPUT_S3_KEY=""
+ENV OUTPUT_S3_BUCKET=""
+ENV OUTPUT_S3_KEY=""
+ENV OPENROUTER_API_KEY=""
+ENV AWS_ACCESS_KEY_ID=""
+ENV AWS_SECRET_ACCESS_KEY=""
+ENV AWS_DEFAULT_REGION="us-east-1"
+ENV WORKDIR="/app"
+
+# Create a startup script to run the processing workflow
+RUN printf '#!/bin/bash\n\
+    echo "Running yourbench workflow..."\n\
+    exec python run_yourbench.py\n' > /app/entrypoint.sh && \
+    chmod +x /app/entrypoint.sh
+
+# Use the startup script as entry point
+ENTRYPOINT ["/app/entrypoint.sh"]

README.docker.md

@@ -0,0 +1,67 @@
+# YourbenchProcessor Docker Container
+
+This Docker container automates the process of:
+1. Downloading data from AWS S3
+2. Processing with yourbench
+3. Uploading results back to AWS S3
+
+## Required Environment Variables
+
+The container requires the following environment variables:
+
+- `INPUT_S3_BUCKET`: S3 bucket name for input data
+- `INPUT_S3_KEY`: S3 object key for input data (ZIP file)
+- `OUTPUT_S3_BUCKET`: S3 bucket name for output results
+- `OUTPUT_S3_KEY`: S3 object key for output results
+- `OPENROUTER_API_KEY`: API key for OpenRouter
+- `AWS_ACCESS_KEY_ID`: AWS access key with S3 permissions
+- `AWS_SECRET_ACCESS_KEY`: AWS secret key with S3 permissions
+- `AWS_DEFAULT_REGION`: AWS region (default: us-east-1)
+
+## Building the Docker Image
+
+```bash
+docker build -t yourbench-processor .
+```
+
+## Running the Container
+
+```bash
+docker run -e INPUT_S3_BUCKET=your-input-bucket \
+           -e INPUT_S3_KEY=input/data.zip \
+           -e OUTPUT_S3_BUCKET=your-output-bucket \
+           -e OUTPUT_S3_KEY=output/results.zip \
+           -e OPENROUTER_API_KEY=your-openrouter-key \
+           -e AWS_ACCESS_KEY_ID=your-aws-key-id \
+           -e AWS_SECRET_ACCESS_KEY=your-aws-secret \
+           -e AWS_DEFAULT_REGION=us-east-1 \
+           yourbench-processor
+```
+
+## Process Flow
+
+1. Downloads the specified zip file from S3
+2. Extracts contents to `task/data/raw` directory
+3. Creates a `config.yaml` file in `task/dataset` directory
+4. Runs yourbench with the created config
+5. Zips the `task/dataset` directory
+6. Uploads the zipped results back to S3
+
+## Local Testing
+
+For local testing without Docker:
+
+```bash
+# Set environment variables
+export INPUT_S3_BUCKET=your-input-bucket
+export INPUT_S3_KEY=input/data.zip
+export OUTPUT_S3_BUCKET=your-output-bucket
+export OUTPUT_S3_KEY=output/results.zip
+export OPENROUTER_API_KEY=your-openrouter-key
+export AWS_ACCESS_KEY_ID=your-aws-key-id
+export AWS_SECRET_ACCESS_KEY=your-aws-secret
+export AWS_DEFAULT_REGION=us-east-1
+
+# Run the script
+python run_yourbench.py
+```