diff --git a/.env.example b/.env.example index abea3f77..b5ca0ba9 100644 --- a/.env.example +++ b/.env.example @@ -44,14 +44,3 @@ AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_DEFAULT_REGION=ap-south-1 AWS_S3_BUCKET_PREFIX="bucket-prefix-name" - -# OpenAI - -OPENAI_API_KEY="this_is_not_a_secret" -LANGFUSE_PUBLIC_KEY="this_is_not_a_secret" -LANGFUSE_SECRET_KEY="this_is_not_a_secret" -LANGFUSE_HOST="this_is_not_a_secret" - -# Misc - -CI="" diff --git a/.env.test.example b/.env.test.example new file mode 100644 index 00000000..9065e4e6 --- /dev/null +++ b/.env.test.example @@ -0,0 +1,30 @@ +ENVIRONMENT=testing + +PROJECT_NAME="AI Platform" +STACK_NAME=ai-platform + +#Backend +SECRET_KEY=changethis +FIRST_SUPERUSER=superuser@example.com +FIRST_SUPERUSER_PASSWORD=changethis +EMAIL_TEST_USER="test@example.com" + +# Postgres + +POSTGRES_SERVER=localhost +POSTGRES_PORT=5432 +POSTGRES_DB=ai_platform_test +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres + +# Configure these with your own Docker registry images + +DOCKER_IMAGE_BACKEND=backend +DOCKER_IMAGE_FRONTEND=frontend + +# AWS + +AWS_ACCESS_KEY_ID=this_is_a_test_key +AWS_SECRET_ACCESS_KEY=this_is_a_test_key +AWS_DEFAULT_REGION=ap-south-1 +AWS_S3_BUCKET_PREFIX="bucket-prefix-name" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 917203f2..daccfd6e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -18,10 +18,6 @@ jobs: count: [100] env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} - LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} - LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }} LOCAL_CREDENTIALS_ORG_OPENAI_API_KEY: ${{ secrets.LOCAL_CREDENTIALS_ORG_OPENAI_API_KEY }} LOCAL_CREDENTIALS_API_KEY: ${{ secrets.LOCAL_CREDENTIALS_API_KEY }} diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index 5b40cdd3..b617c59e 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -15,7 +15,7 @@ jobs: env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_DB: ai_platform + POSTGRES_DB: ai_platform_test ports: - 5432:5432 options: --health-cmd "pg_isready -U postgres" --health-interval 10s --health-timeout 5s --health-retries 5 @@ -34,7 +34,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Making env file - run: cp .env.example .env + run: | + cp .env.test.example .env + cp .env.test.example .env.test - name: Install uv uses: astral-sh/setup-uv@v6 diff --git a/.gitignore b/.gitignore index 0d8e46df..ad2127f4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ node_modules/ /playwright/.cache/ # Environments -.env +.env* .venv env/ venv/ diff --git a/backend/app/api/main.py b/backend/app/api/main.py index df0b1016..9d3ae489 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -37,5 +37,5 @@ api_router.include_router(users.router) api_router.include_router(utils.router) -if settings.ENVIRONMENT == "local": +if settings.ENVIRONMENT in ["development", "testing"]: api_router.include_router(private.router) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 24779bf3..4af13cb6 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -1,7 +1,7 @@ import secrets import warnings import os -from typing import Annotated, Any, Literal +from typing import Any, Literal from pydantic import ( EmailStr, @@ -15,30 +15,30 @@ from typing_extensions import Self -def parse_cors(v: Any) -> list[str] | str: - if isinstance(v, str) and not v.startswith("["): - return [i.strip() for i in v.split(",")] - elif isinstance(v, list | str): - return v - raise ValueError(v) +def parse_cors(origins: Any) -> list[str] | str: + # If it's a plain comma-separated string, split it into a list + if isinstance(origins, str) and not origins.startswith("["): + return [origin.strip() for origin in origins.split(",")] + # If it's already a list or JSON-style string, just return it + elif isinstance(origins, (list, str)): + return origins + raise ValueError(f"Invalid CORS origins format: {origins!r}") class Settings(BaseSettings): model_config = SettingsConfigDict( - # Use top level .env file (one level above ./backend/) - env_file="../.env", + # env_file will be set dynamically in get_settings() env_ignore_empty=True, extra="ignore", ) - LANGFUSE_PUBLIC_KEY: str - LANGFUSE_SECRET_KEY: str - LANGFUSE_HOST: str # 🇪🇺 EU region - OPENAI_API_KEY: str + API_V1_STR: str = "/api/v1" SECRET_KEY: str = secrets.token_urlsafe(32) # 60 minutes * 24 hours * 1 days = 1 days ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 1 - ENVIRONMENT: Literal["local", "staging", "production"] = "local" + ENVIRONMENT: Literal[ + "development", "testing", "staging", "production" + ] = "development" PROJECT_NAME: str SENTRY_DSN: HttpUrl | None = None @@ -84,7 +84,7 @@ def _check_default_secret(self, var_name: str, value: str | None) -> None: f'The value of {var_name} is "changethis", ' "for security, please change it, at least for deployments." ) - if self.ENVIRONMENT == "local": + if self.ENVIRONMENT in ["development", "testing"]: warnings.warn(message, stacklevel=1) else: raise ValueError(message) @@ -100,4 +100,17 @@ def _enforce_non_default_secrets(self) -> Self: return self -settings = Settings() # type: ignore +def get_settings() -> Settings: + """Get settings with appropriate env file based on ENVIRONMENT.""" + environment = os.getenv("ENVIRONMENT", "development") + + # Determine env file + env_files = {"testing": "../.env.test", "development": "../.env"} + env_file = env_files.get(environment, "../.env") + + # Create Settings instance with the appropriate env file + return Settings(_env_file=env_file) + + +# Export settings instance +settings = get_settings() diff --git a/backend/app/core/db.py b/backend/app/core/db.py index ba991fb3..510e9da7 100644 --- a/backend/app/core/db.py +++ b/backend/app/core/db.py @@ -1,10 +1,30 @@ from sqlmodel import Session, create_engine, select from app import crud -from app.core.config import settings from app.models import User, UserCreate -engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) + +def get_engine(): + """Get database engine with current settings.""" + # Import settings dynamically to get the current instance + from app.core.config import settings + + # Configure connection pool settings + # For testing, we need more connections since tests run in parallel + pool_size = 20 if settings.ENVIRONMENT == "development" else 5 + max_overflow = 30 if settings.ENVIRONMENT == "development" else 10 + + return create_engine( + str(settings.SQLALCHEMY_DATABASE_URI), + pool_size=pool_size, + max_overflow=max_overflow, + pool_pre_ping=True, + pool_recycle=300, # Recycle connections after 5 minutes + ) + + +# Create a default engine for backward compatibility +engine = get_engine() # make sure all SQLModel models are imported (app.models) before initializing DB diff --git a/backend/app/load_env.py b/backend/app/load_env.py new file mode 100644 index 00000000..f465b23b --- /dev/null +++ b/backend/app/load_env.py @@ -0,0 +1,13 @@ +import os +from dotenv import load_dotenv + + +def load_environment(): + env = os.getenv("ENVIRONMENT", "development") + + # Use the same path as config.py expects (one level above backend/) + env_file = "../.env" + if env == "testing": + env_file = "../.env.test" + + load_dotenv(env_file) diff --git a/backend/app/main.py b/backend/app/main.py index 903e02ef..0b473484 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,19 +1,24 @@ import sentry_sdk + from fastapi import FastAPI from fastapi.routing import APIRoute from asgi_correlation_id.middleware import CorrelationIdMiddleware from app.api.main import api_router from app.core.config import settings -import app.core.logger from app.core.exception_handlers import register_exception_handlers from app.core.middleware import http_request_logger +from app.load_env import load_environment + +# Load environment variables +load_environment() + def custom_generate_unique_id(route: APIRoute) -> str: return f"{route.tags[0]}-{route.name}" -if settings.SENTRY_DSN and settings.ENVIRONMENT != "local": +if settings.SENTRY_DSN and settings.ENVIRONMENT != "development": sentry_sdk.init(dsn=str(settings.SENTRY_DSN), enable_tracing=True) app = FastAPI( diff --git a/backend/app/tests/conftest.py b/backend/app/tests/conftest.py index 02773871..954059e7 100644 --- a/backend/app/tests/conftest.py +++ b/backend/app/tests/conftest.py @@ -1,9 +1,15 @@ import pytest +import os + +# Set environment before importing ANYTHING else +os.environ["ENVIRONMENT"] = "testing" + from fastapi.testclient import TestClient from sqlmodel import Session from sqlalchemy import event from collections.abc import Generator +# Now import after setting environment from app.core.config import settings from app.core.db import engine from app.api.deps import get_db diff --git a/backend/scripts/tests-start.sh b/backend/scripts/tests-start.sh index 89dcb0da..1f6cc798 100644 --- a/backend/scripts/tests-start.sh +++ b/backend/scripts/tests-start.sh @@ -2,6 +2,16 @@ set -e set -x +# Set environment for testing +export ENVIRONMENT=testing + python app/tests_pre_start.py +# Run pending migrations for test database +uv run alembic upgrade head +if [ $? -ne 0 ]; then + echo 'Error: Test database migrations failed' + exit 1 +fi + bash scripts/test.sh "$@" diff --git a/docker-compose.yml b/docker-compose.yml index e8e1330d..78a4af52 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,10 +72,6 @@ services: - POSTGRES_USER=${POSTGRES_USER?Variable not set} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD?Variable not set} - SENTRY_DSN=${SENTRY_DSN} - - OPENAI_API_KEY=${OPENAI_API_KEY} - - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY} - - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY} - - LANGFUSE_HOST=${LANGFUSE_HOST} - LOCAL_CREDENTIALS_ORG_OPENAI_API_KEY=${LOCAL_CREDENTIALS_ORG_OPENAI_API_KEY} - LOCAL_CREDENTIALS_API_KEY=${LOCAL_CREDENTIALS_API_KEY} - EMAIL_TEST_USER=${EMAIL_TEST_USER} @@ -112,10 +108,6 @@ services: - POSTGRES_USER=${POSTGRES_USER?Variable not set} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD?Variable not set} - SENTRY_DSN=${SENTRY_DSN} - - OPENAI_API_KEY=${OPENAI_API_KEY} - - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY} - - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY} - - LANGFUSE_HOST=${LANGFUSE_HOST} - LOCAL_CREDENTIALS_ORG_OPENAI_API_KEY=${LOCAL_CREDENTIALS_ORG_OPENAI_API_KEY} - LOCAL_CREDENTIALS_API_KEY=${LOCAL_CREDENTIALS_API_KEY} - EMAIL_TEST_USER=${EMAIL_TEST_USER}