ansari-project · OdyAsh · Sep 28, 2025 · Sep 28, 2025 · Oct 14, 2025 · Oct 17, 2025
diff --git a/.env.example b/.env.example
@@ -1,6 +1,45 @@
+# Where ansari-backend is currently deployed
 DEPLOYMENT_TYPE="development"    # Deployment type (development, staging, production)
+
+###################################### Related to ansari-frontend ######################################
+
 FRONTEND_URL="http://localhost:8081"
 
+###################################### Related to ansari-whatsapp ######################################
+
+# Shared API key for authenticating requests from ansari-whatsapp microservice
+# References:
+# - https://fastapi.tiangolo.com/tutorial/security/
+# - https://fastapi.tiangolo.com/advanced/security/http-basic-auth/
+# - https://www.python-httpx.org/advanced/#client-instances
+# Security: Must match the WHATSAPP_SERVICE_API_KEY in ansari-whatsapp's .env file
+# Used to verify X-Whatsapp-Api-Key header on /whatsapp/v2/* endpoints
+WHATSAPP_SERVICE_API_KEY="your_generated_secret_key_here"
+
+###################################### Related to Ansari for Android/IOS ######################################
+
+# iOS app build versions
+IOS_MINIMUM_BUILD_VERSION="1"           # Minimum build version required for iOS app
+IOS_LATEST_BUILD_VERSION="1"            # Latest available build version for iOS app
+
+# Android app build versions
+ANDROID_MINIMUM_BUILD_VERSION="1"       # Minimum build version required for Android app
+ANDROID_LATEST_BUILD_VERSION="1"        # Latest available build version for Android app
+
+
+###################################### Related to CORS ######################################
+
+# Origins to be allowed by the backend
+ORIGINS="https://ansari.chat,https://www.ansari.chat,https://pre.ansari.chat"
+
+###################################### Related to the DB ######################################
+
+# Database connection string
+MONGO_URL="mongodb://localhost:27017"
+MONGO_DB_NAME="ansari_db"
+
+###################################### Related to 3rd Party Services ######################################
+
 KALEMAT_API_KEY=""          # Token for Qur'an and Hadith search
 ANTHROPIC_API_KEY=""        # API key for Claude AI model
 OPENAI_API_KEY=""           # Token for GPT-4 (Optional)
@@ -12,76 +51,20 @@ MAILCHIMP_API_KEY=""         # API key
 MAILCHIMP_SERVER_PREFIX="" # Server prefix (data center)
 MAILCHIMP_LIST_ID=""      # List ID
 
-# Database connection string
-MONGO_URL="mongodb://localhost:27017"
-MONGO_DB_NAME="ansari_db"
-
-SECRET_KEY="secret"         # Secret key for signing tokens
-
-# Origins to be allowed by the backend
-ORIGINS="https://ansari.chat,https://www.ansari.chat,https://pre.ansari.chat"
-
 # Vectara search engine configuration
 VECTARA_API_KEY=""           # Authentication token for Vectara API
 
 QURAN_DOT_COM_API_KEY="" # This is the API key we give to quran.com to access us, not for us to access them
 
-# Directory for storing templates
-template_dir="."             # Directory path for templates
-
-# Related to WhatsApp Business and Meta (leave empty if you're not planning to use WhatsApp)
-# Source 1: https://www.youtube.com/watch?v=KP6_BUw3i0U
-#   Watch Until 32:25, while quickly skimming through the non-python code parts
-# Source 2 (mentioned in video above): https://glitch.com/edit/#!/insidious-tartan-alvarezsaurus
-#           (the `verification_webhook` endpoint in `main_whatsapp` is inspired by the above URL)
-# Source 3 (optional): https://developers.facebook.com/blog/post/2022/10/24/sending-messages-with-whatsapp-in-your-python-applications/#u_0_39_8q
-
-# Moreover, if want to test whatsapp's webhook locally, you can use zrok on a reserved URL with a zrok "share token"
-# obtained by contacting its current holder: https://github.com/OdyAsh (source 1, 2 below)
-# Alternatively, you can change the webhook url all together (source 3, 4 below)
-# Check these sources for more details:
-# Source 1: https://dev.to/odyash/quickly-share-your-app-with-zrok-4ihp
-# Source 2: https://openziti.discourse.group/t/how-do-i-use-a-reserved-share-on-different-devices/2379/2
-# Source 3: https://youtu.be/KP6_BUw3i0U?t=1294
-#   (@21:33 and 25:30, however they use glitch instead of zrok, so the video here is just to give you an idea how to setup a webhook)
-# Source 4 (where you can change callback url, given that your facebook account gets access by the app's admins):
-#   https://developers.facebook.com/apps/871020755148175/whatsapp-business/wa-settings/
-#   NOTE 1: When you see the `Callback URL`, it will be something like "https://ZROK_SHARE_TOKEN.share.zrok.io/whatsapp/v1"
-#           (The `/whatsapp/v1` endpoint can be found in `main_whatsapp.py`'s endpoints, that's why it's in the url above)
-#   NOTE 2: If an unexpected 3rd party discovers the ZROK_SHARE_TOKEN,
-#   a new one will have to be generated, then added to Meta's callback URL of the *testing* app
-#   (Noting that the *production* app's callback URL will be different anyway, so the 3rd party won't be able to access that app)
-#   (but we still don't want random calls to be made to our testing app, so that's why we'll still have to change an exposed token :])
-#   NOTE 3: Obviously, that `871...175` in the above URL is the testing app's public id, so if this link still doesn't work even after you gain access,
-#   then the admins most probably created a new test app instance
-
-WHATSAPP_API_VERSION="<<CURRENT-VERSION-AS-MENTIONED-IN-SOURCE-URL-ABOVE>>"
-
-# NOTE 1: Contact the team to see whatsapp's 2 phone nums -> one for prod. env. and the other for local/stage testing
-# NOTE 2: If you encounter this error: 
-#   "Client error '400 Bad Request' for url 'https://graph.facebook.com/v22.0/<WHATSAPP_BUSINESS_PHONE_NUMBER_ID>/messages'"
-#   then either:
-#       (1) the <WHATSAPP_BUSINESS_PHONE_NUMBER_ID> got deleted by Meta due to inactive use for a long time.
-#           If this happens, then open the page in source 3 (i.e., youtube video URL above) @6:11. 
-#           Then, under "step 1" in the middle of the page, you'll see a selection box with the live WhatsApp number;
-#           click on it, then click "Get new test number". Then, wait for a while, then you'll see the new number; 
-#           Now copy its number id and paste it in <WHATSAPP_BUSINESS_PHONE_NUMBER_ID>
-#       (2) Meta updated its version and so now <WHATSAPP_API_VERSION> is outdated.
-#           You can verify this by opening source 3 @26:13 and checking the version associated with `messages`
-#       (3) you sent a message with an incorrect format. Check source 3; 26:10->26:28, click on `messages` to see how it should be sent
-WHATSAPP_BUSINESS_PHONE_NUMBER_ID="<<YOUR-WHATSAPP-BUSINESS-PHONE-NUMBER-ID>>"
-
-# NOTE 1: check video in source 3 above from 30:45 to 32:15 to see where we get the access token
-# NOTE 2: Contact the team to see their 2 access tokens -> one for prod. env. and the other for local/stage testing
-# NOTE 3: If you want to debug or check the validity of an access token, check this URL:
-#   https://developers.facebook.com/tools/debug/accesstoken/?access_token=<YOUR-ACCESS-TOKEN>
-WHATSAPP_ACCESS_TOKEN_FROM_SYS_USER="<<YOUR-SYSTEM-USER-ACCESS-TOKEN>"
-
-WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK="<<The-VERIFIFY-TOKEN-CURRENTLY-USED-TO-VERIFY-META'S-CALLBACK-URL>>"
-WHATSAPP_CHAT_RETENTION_HOURS=3
-ZROK_SHARE_TOKEN="<<THE-ZROK-SHARE-TOKEN-CURRENTLY-USED-IN-META'S-CALLBACK-URL>>"
-
-# Related to internal code logic
+SENTRY_DSN=""                # Sentry DSN for error tracking
+
+###################################### Related to Internal Code Logic ######################################
+
+SECRET_KEY="secret"         # Secret key for signing tokens
+
+# Directory path for storing templates
+template_dir="."             
+
 # Leave the values below when locally debugging the application
 # In production, don't add them to environment variables, or add them as "INFO"/"False" respectively
 LOGGING_LEVEL="DEBUG"
@@ -90,16 +73,6 @@ DEV_MODE="True"
 # Application version control settings
 MAINTENANCE_MODE="False"                # Whether the application is in maintenance mode
 
-# iOS app build versions
-IOS_MINIMUM_BUILD_VERSION="1"           # Minimum build version required for iOS app
-IOS_LATEST_BUILD_VERSION="1"            # Latest available build version for iOS app
-
-# Android app build versions
-ANDROID_MINIMUM_BUILD_VERSION="1"       # Minimum build version required for Android app
-ANDROID_LATEST_BUILD_VERSION="1"        # Latest available build version for Android app
-
-SENTRY_DSN=""                # Sentry DSN for error tracking
-
 # To get rid of .py[cod] files (This should key should NOT be set in production!)
 # This is only to de-clutter your local development environment
 # Details: https://docs.python-guide.org/writing/gotchas/#disabling-bytecode-pyc-files

diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
@@ -65,6 +65,7 @@ jobs:
           USUL_API_TOKEN: ${{ format('{0}{1}', secrets.SSM_ROOT, 'usul-api-token') }}
           SENTRY_DSN: ${{ format('{0}{1}', secrets.SSM_ROOT, 'sentry-dsn') }}
 
+          WHATSAPP_SERVICE_API_KEY: ${{ format('{0}{1}', secrets.SSM_ROOT, 'whatsapp-service-api-key') }}
           WHATSAPP_ACCESS_TOKEN_FROM_SYS_USER: ${{ format('{0}{1}', secrets.SSM_ROOT, 'whatsapp-access-token-from-sys-user') }}
           WHATSAPP_BUSINESS_PHONE_NUMBER_ID: ${{ format('{0}{1}', secrets.SSM_ROOT, 'whatsapp-business-phone-number-id') }}
           WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK: ${{ format('{0}{1}', secrets.SSM_ROOT, 'whatsapp-verify-token-for-webhook') }}
@@ -109,6 +110,7 @@ jobs:
             USUL_API_TOKEN
             SENTRY_DSN
 
+            WHATSAPP_SERVICE_API_KEY
             WHATSAPP_ACCESS_TOKEN_FROM_SYS_USER
             WHATSAPP_BUSINESS_PHONE_NUMBER_ID
             WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK

diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
@@ -65,6 +65,7 @@ jobs:
           USUL_API_TOKEN: ${{ format('{0}{1}', secrets.SSM_ROOT, 'usul-api-token') }}
           SENTRY_DSN: ${{ format('{0}{1}', secrets.SSM_ROOT, 'sentry-dsn') }}
 
+          WHATSAPP_SERVICE_API_KEY: ${{ format('{0}{1}', secrets.SSM_ROOT, 'whatsapp-service-api-key') }}
           WHATSAPP_ACCESS_TOKEN_FROM_SYS_USER: ${{ format('{0}{1}', secrets.SSM_ROOT, 'whatsapp-access-token-from-sys-user') }}
           WHATSAPP_BUSINESS_PHONE_NUMBER_ID: ${{ format('{0}{1}', secrets.SSM_ROOT, 'whatsapp-business-phone-number-id') }}
           WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK: ${{ format('{0}{1}', secrets.SSM_ROOT, 'whatsapp-verify-token-for-webhook') }}
@@ -109,6 +110,7 @@ jobs:
             USUL_API_TOKEN
             SENTRY_DSN
 
+            WHATSAPP_SERVICE_API_KEY
             WHATSAPP_ACCESS_TOKEN_FROM_SYS_USER
             WHATSAPP_BUSINESS_PHONE_NUMBER_ID
             WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 # Folders
 .conda/
 .claude/
+.specstory/
 .venv/
 .vscode/
 abandoned/

diff --git a/CLAUDE.md b/CLAUDE.md
@@ -31,8 +31,18 @@
 - Delete branches after they're merged to keep the repository clean
 
 ## Build/Test/Lint Commands
-- Install dependencies: `pip install -r requirements.txt`
-- Run backend service: `uvicorn main_api:app --reload`
+- Install dependencies: `uv sync` - Installs all dependencies from pyproject.toml and uv.lock
+- Run backend service:
+  1. Use venv python directly: `.venv/Scripts/python.exe src/ansari/app/main_api.py`
+  - Alternative (if uvicorn is available): `uvicorn main_api:app --reload`
+
+  **Note**: Direct venv python path is used because `source .venv/Scripts/activate` may not properly activate the virtual environment in bash.
+
+  **Testing changes**: Auto-reload can be unreliable. For reliable testing after code changes:
+  1. Kill the running server (`KillShell` tool)
+  2. Start new server: `.venv/Scripts/python.exe src/ansari/app/main_api.py`
+  3. Wait 10 seconds for startup to complete
+  4. Then test with curl
 - Run CLI version (interactive):
   - Claude: `python src/ansari/app/main_stdio.py -a AnsariClaude`
   - OpenAI: `python src/ansari/app/main_stdio.py -a Ansari`
@@ -48,6 +58,14 @@
   - Build package: `python -m build`
   - Upload to PyPI: `twine upload dist/*` (requires PyPI credentials)
 
+## Package Management
+- **Install dependencies**: `uv sync` - Installs all dependencies from pyproject.toml and uv.lock
+- **Add new package**: `uv add <package>` - Adds package to dependencies and updates lock file
+- **Add development dependency**: `uv add --dev <package>` - Adds package to dev dependencies
+- **Remove package**: `uv remove <package>` - Removes package from dependencies
+- **Create virtual environment**: `uv venv` - Creates .venv directory (if not exists)
+- **Update dependencies**: `uv lock` - Updates uv.lock file with latest compatible versions
+
 ## Code Style Guidelines
 - **Imports**: Use absolute imports within the `ansari` package
 - **Formatting**: Double quotes for strings, 4-space indentation

diff --git a/docs/fastapi/async_await_backgroundtasks_logs_for_tracing.log b/docs/fastapi/async_await_backgroundtasks_logs_for_tracing.log
@@ -1,4 +1,4 @@
-(manually inserted log) ------------------------------------------------------------------------------ FastAPI receives a message from a whatsapp user on /whatsapp/v1 endpoint, so the event loop picks the `main_webhook()` to be executed now ------------------------------------------------------------------------------
+(manually inserted log) ------------------------------------------------------------------------------ FastAPI receives a message from a whatsapp user on /whatsapp/v2 endpoint, so the event loop picks the `main_webhook()` to be executed now ------------------------------------------------------------------------------
 (manually inserted log) ------------------------------------------------------------------------------ Execution in `main_webhook()` starts ------------------------------------------------------------------------------
 2025-04-20 07:56:22 | DEBUG | ansari.app.main_whatsapp:main_webhook:112 | ! Before `background_tasks -> send_typing_indicator_then_start_loop`
 2025-04-20 07:56:22 | DEBUG | ansari.app.main_whatsapp:main_webhook:112 | ! After `background_tasks -> send_typing_indicator_then_start_loop`

diff --git a/.../structure_of_api_responses/meta_whatsapp_api_structure_of_a_request_sent_using_zrok.json b/.../structure_of_api_responses/meta_whatsapp_api_structure_of_a_request_sent_using_zrok.json
@@ -11,8 +11,8 @@
         "scheme": "https",
         "method": "POST",
         "root_path": "",
-        "path": "/whatsapp/v1",
-        "raw_path": "/whatsapp/v1",
+        "path": "/whatsapp/v2",
+        "raw_path": "/whatsapp/v2",
         "query_string": "",
         "headers": [
             ["host", "YOUR_ZROK_SHARE_TOKEN.share.zrok.io"],
@@ -41,7 +41,7 @@
         "endpoint": "<function main_webhook>",
         "path_params": {},
         "route": {
-            "path": "/whatsapp/v1",
+            "path": "/whatsapp/v2",
             "name": "main_webhook",
             "methods": ["POST"]
         }

diff --git a/src/ansari/app/main_api.py b/src/ansari/app/main_api.py
@@ -38,7 +38,7 @@
 from ansari.agents.ansari_workflow import AnsariWorkflow
 from ansari.ansari_db import AnsariDB, MessageLogger, SourceType
 from ansari.ansari_logger import get_logger
-from ansari.app.main_whatsapp import router as whatsapp_router
+from ansari.routers.whatsapp_router import router as whatsapp_router
 from ansari.config import Settings, get_settings
 from ansari.presenters.api_presenter import ApiPresenter
 from ansari.util.general_helpers import CORSMiddlewareWithLogging, get_extended_origins, register_to_mailing_list
@@ -88,7 +88,7 @@ async def lifespan(app: FastAPI):
 
 app = FastAPI(lifespan=lifespan)
 
-# Include the WhatsApp router
+# Include the WhatsApp API router
 app.include_router(whatsapp_router)
 
 
@@ -151,6 +151,7 @@ def add_app_middleware():
 
 cache = FanoutCache(get_settings().diskcache_dir, shards=4, timeout=1)
 
+
 if __name__ == "__main__" and get_settings().DEV_MODE:
     # Programatically start a Uvicorn server while debugging (development) for easier control/accessibility
     #   I.e., just run:
@@ -178,6 +179,10 @@ def add_app_middleware():
         log_level="debug",
     )
 
+@app.get("/")
+async def root():
+    """Root endpoint for health checks."""
+    return {"status": "ok", "message": "Ansari Backend service is running"}
 
 class RegisterRequest(BaseModel):
     email: EmailStr