From 4b30d81b270d9cfd4b5a94bceffa3ff0159580fd Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 01:43:18 +0100 Subject: [PATCH 01/23] Fix Firestore client initialization with explicit project ID --- backend/database/_client.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/database/_client.py b/backend/database/_client.py index a99ee93813..c473b290fe 100644 --- a/backend/database/_client.py +++ b/backend/database/_client.py @@ -11,7 +11,21 @@ with open('google-credentials.json', 'w') as f: json.dump(service_account_info, f) -db = firestore.Client() +# Read the project ID from google-credentials.json +project_id = None +try: + with open('google-credentials.json', 'r') as f: + credentials = json.load(f) + # Try to get project_id, if not available use quota_project_id + project_id = credentials.get('project_id') or credentials.get('quota_project_id') +except Exception as e: + print(f"Error reading google-credentials.json: {e}") + +if not project_id: + raise EnvironmentError("Project ID could not be determined from google-credentials.json") + +# Initialize Firestore with explicit project ID +db = firestore.Client(project=project_id) def get_users_uid(): From 763c7bc1749dadf757a3156e3381213374681aad Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 01:45:55 +0100 Subject: [PATCH 02/23] Pinecone vector database initialization with configuration handling --- backend/database/vector_db.py | 41 ++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/backend/database/vector_db.py b/backend/database/vector_db.py index 2a26e300ec..60535f5926 100644 --- a/backend/database/vector_db.py +++ b/backend/database/vector_db.py @@ -9,10 +9,45 @@ from models.memory import Memory from utils.llm import embeddings -if os.getenv('PINECONE_API_KEY') is not None: - pc = Pinecone(api_key=os.getenv('PINECONE_API_KEY', '')) - index = pc.Index(os.getenv('PINECONE_INDEX_NAME', '')) +# Check if Pinecone is properly configured +pinecone_api_key = os.getenv('PINECONE_API_KEY') +pinecone_index_name = os.getenv('PINECONE_INDEX_NAME') + +if pinecone_api_key and pinecone_index_name: + # Both API key and index name are provided + pc = Pinecone(api_key=pinecone_api_key) + index = pc.Index(pinecone_index_name) + print(f"Connected to Pinecone index: {pinecone_index_name}") +elif pinecone_api_key: + # API key is provided but index name is missing + print("WARNING: PINECONE_INDEX_NAME is not set in .env file. Using a mock index for development.") + # Create a mock index for development + class MockPineconeIndex: + def __init__(self): + self.vectors = {} + + def upsert(self, vectors, namespace=None): + for vector in vectors: + self.vectors[vector['id']] = { + 'values': vector['values'], + 'metadata': vector.get('metadata', {}) + } + return {'upserted_count': len(vectors)} + + def query(self, vector, namespace=None, top_k=10, filter=None, include_metadata=True): + # Return empty results for mock implementation + return {'matches': []} + + def delete(self, ids, namespace=None): + for id in ids: + if id in self.vectors: + del self.vectors[id] + return {'deleted_count': len(ids)} + + index = MockPineconeIndex() else: + # No Pinecone configuration + print("WARNING: Pinecone is not configured. Vector search functionality will be disabled.") index = None From cbe4a95913640f4f41c8e05598b43d02461a9e03 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 01:46:31 +0100 Subject: [PATCH 03/23] Add project ID handling for Google Cloud Storage client initialization --- backend/utils/other/storage.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/backend/utils/other/storage.py b/backend/utils/other/storage.py index dab97cf3ae..ff98c48722 100644 --- a/backend/utils/other/storage.py +++ b/backend/utils/other/storage.py @@ -9,12 +9,23 @@ from database.redis_db import cache_signed_url, get_cached_signed_url +# Get project ID from google-credentials.json +project_id = None +try: + with open('google-credentials.json', 'r') as f: + credentials = json.load(f) + # Try to get project_id, if not available use quota_project_id + project_id = credentials.get('project_id') or credentials.get('quota_project_id') + print(f"Using Google Cloud project ID for storage: {project_id}") +except Exception as e: + print(f"Error reading google-credentials.json: {e}") + if os.environ.get('SERVICE_ACCOUNT_JSON'): service_account_info = json.loads(os.environ["SERVICE_ACCOUNT_JSON"]) credentials = service_account.Credentials.from_service_account_info(service_account_info) - storage_client = storage.Client(credentials=credentials) + storage_client = storage.Client(credentials=credentials, project=project_id) else: - storage_client = storage.Client() + storage_client = storage.Client(project=project_id) speech_profiles_bucket = os.getenv('BUCKET_SPEECH_PROFILES') postprocessing_audio_bucket = os.getenv('BUCKET_POSTPROCESSING') @@ -280,4 +291,3 @@ def upload_multi_chat_files(files_name: List[str], uid: str): files.append(f'https://storage.googleapis.com/{chat_files_bucket}/{uid}/name') dictFiles[name] = f'https://storage.googleapis.com/{chat_files_bucket}/{uid}/{name}' return dictFiles - From 715e091b82b9ed66f1e2ceb9cc701f7d5159ba99 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 02:09:01 +0100 Subject: [PATCH 04/23] Enhance GitHub API token handling and documentation - Update README.md with detailed instructions for setting up GitHub token - Improve GitHub API request handling in firmware and plugins utils - Add warning messages for missing GitHub token - Handle rate limit scenarios - Provide clearer guidance on optional GitHub token configuration --- backend/README.md | 24 ++++++++++++++++++------ backend/routers/firmware.py | 15 ++++++++++++++- backend/utils/plugins.py | 12 +++++++++++- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/backend/README.md b/backend/README.md index d0d72d6e83..dbfd16ea2e 100644 --- a/backend/README.md +++ b/backend/README.md @@ -10,17 +10,30 @@ gcloud auth application-default login --project ``` Replace `` with your Google Cloud Project ID - This should generate the `application_default_credentials.json` file in the `~/.config/gcloud` directory. This file is read automatically by gcloud in Python, so you don’t have to manually add any env for the service account. + This should generate the `application_default_credentials.json` file in the `~/.config/gcloud` directory. This file is read automatically by gcloud in Python, so you don't have to manually add any env for the service account. 5. Install Python (use brew if on mac) (or with nix env it will be done for you) -6. Install `pip` (if it doesn’t exist) +6. Install `pip` (if it doesn't exist) 7. Install `git `and `ffmpeg` (use brew if on mac) (again nix env installs this for you) 8. Move to the backend directory (`cd backend`) 9. Run the command `cat .env.template > .env` 10. For Redis (you can use [upstash](https://upstash.com/), sign up and create a free instance) -11. Add the necessary keys in the env file (OpenAI, Deepgram, Redis, set ADMIN_KEY to 123) +11. Add the necessary API keys in the `.env` file: + - [OpenAI API Key](https://platform.openai.com/settings/organization/api-keys) + - [Deepgram API Key](https://console.deepgram.com/api-keys) + - Redis credentials from your [Upstash Console](https://console.upstash.com/) + - Set `ADMIN_KEY` to a temporary value (e.g., `123`) for local development + - **IMPORTANT:** For Pinecone vector database: + - Make sure to set `PINECONE_INDEX_NAME` to the name of your Pinecone index + - If you don't have a Pinecone index yet, [create one in the Pinecone Console](https://app.pinecone.io/) + - The index should be created with the appropriate dimension setting (e.g., 1536 for OpenAI embeddings) + - **Optional but recommended:** Set `GITHUB_TOKEN` to a [GitHub personal access token](https://github.com/settings/tokens) + - This is used to access GitHub's API for retrieving firmware updates and documentation + - Without this token, GitHub API requests will have lower rate limits + - You can create a token with `public_repo` scope only + - [Generate a new token here](https://github.com/settings/tokens/new?scopes=public_repo&description=Omi%20Backend%20Access) 12. Run the command `pip install -r requirements.txt` to install required dependencies 13. Sign Up on [ngrok](https://ngrok.com/) and follow the steps to configure it -14. During the onboarding flow, under the `Static Domain` section, Ngrok should provide you with a static domain and a command to point your localhost to that static domain. Replace the port from 80 to 8000 in that command and run it in your terminal +14. During the onboarding flow, under the `Static Domain` section, Ngrok should provide you with a static domain and a command to point your localhost to that static domain. Replace the port from 80 to 8000 in that command and run it in your terminal ``` ngrok http --domain=example.ngrok-free.app 8000 ``` @@ -34,6 +47,5 @@ ssl._create_default_https_context = ssl._create_unverified_context ``` 17. Now try running the `uvicorn main:app --reload --env-file .env` command again. -18. Assign the url given by ngrok in the app’s env to `API_BASE_URL` +18. Assign the url given by ngrok in the app's env to `API_BASE_URL` 19. Now your app should be using your local backend - diff --git a/backend/routers/firmware.py b/backend/routers/firmware.py index 7ae18e3bdb..a977e8a6fc 100644 --- a/backend/routers/firmware.py +++ b/backend/routers/firmware.py @@ -44,10 +44,23 @@ async def get_omi_github_releases(cache_key: str) -> Optional[list]: headers = { "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", - "Authorization": f"Bearer {os.getenv('GITHUB_TOKEN')}", } + + # Add GitHub token if available + github_token = os.getenv('GITHUB_TOKEN') + if github_token: + headers["Authorization"] = f"Bearer {github_token}" + else: + print("WARNING: GITHUB_TOKEN not set. Using unauthenticated GitHub API requests with lower rate limits.") + response = await client.get(url, headers=headers) if response.status_code != 200: + if response.status_code == 403 and not github_token: + # Rate limit exceeded without token + raise HTTPException( + status_code=403, + detail="GitHub API rate limit exceeded. Set GITHUB_TOKEN in .env file to increase rate limits." + ) raise HTTPException(status_code=response.status_code, detail="Failed to fetch latest release") releases = response.json() # Cache successful response for 30 minutes diff --git a/backend/utils/plugins.py b/backend/utils/plugins.py index 09fab6dbe9..68015613bd 100644 --- a/backend/utils/plugins.py +++ b/backend/utils/plugins.py @@ -39,13 +39,23 @@ def get_github_docs_content(repo="BasedHardware/omi", path="docs/docs"): if cached := get_generic_cache(f'get_github_docs_content_{repo}_{path}'): return cached docs_content = {} - headers = {"Authorization": f"token {os.getenv('GITHUB_TOKEN')}"} + + # Set up headers with GitHub token if available + headers = {} + github_token = os.getenv('GITHUB_TOKEN') + if github_token: + headers["Authorization"] = f"token {github_token}" + else: + print("WARNING: GITHUB_TOKEN not set. Using unauthenticated GitHub API requests with lower rate limits.") def get_contents(path): url = f"https://api.github.com/repos/{repo}/contents/{path}" response = requests.get(url, headers=headers) if response.status_code != 200: + if response.status_code == 403 and not github_token: + print(f"GitHub API rate limit exceeded. Set GITHUB_TOKEN in .env file to increase rate limits.") + return print(f"Failed to fetch contents for {path}: {response.status_code}") return From dd659bcd39e7e897bb4412d5002ad497c1a9b5a3 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 02:14:52 +0100 Subject: [PATCH 05/23] Improve Silero VAD model loading with error handling and fallback mechanism - Add SSL certificate verification bypass - Implement try-except block for model loading - Create mock VAD functions for graceful degradation - Handle potential model loading failures --- backend/utils/stt/vad.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/backend/utils/stt/vad.py b/backend/utils/stt/vad.py index 984fa43590..50ad8594b9 100644 --- a/backend/utils/stt/vad.py +++ b/backend/utils/stt/vad.py @@ -9,10 +9,44 @@ from database import redis_db +# Fix SSL certificate issues +import ssl +ssl._create_default_https_context = ssl._create_unverified_context + torch.set_num_threads(1) torch.hub.set_dir('pretrained_models') -model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad') -(get_speech_timestamps, save_audio, read_audio, VADIterator, collect_chunks) = utils + +# Try to load the model with error handling +try: + model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad', trust_repo=True) + (get_speech_timestamps, save_audio, read_audio, VADIterator, collect_chunks) = utils +except Exception as e: + print(f"Error loading Silero VAD model: {e}") + print("Using mock VAD model instead. Some functionality may be limited.") + + # Create mock functions for VAD + def get_speech_timestamps(audio_data, **kwargs): + # Return the entire audio as one speech segment + return [{'start': 0, 'end': len(audio_data)}] + + def save_audio(path, tensor, **kwargs): + pass + + def read_audio(path, **kwargs): + return torch.zeros(1000) + + def collect_chunks(chunks, **kwargs): + return torch.cat(chunks) if chunks else torch.zeros(1) + + class MockVADIterator: + def __init__(self, *args, **kwargs): + pass + + def __call__(self, x, return_seconds=False): + return 0.9 # Always return high speech probability + + VADIterator = MockVADIterator + model = None class SpeechState(str, Enum): From b8a5a3be54a6d395f08fc0ba0ba6d95e10989dc8 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 02:24:06 +0100 Subject: [PATCH 06/23] doc: Add Opus library troubleshooting - Include detailed steps for resolving Opus library installation issues - Add verification step to confirm successful library import --- backend/README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/backend/README.md b/backend/README.md index dbfd16ea2e..05e5acdc28 100644 --- a/backend/README.md +++ b/backend/README.md @@ -49,3 +49,39 @@ 17. Now try running the `uvicorn main:app --reload --env-file .env` command again. 18. Assign the url given by ngrok in the app's env to `API_BASE_URL` 19. Now your app should be using your local backend + +## Troubleshooting + +### Opus Library Issues + +If you encounter an error related to the opus library like `Exception: Could not find Opus library. Make sure it is installed.`, follow these steps: + +1. Make sure you have installed the opus library at the system level: + ```bash + # On macOS + brew install opus + + # On Ubuntu/Debian + sudo apt-get install libopus-dev + ``` + +2. Create a symbolic link to the opus library in your virtual environment: + ```bash + # On macOS + # First, find where the library is installed + find /usr -name "libopus.dylib" 2>/dev/null || find /opt -name "libopus.dylib" 2>/dev/null + + # Then create a symbolic link (replace the path with your actual path) + mkdir -p venv/lib + ln -sf /opt/homebrew/lib/libopus.dylib venv/lib/libopus.dylib + + # Then create a symbolic link (replace the path with your actual path) + mkdir -p venv/lib + ln -sf /usr/lib/x86_64-linux-gnu/libopus.so venv/lib/libopus.so + ``` + +3. Verify that the opuslib module can now be imported correctly: + ```bash + source venv/bin/activate + python -c "import opuslib; print('opuslib imported successfully')" + ``` From 5cdbd460da8cb5a026114578383f6e8a7ab75ba3 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 02:28:38 +0100 Subject: [PATCH 07/23] feat: Add Typesense mock client for development and configuration handling - Implement conditional Typesense client initialization - Create mock Typesense client for development when configuration is missing - Update README.md with Typesense configuration instructions - Add warning message for unconfigured Typesense setup --- backend/README.md | 23 +++++++++ backend/utils/memories/search.py | 83 ++++++++++++++++++++++++++++---- 2 files changed, 97 insertions(+), 9 deletions(-) diff --git a/backend/README.md b/backend/README.md index 05e5acdc28..1003b74b3f 100644 --- a/backend/README.md +++ b/backend/README.md @@ -26,6 +26,10 @@ - Make sure to set `PINECONE_INDEX_NAME` to the name of your Pinecone index - If you don't have a Pinecone index yet, [create one in the Pinecone Console](https://app.pinecone.io/) - The index should be created with the appropriate dimension setting (e.g., 1536 for OpenAI embeddings) + - **Optional:** For Typesense search functionality: + - Set `TYPESENSE_HOST`, `TYPESENSE_HOST_PORT`, and `TYPESENSE_API_KEY` if you want to use Typesense + - If not set, a mock client will be used for development + - You can [sign up for Typesense Cloud](https://cloud.typesense.org/) or [self-host it](https://typesense.org/docs/guide/install-typesense.html) - **Optional but recommended:** Set `GITHUB_TOKEN` to a [GitHub personal access token](https://github.com/settings/tokens) - This is used to access GitHub's API for retrieving firmware updates and documentation - Without this token, GitHub API requests will have lower rate limits @@ -85,3 +89,22 @@ If you encounter an error related to the opus library like `Exception: Could not source venv/bin/activate python -c "import opuslib; print('opuslib imported successfully')" ``` + +### GitHub API Rate Limits + +If you encounter GitHub API rate limit errors, make sure to set the `GITHUB_TOKEN` environment variable in your `.env` file. This will increase your rate limits for GitHub API requests. + +### Typesense Configuration + +If you want to use Typesense for search functionality: + +1. Sign up for [Typesense Cloud](https://cloud.typesense.org/) or [self-host Typesense](https://typesense.org/docs/guide/install-typesense.html) +2. Get your Typesense API key, host, and port +3. Update your `.env` file with the following values: + ``` + TYPESENSE_HOST=your-typesense-host + TYPESENSE_HOST_PORT=your-typesense-port + TYPESENSE_API_KEY=your-typesense-api-key + ``` + +If you don't configure Typesense, a mock client will be used for development, which will return empty search results. diff --git a/backend/utils/memories/search.py b/backend/utils/memories/search.py index 17f24bb97a..25323c7e8d 100644 --- a/backend/utils/memories/search.py +++ b/backend/utils/memories/search.py @@ -5,15 +5,80 @@ import typesense -client = typesense.Client({ - 'nodes': [{ - 'host': os.getenv('TYPESENSE_HOST'), - 'port': os.getenv('TYPESENSE_HOST_PORT'), - 'protocol': 'https' - }], - 'api_key': os.getenv('TYPESENSE_API_KEY'), - 'connection_timeout_seconds': 2 -}) +# Check if Typesense is configured +typesense_host = os.getenv('TYPESENSE_HOST') +typesense_port = os.getenv('TYPESENSE_HOST_PORT') +typesense_api_key = os.getenv('TYPESENSE_API_KEY') + +if typesense_host and typesense_port and typesense_api_key: + # Typesense is properly configured + client = typesense.Client({ + 'nodes': [{ + 'host': typesense_host, + 'port': typesense_port, + 'protocol': 'https' + }], + 'api_key': typesense_api_key, + 'connection_timeout_seconds': 2 + }) + print(f"Connected to Typesense at {typesense_host}:{typesense_port}") +else: + # Typesense is not configured, create a mock client + print("WARNING: Typesense is not configured. Using a mock client for development.") + + class MockTypesenseClient: + def __init__(self): + self.collections = {} + + def collections(self): + return MockCollections(self.collections) + + def collection(self, name): + if name not in self.collections: + self.collections[name] = {} + return MockCollection(self.collections, name) + + class MockCollections: + def __init__(self, collections): + self.collections = collections + + def create(self, schema): + name = schema.get('name') + if name not in self.collections: + self.collections[name] = {} + return {'name': name} + + def retrieve(self): + return [{'name': name} for name in self.collections.keys()] + + class MockCollection: + def __init__(self, collections, name): + self.collections = collections + self.name = name + + def documents(self): + return MockDocuments(self.collections, self.name) + + def search(self, search_parameters): + return { + 'found': 0, + 'hits': [], + 'page': 1, + 'request_params': search_parameters + } + + class MockDocuments: + def __init__(self, collections, collection_name): + self.collections = collections + self.collection_name = collection_name + + def create(self, document): + return {'id': document.get('id')} + + def delete(self, document_id): + return {'id': document_id} + + client = MockTypesenseClient() def search_memories( From dc0ee8c99fef3ffa23b0102fe6bf79fec3f90016 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 02:39:30 +0100 Subject: [PATCH 08/23] docs: Enhance Typesense configuration guide in README - Provide setup instructions for Typesense Cloud and self-hosted environments - Clarify host, port, and API key configuration steps --- backend/README.md | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/backend/README.md b/backend/README.md index 1003b74b3f..6afbfda5fc 100644 --- a/backend/README.md +++ b/backend/README.md @@ -99,12 +99,39 @@ If you encounter GitHub API rate limit errors, make sure to set the `GITHUB_TOKE If you want to use Typesense for search functionality: 1. Sign up for [Typesense Cloud](https://cloud.typesense.org/) or [self-host Typesense](https://typesense.org/docs/guide/install-typesense.html) -2. Get your Typesense API key, host, and port -3. Update your `.env` file with the following values: + +2. Get your Typesense API key, host, and port: + + **For Typesense Cloud:** + - After creating a cluster, go to the "API Keys" section + - Use the "Search Only API Key" or "Admin API Key" depending on your needs + - For the host, use the hostname shown in the "Cluster Overview" (e.g., `xxx.a1.typesense.net`) + - For the port, use `443` for HTTPS connections + + **For Self-hosted Typesense:** + - The API key is the one you specified when starting the Typesense server (with `--api-key`) + - For the host, use the server's hostname or IP address (e.g., `localhost` or `192.168.1.100`) + - For the port, use the port you configured when starting the server (default is `8108`) + +3. Update your `.env` file with these values: ``` TYPESENSE_HOST=your-typesense-host TYPESENSE_HOST_PORT=your-typesense-port TYPESENSE_API_KEY=your-typesense-api-key ``` + Example for Typesense Cloud: + ``` + TYPESENSE_HOST=xyz123.a1.typesense.net + TYPESENSE_HOST_PORT=443 + TYPESENSE_API_KEY=xyzABC123... + ``` + + Example for self-hosted Typesense: + ``` + TYPESENSE_HOST=localhost + TYPESENSE_HOST_PORT=8108 + TYPESENSE_API_KEY=xyz123... + ``` + If you don't configure Typesense, a mock client will be used for development, which will return empty search results. From 34650a8b036f552119cb8e9df47798bd2bd20989 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 02:52:12 +0100 Subject: [PATCH 09/23] feat: Improve Silero VAD model download and loading mechanism - Implement direct model file download from GitHub repository - Add fallback mechanism for Silero VAD model loading - Handle potential download and loading errors --- backend/README.md | 18 +++++++++++++ backend/utils/stt/vad.py | 57 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/backend/README.md b/backend/README.md index 6afbfda5fc..9c56d3fefc 100644 --- a/backend/README.md +++ b/backend/README.md @@ -135,3 +135,21 @@ If you want to use Typesense for search functionality: ``` If you don't configure Typesense, a mock client will be used for development, which will return empty search results. + +### Silero VAD Model Issues + +If you encounter an error like `HTTP Error 401: Unauthorized` when loading the Silero VAD model, it's likely due to GitHub's rate limiting or authentication issues. The application will automatically fall back to a mock implementation, but if you want to use the real model: + +1. The code has been updated to download the model files directly from GitHub and load them locally using onnxruntime. + +2. If you still encounter issues, you can manually download the model files: + ```bash + # Create the model directory + mkdir -p pretrained_models/silero_vad + + # Download the model files + curl -L https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx -o pretrained_models/silero_vad/model.onnx + curl -L https://github.com/snakers4/silero-vad/raw/master/utils_vad.py -o pretrained_models/silero_vad/utils.py + ``` + +3. If you need to use a different model or have specific requirements, you can modify the `utils/stt/vad.py` file to use your own implementation. diff --git a/backend/utils/stt/vad.py b/backend/utils/stt/vad.py index 50ad8594b9..9073eb2ee4 100644 --- a/backend/utils/stt/vad.py +++ b/backend/utils/stt/vad.py @@ -1,9 +1,13 @@ import os from enum import Enum +import json +import urllib.request +from pathlib import Path import numpy as np import requests import torch +import onnxruntime from fastapi import HTTPException from pydub import AudioSegment @@ -16,10 +20,59 @@ torch.set_num_threads(1) torch.hub.set_dir('pretrained_models') +# Define model directory and files +MODEL_DIR = Path('pretrained_models/silero_vad') +MODEL_FILE = MODEL_DIR / 'model.onnx' +UTILS_FILE = MODEL_DIR / 'utils.py' +EXAMPLE_FILE = MODEL_DIR / 'example.py' +CONFIG_FILE = MODEL_DIR / 'config.json' + +# Create model directory if it doesn't exist +MODEL_DIR.mkdir(parents=True, exist_ok=True) + # Try to load the model with error handling try: - model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad', trust_repo=True) - (get_speech_timestamps, save_audio, read_audio, VADIterator, collect_chunks) = utils + # Check if model files already exist + if not MODEL_FILE.exists() or not UTILS_FILE.exists() or not CONFIG_FILE.exists(): + print("Downloading Silero VAD model files...") + + # Download model file + urllib.request.urlretrieve( + "https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx", + MODEL_FILE + ) + + # Download utils file + urllib.request.urlretrieve( + "https://github.com/snakers4/silero-vad/raw/master/utils_vad.py", + UTILS_FILE + ) + + # Download example file for reference + urllib.request.urlretrieve( + "https://github.com/snakers4/silero-vad/raw/master/examples/vad_examples.py", + EXAMPLE_FILE + ) + + # Create a simple config file + with open(CONFIG_FILE, 'w') as f: + json.dump({ + "sampling_rate": 16000, + "window_size_samples": 1536 + }, f) + + print("Model files downloaded successfully") + + # Load the ONNX model directly using onnxruntime + model = onnxruntime.InferenceSession(str(MODEL_FILE)) + + # Import functions from our local utils.py + from pretrained_models.silero_vad.utils import ( + get_speech_timestamps, save_audio, read_audio, VADIterator, collect_chunks + ) + + print("Silero VAD model loaded successfully") + except Exception as e: print(f"Error loading Silero VAD model: {e}") print("Using mock VAD model instead. Some functionality may be limited.") From 3ab94bd0ace1aa2df42ca06eedab3fbf0ac20b92 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 03:08:49 +0100 Subject: [PATCH 10/23] fix: Update Silero VAD model download URL and simplify file retrieval - Update model file download URL to latest repository location - Remove redundant file downloads for utils and example files - Streamline model file retrieval process --- backend/README.md | 24 ++++++++++++++++++------ backend/requirements.txt | 1 + backend/utils/stt/vad.py | 20 +++++++------------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/backend/README.md b/backend/README.md index 9c56d3fefc..df79b3879a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -138,18 +138,30 @@ If you don't configure Typesense, a mock client will be used for development, wh ### Silero VAD Model Issues -If you encounter an error like `HTTP Error 401: Unauthorized` when loading the Silero VAD model, it's likely due to GitHub's rate limiting or authentication issues. The application will automatically fall back to a mock implementation, but if you want to use the real model: +If you encounter an error like `HTTP Error 401: Unauthorized` or `HTTP Error 404: Not Found` when loading the Silero VAD model, there are several solutions: -1. The code has been updated to download the model files directly from GitHub and load them locally using onnxruntime. +1. **The package is already included in requirements.txt:** + The `silero-vad>=5.1.0` package is already included in the project's requirements.txt file, so it should be installed when you run: + ```bash + pip install -r requirements.txt + ``` + If you need to install it separately or update it: + ```bash + # Activate your virtual environment (if not already activated) + source venv/bin/activate + + # Install or update the Silero VAD package + pip install silero-vad==5.1.2 + ``` -2. If you still encounter issues, you can manually download the model files: +2. **Manual download of model files:** + If you prefer to download the model files manually: ```bash # Create the model directory mkdir -p pretrained_models/silero_vad # Download the model files - curl -L https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx -o pretrained_models/silero_vad/model.onnx - curl -L https://github.com/snakers4/silero-vad/raw/master/utils_vad.py -o pretrained_models/silero_vad/utils.py + curl -L https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx -o pretrained_models/silero_vad/model.onnx ``` -3. If you need to use a different model or have specific requirements, you can modify the `utils/stt/vad.py` file to use your own implementation. +The application is configured to fall back to a mock implementation if the model fails to load, but this will limit voice activity detection functionality. diff --git a/backend/requirements.txt b/backend/requirements.txt index 630aabcb57..a0fc09bd05 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -202,6 +202,7 @@ semver==3.0.2 sentencepiece==0.2.0 shellingham==1.5.4 sigtools==4.0.1 +silero-vad>=5.1.0 six==1.16.0 smmap==5.0.1 sniffio==1.3.1 diff --git a/backend/utils/stt/vad.py b/backend/utils/stt/vad.py index 9073eb2ee4..d7b862a10e 100644 --- a/backend/utils/stt/vad.py +++ b/backend/utils/stt/vad.py @@ -36,23 +36,17 @@ if not MODEL_FILE.exists() or not UTILS_FILE.exists() or not CONFIG_FILE.exists(): print("Downloading Silero VAD model files...") - # Download model file + # Download model file - updated URL urllib.request.urlretrieve( - "https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx", + "https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx", MODEL_FILE ) - # Download utils file - urllib.request.urlretrieve( - "https://github.com/snakers4/silero-vad/raw/master/utils_vad.py", - UTILS_FILE - ) - - # Download example file for reference - urllib.request.urlretrieve( - "https://github.com/snakers4/silero-vad/raw/master/examples/vad_examples.py", - EXAMPLE_FILE - ) + # Create utils.py file - we'll use our local implementation + if not UTILS_FILE.exists(): + # We already have a utils.py file in the pretrained_models/silero_vad directory + # If not, we'll use the one we created + pass # Create a simple config file with open(CONFIG_FILE, 'w') as f: From ffd5e695271d883d01a3a217195f8498c5a3b1ee Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 03:32:08 +0100 Subject: [PATCH 11/23] feat: Add PyOgg error handling and fallback mechanism for Opus codec - handling of PyOgg import errors - Create mock OpusDecoder class for limited functionality - Update README with troubleshooting steps for PyOgg library --- backend/README.md | 45 ++++++++++++++++++++++++++++++++++++++++++ backend/utils/audio.py | 30 ++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/backend/README.md b/backend/README.md index df79b3879a..383e522ce0 100644 --- a/backend/README.md +++ b/backend/README.md @@ -165,3 +165,48 @@ If you encounter an error like `HTTP Error 401: Unauthorized` or `HTTP Error 404 ``` The application is configured to fall back to a mock implementation if the model fails to load, but this will limit voice activity detection functionality. + +### PyOgg Import Issues + +If you encounter an error like `NameError: name 'c_int_p' is not defined` when starting the server, it's due to an issue with the PyOgg library. The application has been updated to handle this error gracefully with a fallback mechanism, but Opus codec functionality will be limited. + +To fix the PyOgg library directly: + +1. Open the PyOgg opus.py file: + ```bash + # Find the file location + find venv -name opus.py + ``` + +2. Edit the file to add the missing POINTER import and replace c_int_p with POINTER(c_int): + ```python + # Add this import at the top with other ctypes imports + from ctypes import POINTER + + # Then replace all instances of c_int_p with POINTER(c_int) + ``` + +3. Alternatively, you can use the following Python script to fix it automatically: + ```python + file_path = 'venv/lib/python3.10/site-packages/pyogg/opus.py' # Adjust path as needed + + with open(file_path, 'r') as f: + content = f.read() + + # Add the missing import + if 'from ctypes import POINTER' not in content: + import re + content = re.sub( + r'from ctypes import.*?(?=\n)', + r'\g<0>, POINTER', + content, + count=1, + flags=re.DOTALL + ) + + # Replace c_int_p with POINTER(c_int) + content = content.replace('c_int_p', 'POINTER(c_int)') + + with open(file_path, 'w') as f: + f.write(content) + ``` diff --git a/backend/utils/audio.py b/backend/utils/audio.py index 558135b6f7..d4dbb0207f 100644 --- a/backend/utils/audio.py +++ b/backend/utils/audio.py @@ -1,7 +1,29 @@ import wave - +import logging from pydub import AudioSegment -from pyogg import OpusDecoder + +# Try to import PyOgg, and if it fails, create a mock implementation +try: + from pyogg import OpusDecoder + PYOGG_AVAILABLE = True +except (ImportError, NameError) as e: + logging.warning(f"PyOgg import failed: {e}. Opus codec will not be available.") + PYOGG_AVAILABLE = False + + # Mock OpusDecoder class + class OpusDecoder: + def __init__(self): + logging.warning("Using mock OpusDecoder. Opus codec functionality is limited.") + + def set_channels(self, channels): + pass + + def set_sampling_frequency(self, freq): + pass + + def decode(self, packet): + # Return empty bytes as we can't actually decode + return b'' def merge_wav_files(dest_file_path: str, source_files: [str], silent_seconds: [int]): @@ -23,6 +45,10 @@ def create_wav_from_bytes( ): # opus if codec == "opus": + if not PYOGG_AVAILABLE: + logging.error("Cannot process opus codec: PyOgg is not available") + raise Exception("Opus codec is not available due to PyOgg import issues") + # Create an Opus decoder opus_decoder = OpusDecoder() opus_decoder.set_channels(channels) From 0710acd0948249455335327c018061fed866097e Mon Sep 17 00:00:00 2001 From: Neo <54811660+Neotastisch@users.noreply.github.com> Date: Wed, 12 Mar 2025 13:08:27 +1000 Subject: [PATCH 12/23] Fixed App Store Icon in ReadMe --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 01c5bdad03..53350b8d99 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,12 @@ transcriptions of meetings, chats, and voice memos wherever you are.

-[Homepage](https://omi.me/) | [Documentation](https://docs.omi.me/) | [Buy Assembled Device](https://www.omi.me/products/omi-dev-kit-2) +[Homepage](https://omi.me/) | [Documentation](https://docs.omi.me/) | [Buy Consumer device](https://www.omi.me/cart/50230946562340:1) | [Buy Developer Kit](https://www.omi.me/products/omi-dev-kit-2)

[Get it on Google Play](https://play.google.com/store/apps/details?id=com.friend.ios) -[Download on the App Store](https://apps.apple.com/us/app/friend-ai-wearable/id6502156163) +[Download on the App Store](https://apps.apple.com/us/app/friend-ai-wearable/id6502156163) From e368cc6fab93a572861014bcce20fa6701f093c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Wed, 12 Mar 2025 11:54:02 +0700 Subject: [PATCH 13/23] Removing the VAD on external trigger for audio-bytes; pushing the pusher service to the max capacity --- backend/routers/transcribe.py | 39 ++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/backend/routers/transcribe.py b/backend/routers/transcribe.py index d880951b58..a46b70995e 100644 --- a/backend/routers/transcribe.py +++ b/backend/routers/transcribe.py @@ -583,27 +583,28 @@ async def receive_audio(dg_socket1, dg_socket2, soniox_socket, speechmatics_sock data = decoder.decode(bytes(data), frame_size=160) # audio_data.extend(data) + # STT + has_speech = True if include_speech_profile and codec != 'opus': # don't do for opus 1.0.4 for now has_speech = _has_speech(data, sample_rate) - if not has_speech: - continue - - if soniox_socket is not None: - await soniox_socket.send(data) - - if speechmatics_socket1 is not None: - await speechmatics_socket1.send(data) - - if dg_socket1 is not None: - elapsed_seconds = time.time() - timer_start - if elapsed_seconds > speech_profile_duration or not dg_socket2: - dg_socket1.send(data) - if dg_socket2: - print('Killing socket2', uid) - dg_socket2.finish() - dg_socket2 = None - else: - dg_socket2.send(data) + + if has_speech: + if soniox_socket is not None: + await soniox_socket.send(data) + + if speechmatics_socket1 is not None: + await speechmatics_socket1.send(data) + + if dg_socket1 is not None: + elapsed_seconds = time.time() - timer_start + if elapsed_seconds > speech_profile_duration or not dg_socket2: + dg_socket1.send(data) + if dg_socket2: + print('Killing socket2', uid) + dg_socket2.finish() + dg_socket2 = None + else: + dg_socket2.send(data) # Send to external trigger if audio_bytes_send is not None: From 8af4897bc230412e35323a62150d81841c260201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Wed, 12 Mar 2025 14:41:59 +0700 Subject: [PATCH 14/23] x2 connections for the external integration --- backend/routers/transcribe.py | 67 ++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/backend/routers/transcribe.py b/backend/routers/transcribe.py index a46b70995e..940d89e4c9 100644 --- a/backend/routers/transcribe.py +++ b/backend/routers/transcribe.py @@ -378,11 +378,9 @@ async def deepgram_socket_send(data): def create_pusher_task_handler(): nonlocal websocket_active - pusher_ws = None pusher_connect_lock = asyncio.Lock() - pusher_connected = False - - # Transcript + pusher_transcript_connected = False + pusher_audio_connected = False transcript_ws = None segment_buffers = [] in_progress_memory_id = None @@ -398,7 +396,7 @@ async def transcript_consume(): nonlocal segment_buffers nonlocal in_progress_memory_id nonlocal transcript_ws - nonlocal pusher_connected + nonlocal pusher_transcript_connected while websocket_active or len(segment_buffers) > 0: await asyncio.sleep(1) if transcript_ws and len(segment_buffers) > 0: @@ -412,8 +410,8 @@ async def transcript_consume(): except websockets.exceptions.ConnectionClosed as e: print(f"Pusher transcripts Connection closed: {e}", uid) transcript_ws = None - pusher_connected = False - await reconnect() + pusher_transcript_connected = False + await connect_transcript() except Exception as e: print(f"Pusher transcripts failed: {e}", uid) @@ -430,7 +428,7 @@ async def audio_bytes_consume(): nonlocal websocket_active nonlocal audio_buffers nonlocal audio_bytes_ws - nonlocal pusher_connected + nonlocal pusher_audio_connected while websocket_active or len(audio_buffers) > 0: await asyncio.sleep(1) if audio_bytes_ws and len(audio_buffers) > 0: @@ -444,37 +442,58 @@ async def audio_bytes_consume(): except websockets.exceptions.ConnectionClosed as e: print(f"Pusher audio_bytes Connection closed: {e}", uid) audio_bytes_ws = None - pusher_connected = False - await reconnect() + pusher_audio_connected = False + await connect_audio() except Exception as e: print(f"Pusher audio_bytes failed: {e}", uid) - async def reconnect(): - nonlocal pusher_connected + async def connect(): + await connect_transcript() + await connect_audio() + + async def connect_transcript(): + nonlocal pusher_transcript_connected nonlocal pusher_connect_lock async with pusher_connect_lock: - if pusher_connected: + if pusher_transcript_connected: return - await connect() + await _connect_transcript() - async def connect(): - nonlocal pusher_ws + async def connect_audio(): + nonlocal pusher_audio_connected + nonlocal pusher_connect_lock + async with pusher_connect_lock: + if pusher_audio_connected: + return + await _connect_audio() + + async def _connect_transcript(): nonlocal transcript_ws + nonlocal pusher_transcript_connected + try: + transcript_ws = await connect_to_trigger_pusher(uid, sample_rate) + pusher_transcript_connected = True + except Exception as e: + print(f"Exception in connect transcript pusher: {e}") + + async def _connect_audio(): nonlocal audio_bytes_ws nonlocal audio_bytes_enabled - nonlocal pusher_connected + nonlocal pusher_audio_connected + + if not audio_bytes_enabled: + return try: - pusher_ws = await connect_to_trigger_pusher(uid, sample_rate) - pusher_connected = True - transcript_ws = pusher_ws - if audio_bytes_enabled: - audio_bytes_ws = pusher_ws + audio_bytes_ws = await connect_to_trigger_pusher(uid, sample_rate) + pusher_audio_connected = True except Exception as e: - print(f"Exception in connect: {e}") + print(f"Exception in connect audio pusher: {e}") async def close(code: int = 1000): - await pusher_ws.close(code) + await transcript_ws.close(code) + if audio_bytes_ws: + await audio_bytes_ws.close(code) return (connect, close, transcript_send, transcript_consume, From 88215d1fac30dad136f50914dfa409901f0ebc9e Mon Sep 17 00:00:00 2001 From: smian1 Date: Wed, 12 Mar 2025 02:25:30 -0700 Subject: [PATCH 15/23] refactor: Restructure app list page layout and category rendering --- frontend/src/app/apps/components/app-list.tsx | 100 +++++++++--------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/frontend/src/app/apps/components/app-list.tsx b/frontend/src/app/apps/components/app-list.tsx index 87ee18e5af..ecb8988160 100644 --- a/frontend/src/app/apps/components/app-list.tsx +++ b/frontend/src/app/apps/components/app-list.tsx @@ -116,24 +116,6 @@ export default function AppList({ initialPlugins, initialStats }: AppListProps)
- {/* New This Week Section */} -
-
-

- New This Week -

-
-
- {newThisWeek.map((plugin) => ( - - ))} -
-
- {/* Most Popular Section */}
@@ -153,6 +135,38 @@ export default function AppList({ initialPlugins, initialStats }: AppListProps)
+ {/* Productivity Section - Moved to top */} + {sortedCategories['productivity-and-organization'] && ( +
+
+ + {sortedCategories['productivity-and-organization'].length > 4 && ( + + See all + + + )} +
+
+ {sortedCategories['productivity-and-organization'] + ?.slice(0, 4) + .map((plugin) => ( + s.id === plugin.id)} + /> + ))} +
+
+ )} + {/* Integration Apps Section */} {integrationApps.length > 0 && (
@@ -183,36 +197,23 @@ export default function AppList({ initialPlugins, initialStats }: AppListProps)
)} - {/* Category Sections */} - {Object.entries(sortedCategories).map(([category, plugins]) => ( -
-
- - {plugins.length > - (category === 'productivity-and-organization' ? 4 : 9) && ( - - See all - - - )} -
- - {category === 'productivity-and-organization' ? ( - // Productivity section with featured tiles -
- {plugins.slice(0, 4).map((plugin) => ( - s.id === plugin.id)} - /> - ))} + {/* Category Sections - Excluding productivity */} + {Object.entries(sortedCategories) + .filter(([category]) => category !== 'productivity-and-organization') + .map(([category, plugins]) => ( +
+
+ + {plugins.length > 9 && ( + + See all + + + )}
- ) : ( - // Other categories with compact tiles
{plugins.slice(0, 9).map((plugin, index) => ( ))}
- )} -
- ))} +
+ ))}
From 1082646eb279d01363453e31ab304653cad3726c Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 14:13:07 +0100 Subject: [PATCH 16/23] feat: Improve Google Cloud credentials handling in backend --- backend/README.md | 15 ++++++++++++++- backend/database/_client.py | 35 ++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/backend/README.md b/backend/README.md index 5cfef157ee..d64f398451 100644 --- a/backend/README.md +++ b/backend/README.md @@ -11,6 +11,7 @@ This README provides a quick setup guide for the Omi backend. For a comprehensiv 2. You will need to have your own Google Cloud Project (please refer to the [App Docs]([url](https://docs.omi.me/docs/developer/AppSetup#7-setup-firebase)) on how to setup Firebase). If you did setup Firebase for the App, then you'll already have a Project in Google Cloud. Make sure you have the `Cloud Resource Manager` and `Firebase Management API` permissions at the minimum in the [Google Cloud API Console](https://console.cloud.google.com/apis/dashboard) + 3. Run the following commands one by one ``` gcloud auth login @@ -18,7 +19,19 @@ This README provides a quick setup guide for the Omi backend. For a comprehensiv gcloud auth application-default login --project ``` Replace `` with your Google Cloud Project ID - This should generate the `application_default_credentials.json` file in the `~/.config/gcloud` directory. This file is read automatically by gcloud in Python, so you don’t have to manually add any env for the service account. + + This should generate the `application_default_credentials.json` file in the `~/.config/gcloud` directory. + +4. **Important**: In your `.env` file, set the `GOOGLE_APPLICATION_CREDENTIALS` to the absolute path of your credentials file: + ``` + # For macOS/Linux users (replace 'username' with your actual username) + GOOGLE_APPLICATION_CREDENTIALS=/Users/username/.config/gcloud/application_default_credentials.json + + # For Windows users (replace 'Username' with your actual username) + GOOGLE_APPLICATION_CREDENTIALS=C:\Users\Username\.config\gcloud\application_default_credentials.json + ``` + Do not use the tilde (~) in the path as it may not be properly expanded. + 5. Install Python (use brew if on mac) (or with nix env it will be done for you) 6. Install `pip` (if it doesn’t exist) 7. Install `git `and `ffmpeg` (use brew if on mac) (again nix env installs this for you) diff --git a/backend/database/_client.py b/backend/database/_client.py index c473b290fe..c318d74090 100644 --- a/backend/database/_client.py +++ b/backend/database/_client.py @@ -2,6 +2,7 @@ import json import os import uuid +import pathlib from google.cloud import firestore @@ -11,18 +12,38 @@ with open('google-credentials.json', 'w') as f: json.dump(service_account_info, f) -# Read the project ID from google-credentials.json +# Check if GOOGLE_APPLICATION_CREDENTIALS is set and expand the path if needed +credentials_path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') +if credentials_path and '~' in credentials_path: + # Expand the tilde to the user's home directory + credentials_path = os.path.expanduser(credentials_path) + os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials_path + print(f"Expanded credentials path to: {credentials_path}") + +# Read the project ID from credentials file project_id = None try: - with open('google-credentials.json', 'r') as f: - credentials = json.load(f) - # Try to get project_id, if not available use quota_project_id - project_id = credentials.get('project_id') or credentials.get('quota_project_id') + # First try to read from google-credentials.json if it exists + if os.path.exists('google-credentials.json'): + with open('google-credentials.json', 'r') as f: + credentials = json.load(f) + project_id = credentials.get('project_id') or credentials.get('quota_project_id') + + # If not found, try to read from the application default credentials + if not project_id and credentials_path and os.path.exists(credentials_path): + with open(credentials_path, 'r') as f: + credentials = json.load(f) + project_id = credentials.get('project_id') or credentials.get('quota_project_id') + + if project_id: + print(f"Using project ID: {project_id}") + else: + print("Project ID not found in credentials files") except Exception as e: - print(f"Error reading google-credentials.json: {e}") + print(f"Error reading credentials file: {e}") if not project_id: - raise EnvironmentError("Project ID could not be determined from google-credentials.json") + raise EnvironmentError("Project ID could not be determined from credentials files. Please ensure your Google Cloud credentials are properly set up.") # Initialize Firestore with explicit project ID db = firestore.Client(project=project_id) From 2d7942ce6d887d8ade5c54e5aef0ebdb688af2e4 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 18:59:37 +0100 Subject: [PATCH 17/23] add docker compose --- backend/Dockerfile | 4 +-- backend/README.md | 63 ++++++++++++++++++++++++++++++++++++-- backend/docker-compose.yml | 13 ++++++++ 3 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 backend/docker-compose.yml diff --git a/backend/Dockerfile b/backend/Dockerfile index 0f39775cee..b6d7a3c58d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.11 AS builder ENV PATH="/opt/venv/bin:$PATH" RUN python -m venv /opt/venv -COPY backend/requirements.txt /tmp/requirements.txt +COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt FROM python:3.11-slim @@ -14,7 +14,7 @@ ENV PATH="/opt/venv/bin:$PATH" RUN apt-get update && apt-get -y install ffmpeg curl unzip && rm -rf /var/lib/apt/lists/* COPY --from=builder /opt/venv /opt/venv -COPY backend/ . +COPY . . EXPOSE 8080 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/backend/README.md b/backend/README.md index d64f398451..62dee482b2 100644 --- a/backend/README.md +++ b/backend/README.md @@ -33,7 +33,7 @@ This README provides a quick setup guide for the Omi backend. For a comprehensiv Do not use the tilde (~) in the path as it may not be properly expanded. 5. Install Python (use brew if on mac) (or with nix env it will be done for you) -6. Install `pip` (if it doesn’t exist) +6. Install `pip` (if it doesn't exist) 7. Install `git `and `ffmpeg` (use brew if on mac) (again nix env installs this for you) 8. Move to the backend directory (`cd backend`) 9. Run the command `cat .env.template > .env` @@ -55,7 +55,7 @@ This README provides a quick setup guide for the Omi backend. For a comprehensiv ssl._create_default_https_context = ssl._create_unverified_context ``` 17. Now try running the `uvicorn main:app --reload --env-file .env` command again. -18. Assign the url given by ngrok in the app’s env to `API_BASE_URL` +18. Assign the url given by ngrok in the app's env to `API_BASE_URL` 19. Now your app should be using your local backend 20. If you used a virtual environment, when you're done, deactivate it by running: @@ -63,6 +63,65 @@ This README provides a quick setup guide for the Omi backend. For a comprehensiv deactivate ``` +## Docker Setup + +If you prefer to run the backend using Docker, follow these steps: + +1. Make sure you have Docker installed on your system: + - Mac: Install Docker Desktop + - Windows: Install Docker Desktop + - Linux: Follow the [official Docker installation guide](https://docs.docker.com/engine/install/) + +2. Set up your Google Cloud credentials as described in steps 1-3 above. + +3. Create a `.env` file in the backend directory by copying the template: + ```bash + cat .env.template > .env + ``` + +4. Update the `.env` file with your API keys and credentials. For the Google credentials, set: + ``` + GOOGLE_APPLICATION_CREDENTIALS=google-credentials.json + ``` + +5. Copy your Google Cloud application default credentials file to the backend directory: + ```bash + + cp ~/.config/gcloud/application_default_credentials.json ./google-credentials.json/ + ``` + +6. Make sure your Google Cloud application default credentials file exists: + ```bash +cp ~/.config/gcloud/application_default_credentials.json ./google-credentials.json + ``` + +7. Build the Docker image: + ```bash + docker build -t omi-backend . + ``` + +8. Run the container using Docker Compose: + ```bash + docker compose up -d + ``` + +9. To view logs: + ```bash + docker compose logs -f + ``` + +10. To stop the container: + ```bash + docker compose down + ``` + +11. Set up ngrok as described in steps 13-14 above, but point it to port 8080 instead: + ```bash + ngrok http --domain=example.ngrok-free.app 8080 + ``` + +12. Assign the URL given by ngrok in the app's env to `API_BASE_URL` + ## Additional Resources - [Full Backend Setup Documentation](https://docs.omi.me/developer/backend/Backend_Setup) diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml new file mode 100644 index 0000000000..2bac3b143d --- /dev/null +++ b/backend/docker-compose.yml @@ -0,0 +1,13 @@ +services: + backend: + image: omi-backend + container_name: omi-backend + ports: + - "8080:8080" + env_file: + - .env + environment: + - GOOGLE_APPLICATION_CREDENTIALS=/app/credentials/application_default_credentials.json + volumes: + - ~/.config/gcloud/application_default_credentials.json:/app/credentials/application_default_credentials.json + restart: unless-stopped \ No newline at end of file From da4dbbb0295cb6f035d224f60411c57afde39c96 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 19:46:28 +0100 Subject: [PATCH 18/23] chore: Update backend Docker configuration and documentation - Removed Google Cloud credentials configuration from docker-compose.yml - Updated README and Backend_Setup.mdx to reflect changes in credentials handling and installation instructions --- backend/README.md | 5 ++--- backend/docker-compose.yml | 4 ---- docs/docs/developer/backend/Backend_Setup.mdx | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/backend/README.md b/backend/README.md index 62dee482b2..f61b89fc74 100644 --- a/backend/README.md +++ b/backend/README.md @@ -86,13 +86,12 @@ If you prefer to run the backend using Docker, follow these steps: 5. Copy your Google Cloud application default credentials file to the backend directory: ```bash - - cp ~/.config/gcloud/application_default_credentials.json ./google-credentials.json/ + cp ~/.config/gcloud/application_default_credentials.json ./google-credentials.json ``` 6. Make sure your Google Cloud application default credentials file exists: ```bash -cp ~/.config/gcloud/application_default_credentials.json ./google-credentials.json + cp ~/.config/gcloud/application_default_credentials.json ./google-credentials.json ``` 7. Build the Docker image: diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 2bac3b143d..3210eed090 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -6,8 +6,4 @@ services: - "8080:8080" env_file: - .env - environment: - - GOOGLE_APPLICATION_CREDENTIALS=/app/credentials/application_default_credentials.json - volumes: - - ~/.config/gcloud/application_default_credentials.json:/app/credentials/application_default_credentials.json restart: unless-stopped \ No newline at end of file diff --git a/docs/docs/developer/backend/Backend_Setup.mdx b/docs/docs/developer/backend/Backend_Setup.mdx index d77e4af58f..ef38c0957a 100644 --- a/docs/docs/developer/backend/Backend_Setup.mdx +++ b/docs/docs/developer/backend/Backend_Setup.mdx @@ -82,7 +82,7 @@ Before you start, make sure you have the following: - **Install PyOgg:** - **All Platforms:** `pip install PyOgg` - **Install All Required Dependencies:** - - **All Platforms:** `brew install -r requirements.txt` + - **All Platforms:** `pip install -r requirements.txt` 2. **Clone the Backend Repository: 📂** - Open your terminal and navigate to your desired directory From 3eb8df862e3692c198879b73b553a361e3575c1f Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Wed, 12 Mar 2025 19:56:22 +0100 Subject: [PATCH 19/23] Update Docker configuration to use port 8000 Changed exposed port from 8080 to 8000 in Dockerfile and docker-compose.yml --- backend/Dockerfile | 4 ++-- backend/README.md | 4 ++-- backend/docker-compose.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index b6d7a3c58d..cca7a16c6d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -16,5 +16,5 @@ RUN apt-get update && apt-get -y install ffmpeg curl unzip && rm -rf /var/lib/ap COPY --from=builder /opt/venv /opt/venv COPY . . -EXPOSE 8080 -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] +EXPOSE 8000 +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/README.md b/backend/README.md index f61b89fc74..0b4776e7d9 100644 --- a/backend/README.md +++ b/backend/README.md @@ -114,9 +114,9 @@ If you prefer to run the backend using Docker, follow these steps: docker compose down ``` -11. Set up ngrok as described in steps 13-14 above, but point it to port 8080 instead: +11. Set up ngrok as described in steps 13-14 above, but point it to port 8000: ```bash - ngrok http --domain=example.ngrok-free.app 8080 + ngrok http --domain=example.ngrok-free.app 8000 ``` 12. Assign the URL given by ngrok in the app's env to `API_BASE_URL` diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 3210eed090..6ab3b974b4 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -3,7 +3,7 @@ services: image: omi-backend container_name: omi-backend ports: - - "8080:8080" + - "8000:8000" env_file: - .env restart: unless-stopped \ No newline at end of file From 77d7ad30b3332214deb87d8312266edd986d5253 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Sat, 15 Mar 2025 00:16:00 +0100 Subject: [PATCH 20/23] doc: Add GOOGLE_CLOUD_PROJECT variable to .env.template and update README --- backend/.env.template | 1 + backend/README.md | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/backend/.env.template b/backend/.env.template index 15b57e3969..665b80456e 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -4,6 +4,7 @@ BUCKET_BACKUPS= BUCKET_PLUGINS_LOGOS= GOOGLE_APPLICATION_CREDENTIALS=google-credentials.json +GOOGLE_CLOUD_PROJECT= PINECONE_API_KEY= PINECONE_INDEX_NAME= diff --git a/backend/README.md b/backend/README.md index 0b4776e7d9..10a3a63d27 100644 --- a/backend/README.md +++ b/backend/README.md @@ -32,6 +32,12 @@ This README provides a quick setup guide for the Omi backend. For a comprehensiv ``` Do not use the tilde (~) in the path as it may not be properly expanded. + Also, make sure to set your `GOOGLE_CLOUD_PROJECT` environment variable to your Google Cloud Project ID: + ``` + GOOGLE_CLOUD_PROJECT=your-project-id + ``` + This is required for Firebase authentication to work properly. + 5. Install Python (use brew if on mac) (or with nix env it will be done for you) 6. Install `pip` (if it doesn't exist) 7. Install `git `and `ffmpeg` (use brew if on mac) (again nix env installs this for you) From 1d2ac3f4ce2466ff8b4dbab771cab687316e8006 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Sun, 16 Mar 2025 22:48:42 +0100 Subject: [PATCH 21/23] gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0cf8f5864b..0a186e13bb 100644 --- a/.gitignore +++ b/.gitignore @@ -141,8 +141,7 @@ backend/scripts/rag/visualizations/ */pretrained_models/ /backend/_speech_profiles -backend/google-credentials.json -backend/google-credentials-dev.json +backend/google-credentials*.json startup-script.sh /app/lib/firebase_options_dev.dart From 7e9c238f5a619d8546005a5ca6866f9088902b86 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Thu, 10 Apr 2025 11:39:31 +0200 Subject: [PATCH 22/23] clenap optional variables and wrap try catch to make it work without keys --- backend/.env.template | 54 +++-- backend/README.md | 90 +++++++++ backend/database/vector_db.py | 274 ++++++++++++++------------ backend/utils/conversations/search.py | 223 ++++++++++++++------- backend/utils/other/hume.py | 40 +++- backend/utils/other/storage.py | 259 ++++++++++++++++++------ backend/utils/stt/streaming.py | 84 +++++++- 7 files changed, 735 insertions(+), 289 deletions(-) diff --git a/backend/.env.template b/backend/.env.template index 665b80456e..65708e8a6d 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -1,38 +1,66 @@ -HUGGINGFACE_TOKEN= -BUCKET_SPEECH_PROFILES= -BUCKET_BACKUPS= -BUCKET_PLUGINS_LOGOS= +################################################# +# REQUIRED ENVIRONMENT VARIABLES # +################################################# +# These variables are essential for core functionality +# Firebase/Google Cloud Authentication GOOGLE_APPLICATION_CREDENTIALS=google-credentials.json GOOGLE_CLOUD_PROJECT= -PINECONE_API_KEY= -PINECONE_INDEX_NAME= - +# Database Connection REDIS_DB_HOST= REDIS_DB_PORT= REDIS_DB_PASSWORD= -SONIOX_API_KEY= +# API Keys for Core Services +OPENAI_API_KEY= DEEPGRAM_API_KEY= - ADMIN_KEY= -OPENAI_API_KEY= -GITHUB_TOKEN= +# Base API URL for client connections +BASE_API_URL= + +################################################# +# OPTIONAL ENVIRONMENT VARIABLES # +################################################# +# These variables enable additional features but are not required for core functionality +# Additional API Keys +HUGGINGFACE_TOKEN= +SONIOX_API_KEY= +GITHUB_TOKEN= WORKFLOW_API_KEY= HUME_API_KEY= HUME_CALLBACK_URL= -HOSTED_PUSHER_API_URL= +# Google Cloud Storage Buckets +BUCKET_SPEECH_PROFILES= +BUCKET_BACKUPS= +BUCKET_PLUGINS_LOGOS= +# Vector Database (for advanced retrieval) +PINECONE_API_KEY= +PINECONE_INDEX_NAME= + +# Search and Indexing TYPESENSE_HOST= TYPESENSE_HOST_PORT= TYPESENSE_API_KEY= +# Webhooks and Notifications +HOSTED_PUSHER_API_URL= + +# Payment Processing STRIPE_API_KEY= STRIPE_WEBHOOK_SECRET= STRIPE_CONNECT_WEBHOOK_SECRET= -BASE_API_URL= \ No newline at end of file +# HTTP Timeouts (in seconds) +HTTP_GET_TIMEOUT=30 +HTTP_PUT_TIMEOUT=30 +HTTP_PATCH_TIMEOUT=30 +HTTP_DELETE_TIMEOUT=30 + +# Developer Options +LOCAL_DEVELOPMENT=false +NO_SOCKET_TIMEOUT=false \ No newline at end of file diff --git a/backend/README.md b/backend/README.md index 10a3a63d27..12a48fc8fc 100644 --- a/backend/README.md +++ b/backend/README.md @@ -288,3 +288,93 @@ To fix the PyOgg library directly: with open(file_path, 'w') as f: f.write(content) ``` + +## Environment Variables Structure + +The backend environment variables are now organized into **required** and **optional** categories to simplify setup: + +### Required Environment Variables + +These variables are essential for core functionality: + +1. **Firebase/Google Cloud Authentication** + ``` + GOOGLE_APPLICATION_CREDENTIALS=google-credentials.json + GOOGLE_CLOUD_PROJECT=your-project-id + ``` + These are required for user authentication and database access. + +2. **Database Connection (Redis)** + ``` + REDIS_DB_HOST=your-redis-host + REDIS_DB_PORT=your-redis-port + REDIS_DB_PASSWORD=your-redis-password + ``` + Redis is used for caching and temporary data storage. + +3. **API Keys for Core Services** + ``` + OPENAI_API_KEY=your-openai-api-key + DEEPGRAM_API_KEY=your-deepgram-api-key + ADMIN_KEY=your-admin-key + ``` + These keys enable core functionality like chat, transcription, and admin access. + +4. **Base API URL** + ``` + BASE_API_URL=your-api-base-url + ``` + This is the URL where your backend can be accessed, used for callbacks and client connections. + +### Optional Environment Variables + +These variables enable additional features but are not required for core functionality: + +1. **Vector Search and Embeddings (Pinecone)** + ``` + PINECONE_API_KEY=your-pinecone-api-key + PINECONE_INDEX_NAME=your-pinecone-index-name + ``` + Used for advanced memory retrieval and search features. If not provided, a local mock implementation will be used. + +2. **Full-Text Search (Typesense)** + ``` + TYPESENSE_HOST=your-typesense-host + TYPESENSE_HOST_PORT=your-typesense-port + TYPESENSE_API_KEY=your-typesense-api-key + ``` + Enables advanced conversation search features. If not configured, a mock implementation will return empty search results. + +3. **Google Cloud Storage** + ``` + BUCKET_SPEECH_PROFILES=your-speech-profiles-bucket-name + BUCKET_BACKUPS=your-backups-bucket-name + BUCKET_PLUGINS_LOGOS=your-plugins-logos-bucket-name + ``` + Used for storing various files. If not configured, local file storage will be used as a fallback. + +4. **Additional AI Services** + ``` + HUGGINGFACE_TOKEN=your-huggingface-token + SONIOX_API_KEY=your-soniox-api-key + HUME_API_KEY=your-hume-api-key + HUME_CALLBACK_URL=your-hume-callback-url + ``` + Provides additional AI capabilities. Mock implementations will be used if not configured. + +5. **Integration Services** + ``` + GITHUB_TOKEN=your-github-token + WORKFLOW_API_KEY=your-workflow-api-key + ``` + Used for external integrations. + +6. **Payment Processing** + ``` + STRIPE_API_KEY=your-stripe-api-key + STRIPE_WEBHOOK_SECRET=your-stripe-webhook-secret + STRIPE_CONNECT_WEBHOOK_SECRET=your-stripe-connect-webhook-secret + ``` + Required only if you need to process payments. + +The system is designed to work with just the required variables, with graceful fallbacks for optional services. This makes development and testing much easier, especially when you don't need all the optional services. diff --git a/backend/database/vector_db.py b/backend/database/vector_db.py index ea6c341ed3..1e7336c0d8 100644 --- a/backend/database/vector_db.py +++ b/backend/database/vector_db.py @@ -2,53 +2,60 @@ import os from collections import defaultdict from datetime import datetime, timezone, timedelta -from typing import List - -from pinecone import Pinecone - -from models.conversation import Conversation -from utils.llm import embeddings +from typing import List, Optional # Check if Pinecone is properly configured pinecone_api_key = os.getenv('PINECONE_API_KEY') pinecone_index_name = os.getenv('PINECONE_INDEX_NAME') +# Only import Pinecone if the API key is available +if pinecone_api_key: + from pinecone import Pinecone + +# Create a mock index for development or when Pinecone is not configured +class MockPineconeIndex: + def __init__(self): + self.vectors = {} + print("Using MockPineconeIndex - vector search will return empty results") + + def upsert(self, vectors, namespace=None): + for vector in vectors: + self.vectors[vector['id']] = { + 'values': vector['values'], + 'metadata': vector.get('metadata', {}) + } + return {'upserted_count': len(vectors)} + + def query(self, vector=None, namespace=None, top_k=10, filter=None, include_metadata=True, include_values=True): + # Return empty results for mock implementation + return {'matches': []} + + def delete(self, ids, namespace=None): + deleted_count = 0 + for id in ids: + if id in self.vectors: + del self.vectors[id] + deleted_count += 1 + return {'deleted_count': deleted_count} + + def update(self, id, set_metadata=None, namespace=None): + if id in self.vectors and set_metadata: + self.vectors[id]['metadata'].update(set_metadata) + return {'id': id} + +# Initialize the index based on configuration if pinecone_api_key and pinecone_index_name: # Both API key and index name are provided pc = Pinecone(api_key=pinecone_api_key) index = pc.Index(pinecone_index_name) print(f"Connected to Pinecone index: {pinecone_index_name}") -elif pinecone_api_key: - # API key is provided but index name is missing - print("WARNING: PINECONE_INDEX_NAME is not set in .env file. Using a mock index for development.") - # Create a mock index for development - class MockPineconeIndex: - def __init__(self): - self.vectors = {} - - def upsert(self, vectors, namespace=None): - for vector in vectors: - self.vectors[vector['id']] = { - 'values': vector['values'], - 'metadata': vector.get('metadata', {}) - } - return {'upserted_count': len(vectors)} - - def query(self, vector, namespace=None, top_k=10, filter=None, include_metadata=True): - # Return empty results for mock implementation - return {'matches': []} - - def delete(self, ids, namespace=None): - for id in ids: - if id in self.vectors: - del self.vectors[id] - return {'deleted_count': len(ids)} - - index = MockPineconeIndex() else: - # No Pinecone configuration - print("WARNING: Pinecone is not configured. Vector search functionality will be disabled.") - index = None + # Either API key or index name is missing or both + if pinecone_api_key: + print("WARNING: PINECONE_INDEX_NAME is not set in .env file. Using a mock index.") + else: + print("INFO: Pinecone is not configured. Vector search functionality will use mock implementation.") + index = MockPineconeIndex() def _get_data(uid: str, conversation_id: str, vector: List[float]): @@ -63,109 +70,130 @@ def _get_data(uid: str, conversation_id: str, vector: List[float]): } -def upsert_vector(uid: str, conversation: Conversation, vector: List[float]): - res = index.upsert(vectors=[_get_data(uid, conversation.id, vector)], namespace="ns1") - print('upsert_vector', res) +def upsert_vector(uid: str, conversation, vector: List[float]): + try: + res = index.upsert(vectors=[_get_data(uid, conversation.id, vector)], namespace="ns1") + print('upsert_vector', res) + except Exception as e: + print(f"Error in upsert_vector: {e}") -def upsert_vector2(uid: str, conversation: Conversation, vector: List[float], metadata: dict): - data = _get_data(uid, conversation.id, vector) - data['metadata'].update(metadata) - res = index.upsert(vectors=[data], namespace="ns1") - print('upsert_vector', res) +def upsert_vector2(uid: str, conversation, vector: List[float], metadata: dict): + try: + data = _get_data(uid, conversation.id, vector) + data['metadata'].update(metadata) + res = index.upsert(vectors=[data], namespace="ns1") + print('upsert_vector', res) + except Exception as e: + print(f"Error in upsert_vector2: {e}") def update_vector_metadata(uid: str, conversation_id: str, metadata: dict): - metadata['uid'] = uid - metadata['memory_id'] = conversation_id - return index.update(f'{uid}-{conversation_id}', set_metadata=metadata, namespace="ns1") - - -def upsert_vectors( - uid: str, vectors: List[List[float]], conversations: List[Conversation] -): - data = [ - _get_data(uid, conversation.id, vector) for conversation, vector in - zip(conversations, vectors) - ] - res = index.upsert(vectors=data, namespace="ns1") - print('upsert_vectors', res) + try: + metadata['uid'] = uid + metadata['memory_id'] = conversation_id + return index.update(f'{uid}-{conversation_id}', set_metadata=metadata, namespace="ns1") + except Exception as e: + print(f"Error in update_vector_metadata: {e}") + return {"error": str(e)} + + +def upsert_vectors(uid: str, vectors: List[List[float]], conversations: List): + try: + data = [ + _get_data(uid, conversation.id, vector) for conversation, vector in + zip(conversations, vectors) + ] + res = index.upsert(vectors=data, namespace="ns1") + print('upsert_vectors', res) + except Exception as e: + print(f"Error in upsert_vectors: {e}") def query_vectors(query: str, uid: str, starts_at: int = None, ends_at: int = None, k: int = 5) -> List[str]: - filter_data = {'uid': uid} - if starts_at is not None: - filter_data['created_at'] = {'$gte': starts_at, '$lte': ends_at} + try: + from utils.llm import embeddings - # print('filter_data', filter_data) - xq = embeddings.embed_query(query) - xc = index.query(vector=xq, top_k=k, include_metadata=False, filter=filter_data, namespace="ns1") - # print(xc) - return [item['id'].replace(f'{uid}-', '') for item in xc['matches']] + filter_data = {'uid': uid} + if starts_at is not None: + filter_data['created_at'] = {'$gte': starts_at, '$lte': ends_at} + + xq = embeddings.embed_query(query) + xc = index.query(vector=xq, top_k=k, include_metadata=False, filter=filter_data, namespace="ns1") + return [item['id'].replace(f'{uid}-', '') for item in xc['matches']] + except Exception as e: + print(f"Error in query_vectors: {e}") + return [] def query_vectors_by_metadata( uid: str, vector: List[float], dates_filter: List[datetime], people: List[str], topics: List[str], entities: List[str], dates: List[str], limit: int = 5, ): - filter_data = {'$and': [ - {'uid': {'$eq': uid}}, - ]} - if people or topics or entities or dates: - filter_data['$and'].append( - {'$or': [ - {'people': {'$in': people}}, - {'topics': {'$in': topics}}, - {'entities': {'$in': entities}}, - # {'dates': {'$in': dates_mentioned}}, - ]} - ) - if dates_filter and len(dates_filter) == 2 and dates_filter[0] and dates_filter[1]: - print('dates_filter', dates_filter) - filter_data['$and'].append( - {'created_at': {'$gte': int(dates_filter[0].timestamp()), '$lte': int(dates_filter[1].timestamp())}} - ) - - print('query_vectors_by_metadata:', json.dumps(filter_data)) - - xc = index.query( - vector=vector, filter=filter_data, namespace="ns1", include_values=False, - include_metadata=True, - top_k=10000 - ) - if not xc['matches']: - if len(filter_data['$and']) == 3: - filter_data['$and'].pop(1) - print('query_vectors_by_metadata retrying without structured filters:', json.dumps(filter_data)) - xc = index.query( - vector=vector, filter=filter_data, namespace="ns1", include_values=False, - include_metadata=True, - top_k=20 + try: + filter_data = {'$and': [ + {'uid': {'$eq': uid}}, + ]} + if people or topics or entities or dates: + filter_data['$and'].append( + {'$or': [ + {'people': {'$in': people}}, + {'topics': {'$in': topics}}, + {'entities': {'$in': entities}}, + # {'dates': {'$in': dates_mentioned}}, + ]} ) - else: - return [] - - conversation_id_to_matches = defaultdict(int) - for item in xc['matches']: - metadata = item['metadata'] - conversation_id = metadata['memory_id'] - for topic in topics: - if topic in metadata.get('topics', []): - conversation_id_to_matches[conversation_id] += 1 - for entity in entities: - if entity in metadata.get('entities', []): - conversation_id_to_matches[conversation_id] += 1 - for person in people: - if person in metadata.get('people_mentioned', []): - conversation_id_to_matches[conversation_id] += 1 - - conversations_id = [item['id'].replace(f'{uid}-', '') for item in xc['matches']] - conversations_id.sort(key=lambda x: conversation_id_to_matches[x], reverse=True) - print('query_vectors_by_metadata result:', conversations_id) - return conversations_id[:limit] if len(conversations_id) > limit else conversations_id + if dates_filter and len(dates_filter) == 2 and dates_filter[0] and dates_filter[1]: + print('dates_filter', dates_filter) + filter_data['$and'].append( + {'created_at': {'$gte': int(dates_filter[0].timestamp()), '$lte': int(dates_filter[1].timestamp())}} + ) + + print('query_vectors_by_metadata:', json.dumps(filter_data)) + + xc = index.query( + vector=vector, filter=filter_data, namespace="ns1", include_values=False, + include_metadata=True, + top_k=10000 + ) + if not xc['matches']: + if len(filter_data['$and']) == 3: + filter_data['$and'].pop(1) + print('query_vectors_by_metadata retrying without structured filters:', json.dumps(filter_data)) + xc = index.query( + vector=vector, filter=filter_data, namespace="ns1", include_values=False, + include_metadata=True, + top_k=20 + ) + else: + return [] + + conversation_id_to_matches = defaultdict(int) + for item in xc['matches']: + metadata = item['metadata'] + conversation_id = metadata['memory_id'] + for topic in topics: + if topic in metadata.get('topics', []): + conversation_id_to_matches[conversation_id] += 1 + for entity in entities: + if entity in metadata.get('entities', []): + conversation_id_to_matches[conversation_id] += 1 + for person in people: + if person in metadata.get('people_mentioned', []): + conversation_id_to_matches[conversation_id] += 1 + + conversations_id = [item['id'].replace(f'{uid}-', '') for item in xc['matches']] + conversations_id.sort(key=lambda x: conversation_id_to_matches[x], reverse=True) + print('query_vectors_by_metadata result:', conversations_id) + return conversations_id[:limit] if len(conversations_id) > limit else conversations_id + except Exception as e: + print(f"Error in query_vectors_by_metadata: {e}") + return [] def delete_vector(conversation_id: str): - # TODO: does this work? - result = index.delete(ids=[conversation_id], namespace="ns1") - print('delete_vector', result) + try: + result = index.delete(ids=[conversation_id], namespace="ns1") + print('delete_vector', result) + except Exception as e: + print(f"Error in delete_vector: {e}") diff --git a/backend/utils/conversations/search.py b/backend/utils/conversations/search.py index 591efa3ffe..099597298d 100644 --- a/backend/utils/conversations/search.py +++ b/backend/utils/conversations/search.py @@ -1,86 +1,127 @@ import math import os from datetime import datetime -from typing import Dict - -import typesense +from typing import Dict, List # Check if Typesense is configured typesense_host = os.getenv('TYPESENSE_HOST') typesense_port = os.getenv('TYPESENSE_HOST_PORT') typesense_api_key = os.getenv('TYPESENSE_API_KEY') -if typesense_host and typesense_port and typesense_api_key: - # Typesense is properly configured - client = typesense.Client({ - 'nodes': [{ - 'host': typesense_host, - 'port': typesense_port, - 'protocol': 'https' - }], - 'api_key': typesense_api_key, - 'connection_timeout_seconds': 2 - }) - print(f"Connected to Typesense at {typesense_host}:{typesense_port}") -else: - # Typesense is not configured, create a mock client - print("WARNING: Typesense is not configured. Using a mock client for development.") - - class MockTypesenseClient: - def __init__(self): - self.collections = {} - - def collections(self): - return MockCollections(self.collections) - - def collection(self, name): - if name not in self.collections: - self.collections[name] = {} - return MockCollection(self.collections, name) - - class MockCollections: - def __init__(self, collections): - self.collections = collections - - def create(self, schema): - name = schema.get('name') - if name not in self.collections: - self.collections[name] = {} - return {'name': name} - - def retrieve(self): - return [{'name': name} for name in self.collections.keys()] - - class MockCollection: - def __init__(self, collections, name): - self.collections = collections - self.name = name - - def documents(self): - return MockDocuments(self.collections, self.name) +# Create a mock Typesense implementation for development or when Typesense is not configured +class MockTypesenseClient: + def __init__(self): + self.collections_data = {} + print("Using MockTypesenseClient - search will return empty results") + + def collections(self): + return MockCollections(self.collections_data) + + def collection(self, name): + if name not in self.collections_data: + self.collections_data[name] = {} + return MockCollection(self.collections_data, name) + +class MockCollections: + def __init__(self, collections): + self.collections = collections + + def create(self, schema): + name = schema.get('name') + if name not in self.collections: + self.collections[name] = {} + return {'name': name} + + def retrieve(self): + return [{'name': name} for name in self.collections.keys()] + +class MockCollection: + def __init__(self, collections, name): + self.collections = collections + self.name = name + # Add documents attribute directly to make it easier to use + self.documents = MockDocuments(self.collections, self.name) + + def search(self, search_parameters): + return { + 'found': 0, + 'hits': [], + 'page': search_parameters.get('page', 1), + 'request_params': search_parameters + } - def search(self, search_parameters): - return { - 'found': 0, - 'hits': [], - 'page': 1, - 'request_params': search_parameters - } +class MockDocuments: + def __init__(self, collections, collection_name): + self.collections = collections + self.collection_name = collection_name - class MockDocuments: - def __init__(self, collections, collection_name): - self.collections = collections - self.collection_name = collection_name + def create(self, document): + doc_id = document.get('id', 'mock_id') + self.collections.setdefault(self.collection_name, {}) + self.collections[self.collection_name][doc_id] = document + return {'id': doc_id} - def create(self, document): - return {'id': document.get('id')} + def delete(self, document_id): + if self.collection_name in self.collections and document_id in self.collections[self.collection_name]: + del self.collections[self.collection_name][document_id] + return {'id': document_id} - def delete(self, document_id): - return {'id': document_id} + def search(self, search_parameters): + return { + 'found': 0, + 'hits': [], + 'page': search_parameters.get('page', 1), + 'request_params': search_parameters + } +# Only import typesense if credentials are available +if typesense_host and typesense_port and typesense_api_key: + try: + import typesense + # Typesense is properly configured + client = typesense.Client({ + 'nodes': [{ + 'host': typesense_host, + 'port': typesense_port, + 'protocol': 'https' + }], + 'api_key': typesense_api_key, + 'connection_timeout_seconds': 2 + }) + print(f"Connected to Typesense at {typesense_host}:{typesense_port}") + + # Create default collections if needed + try: + collections = client.collections().retrieve() + collection_names = [c['name'] for c in collections] + if 'conversations' not in collection_names: + # Create conversations collection with appropriate schema + schema = { + 'name': 'conversations', + 'fields': [ + {'name': 'userId', 'type': 'string', 'facet': True}, + {'name': 'deleted', 'type': 'bool', 'facet': True}, + {'name': 'discarded', 'type': 'bool', 'facet': True}, + {'name': 'created_at', 'type': 'int64', 'facet': True}, + {'name': 'transcript_segments', 'type': 'string[]'}, + {'name': 'structured', 'type': 'string'} + ] + } + client.collections().create(schema) + print("Created 'conversations' collection in Typesense") + except Exception as e: + print(f"WARNING: Could not create default collections in Typesense: {e}") + except ImportError: + print("WARNING: typesense module not installed. Using mock client.") + client = MockTypesenseClient() + except Exception as e: + print(f"WARNING: Could not connect to Typesense: {e}. Using mock client.") + client = MockTypesenseClient() +else: + # Typesense is not configured + print("INFO: Typesense is not configured. Search functionality will use mock implementation.") client = MockTypesenseClient() - def search_conversations( uid: str, query: str, @@ -91,7 +132,6 @@ def search_conversations( end_date: int = None, ) -> Dict: try: - filter_by = f'userId:={uid} && deleted:=false' if not include_discarded: filter_by = filter_by + ' && discarded:=false' @@ -111,18 +151,49 @@ def search_conversations( 'page': page, } - results = client.collections['conversations'].documents.search(search_parameters) + # Safely access the collection and handle potential errors + try: + collection = client.collection('conversations') + results = collection.documents.search(search_parameters) + except AttributeError: + # If client is a mock or collections don't exist yet + try: + results = client.collections.create({ + 'name': 'conversations', + 'fields': [ + {'name': 'userId', 'type': 'string', 'facet': True}, + {'name': 'deleted', 'type': 'bool', 'facet': True}, + {'name': 'discarded', 'type': 'bool', 'facet': True}, + {'name': 'created_at', 'type': 'int64', 'facet': True}, + {'name': 'transcript_segments', 'type': 'string[]'}, + {'name': 'structured', 'type': 'string'} + ] + }) + results = {'hits': [], 'found': 0, 'page': page} + except: + results = {'hits': [], 'found': 0, 'page': page} + memories = [] - for item in results['hits']: - item['document']['created_at'] = datetime.utcfromtimestamp(item['document']['created_at']).isoformat() - item['document']['started_at'] = datetime.utcfromtimestamp(item['document']['started_at']).isoformat() - item['document']['finished_at'] = datetime.utcfromtimestamp(item['document']['finished_at']).isoformat() - memories.append(item['document']) + for item in results.get('hits', []): + doc = item.get('document', {}) + # Convert timestamps to ISO format + for ts_field in ['created_at', 'started_at', 'finished_at']: + if ts_field in doc and isinstance(doc[ts_field], (int, float)): + doc[ts_field] = datetime.utcfromtimestamp(doc[ts_field]).isoformat() + memories.append(doc) + return { 'items': memories, - 'total_pages': math.ceil(results['found'] / per_page), + 'total_pages': math.ceil(results.get('found', 0) / per_page), 'current_page': page, 'per_page': per_page } except Exception as e: - raise Exception(f"Failed to search conversations: {str(e)}") + print(f"Error in search_conversations: {e}") + # Return empty result set on error + return { + 'items': [], + 'total_pages': 0, + 'current_page': page, + 'per_page': per_page + } diff --git a/backend/utils/other/hume.py b/backend/utils/other/hume.py index 79b6b3b8b7..e705832a80 100644 --- a/backend/utils/other/hume.py +++ b/backend/utils/other/hume.py @@ -1,4 +1,5 @@ import os +import time import requests @@ -193,10 +194,41 @@ def request_user_expression_mersurement(self, urls: [str]): return {"result": HumeJobResponseModel.from_dict(resp.json())} -hume_client = HumeClient( - api_key=os.getenv('HUME_API_KEY'), - callback_url=os.getenv('HUME_CALLBACK_URL'), -) +class MockHumeClient: + """A mock implementation of the HumeClient for development and when API keys aren't available""" + + def __init__(self): + print("Using MockHumeClient - Hume API calls will return empty results") + + def request_user_expression_mersurement(self, urls): + """Mock implementation that returns a success response with empty data""" + print(f"MOCK: Hume API called with {len(urls)} URLs") + return { + "result": HumeJobResponseModel.from_dict({ + "id": "mock-job-id-" + str(int(time.time())) + }) + } + +# Initialize the Hume client based on available environment variables +hume_api_key = os.getenv('HUME_API_KEY') +hume_callback_url = os.getenv('HUME_CALLBACK_URL') + +if hume_api_key and hume_callback_url: + # Both API key and callback URL are available + hume_client = HumeClient( + api_key=hume_api_key, + callback_url=hume_callback_url, + ) + print("Hume API client initialized with provided credentials") +else: + # Missing one or both required parameters + if hume_api_key: + print("WARNING: HUME_CALLBACK_URL is not set. Using mock Hume client.") + elif hume_callback_url: + print("WARNING: HUME_API_KEY is not set. Using mock Hume client.") + else: + print("INFO: Hume API is not configured. Using mock client.") + hume_client = MockHumeClient() def get_hume(): diff --git a/backend/utils/other/storage.py b/backend/utils/other/storage.py index 7d287274fc..78db464854 100644 --- a/backend/utils/other/storage.py +++ b/backend/utils/other/storage.py @@ -1,7 +1,8 @@ import datetime import json import os -from typing import List +import tempfile +from typing import List, Optional from google.cloud import storage from google.oauth2 import service_account @@ -20,79 +21,193 @@ except Exception as e: print(f"Error reading google-credentials.json: {e}") -if os.environ.get('SERVICE_ACCOUNT_JSON'): - service_account_info = json.loads(os.environ["SERVICE_ACCOUNT_JSON"]) - credentials = service_account.Credentials.from_service_account_info(service_account_info) - storage_client = storage.Client(credentials=credentials, project=project_id) -else: - storage_client = storage.Client(project=project_id) - -speech_profiles_bucket = os.getenv('BUCKET_SPEECH_PROFILES') -postprocessing_audio_bucket = os.getenv('BUCKET_POSTPROCESSING') -memories_recordings_bucket = os.getenv('BUCKET_MEMORIES_RECORDINGS') -syncing_local_bucket = os.getenv('BUCKET_TEMPORAL_SYNC_LOCAL') -omi_plugins_bucket = os.getenv('BUCKET_PLUGINS_LOGOS') -app_thumbnails_bucket = os.getenv('BUCKET_APP_THUMBNAILS') -chat_files_bucket = os.getenv('BUCKET_CHAT_FILES') +try: + if os.environ.get('SERVICE_ACCOUNT_JSON'): + service_account_info = json.loads(os.environ["SERVICE_ACCOUNT_JSON"]) + credentials = service_account.Credentials.from_service_account_info(service_account_info) + storage_client = storage.Client(credentials=credentials, project=project_id) + else: + storage_client = storage.Client(project=project_id) +except Exception as e: + print(f"WARNING: Could not initialize storage client: {e}") + storage_client = None + +# Get bucket names from environment variables with empty string fallbacks +speech_profiles_bucket = os.getenv('BUCKET_SPEECH_PROFILES', '') +postprocessing_audio_bucket = os.getenv('BUCKET_POSTPROCESSING', '') +memories_recordings_bucket = os.getenv('BUCKET_MEMORIES_RECORDINGS', '') +syncing_local_bucket = os.getenv('BUCKET_TEMPORAL_SYNC_LOCAL', '') +omi_plugins_bucket = os.getenv('BUCKET_PLUGINS_LOGOS', '') +app_thumbnails_bucket = os.getenv('BUCKET_APP_THUMBNAILS', '') +chat_files_bucket = os.getenv('BUCKET_CHAT_FILES', '') + +# Helper function to create a mock blob for cases when storage is not available +class MockBlob: + def __init__(self, name): + self.name = name + self._exists = False + self._local_path = None + + def exists(self): + return self._exists + + def upload_from_filename(self, filename): + # Store the local file path for potential retrieval + self._local_path = filename + self._exists = True + print(f"MOCK: Uploaded {filename} to {self.name}") + return True + + def download_to_filename(self, filename): + # If we have a local path, copy the file to the destination + if self._local_path and os.path.exists(self._local_path): + import shutil + shutil.copy(self._local_path, filename) + print(f"MOCK: Downloaded {self.name} to {filename}") + return True + print(f"MOCK: Could not download {self.name} to {filename}") + return False + + def delete(self): + self._exists = False + print(f"MOCK: Deleted {self.name}") + return True + +# Helper function to get a bucket, with fallback to mock implementation +def get_bucket(bucket_name): + if not storage_client or not bucket_name: + return None + try: + return storage_client.bucket(bucket_name) + except Exception as e: + print(f"Error getting bucket {bucket_name}: {e}") + return None + +# Helper function to generate a mock signed URL for local development +def generate_mock_signed_url(blob_name): + return f"mock-signed-url://{blob_name}?mock=true" # ******************************************* # ************* SPEECH PROFILE ************** # ******************************************* def upload_profile_audio(file_path: str, uid: str): - bucket = storage_client.bucket(speech_profiles_bucket) - path = f'{uid}/speech_profile.wav' - blob = bucket.blob(path) - blob.upload_from_filename(file_path) - return f'https://storage.googleapis.com/{speech_profiles_bucket}/{path}' + try: + bucket = get_bucket(speech_profiles_bucket) + if not bucket: + print(f"WARNING: Speech profiles bucket not configured. Using local file.") + return f"file://{file_path}" + path = f'{uid}/speech_profile.wav' + blob = bucket.blob(path) + blob.upload_from_filename(file_path) + return f'https://storage.googleapis.com/{speech_profiles_bucket}/{path}' + except Exception as e: + print(f"Error uploading profile audio: {e}") + return f"file://{file_path}" -def get_user_has_speech_profile(uid: str) -> bool: - bucket = storage_client.bucket(speech_profiles_bucket) - blob = bucket.blob(f'{uid}/speech_profile.wav') - return blob.exists() - -def get_profile_audio_if_exists(uid: str, download: bool = True) -> str: - bucket = storage_client.bucket(speech_profiles_bucket) - path = f'{uid}/speech_profile.wav' - blob = bucket.blob(path) - if blob.exists(): - if download: - file_path = f'_temp/{uid}_speech_profile.wav' - blob.download_to_filename(file_path) - return file_path - return _get_signed_url(blob, 60) - - return None +def get_user_has_speech_profile(uid: str) -> bool: + try: + bucket = get_bucket(speech_profiles_bucket) + if not bucket: + # Check if there's a local file indicating a speech profile + local_path = f'_speech_profiles/{uid}/speech_profile.wav' + return os.path.exists(local_path) + + blob = bucket.blob(f'{uid}/speech_profile.wav') + return blob.exists() + except Exception as e: + print(f"Error checking speech profile: {e}") + return False + + +def get_profile_audio_if_exists(uid: str, download: bool = True) -> Optional[str]: + try: + bucket = get_bucket(speech_profiles_bucket) + if not bucket: + # Check for local file + local_path = f'_speech_profiles/{uid}/speech_profile.wav' + if os.path.exists(local_path): + return local_path + return None + + path = f'{uid}/speech_profile.wav' + blob = bucket.blob(path) + if blob.exists(): + if download: + file_path = f'_temp/{uid}_speech_profile.wav' + blob.download_to_filename(file_path) + return file_path + return _get_signed_url(blob, 60) + + return None + except Exception as e: + print(f"Error getting profile audio: {e}") + return None def upload_additional_profile_audio(file_path: str, uid: str) -> None: - bucket = storage_client.bucket(speech_profiles_bucket) - path = f'{uid}/additional_profile_recordings/{file_path.split("/")[-1]}' - blob = bucket.blob(path) - blob.upload_from_filename(file_path) + try: + bucket = get_bucket(speech_profiles_bucket) + if not bucket: + # Save to local directory + os.makedirs(f'_speech_profiles/{uid}/additional_profile_recordings', exist_ok=True) + local_path = f'_speech_profiles/{uid}/additional_profile_recordings/{os.path.basename(file_path)}' + import shutil + shutil.copy(file_path, local_path) + print(f"Saved additional profile audio to {local_path}") + return + + path = f'{uid}/additional_profile_recordings/{file_path.split("/")[-1]}' + blob = bucket.blob(path) + blob.upload_from_filename(file_path) + except Exception as e: + print(f"Error uploading additional profile audio: {e}") def delete_additional_profile_audio(uid: str, file_name: str) -> None: - bucket = storage_client.bucket(speech_profiles_bucket) - blob = bucket.blob(f'{uid}/additional_profile_recordings/{file_name}') - if blob.exists(): - print('delete_additional_profile_audio deleting', file_name) - blob.delete() + try: + bucket = get_bucket(speech_profiles_bucket) + if not bucket: + # Delete from local directory + local_path = f'_speech_profiles/{uid}/additional_profile_recordings/{file_name}' + if os.path.exists(local_path): + os.remove(local_path) + print(f"Deleted local file: {local_path}") + return + + blob = bucket.blob(f'{uid}/additional_profile_recordings/{file_name}') + if blob.exists(): + print('delete_additional_profile_audio deleting', file_name) + blob.delete() + except Exception as e: + print(f"Error deleting additional profile audio: {e}") def get_additional_profile_recordings(uid: str, download: bool = False) -> List[str]: - bucket = storage_client.bucket(speech_profiles_bucket) - blobs = bucket.list_blobs(prefix=f'{uid}/additional_profile_recordings/') - if download: - paths = [] - for blob in blobs: - file_path = f'_temp/{uid}_{blob.name.split("/")[-1]}' - blob.download_to_filename(file_path) - paths.append(file_path) - return paths + try: + bucket = get_bucket(speech_profiles_bucket) + if not bucket: + # Check local directory + dir_path = f'_speech_profiles/{uid}/additional_profile_recordings' + if not os.path.exists(dir_path): + return [] + files = os.listdir(dir_path) + return [f'file://{dir_path}/{file}' for file in files] + + blobs = bucket.list_blobs(prefix=f'{uid}/additional_profile_recordings/') + if download: + paths = [] + for blob in blobs: + file_path = f'_temp/{uid}_{blob.name.split("/")[-1]}' + blob.download_to_filename(file_path) + paths.append(file_path) + return paths - return [_get_signed_url(blob, 60) for blob in blobs] + return [_get_signed_url(blob, 60) for blob in blobs] + except Exception as e: + print(f"Error getting additional profile recordings: {e}") + return [] # ******************************************** @@ -241,13 +356,31 @@ def delete_syncing_temporal_file(file_path: str): # ************* UTILS ************** # ********************************** -def _get_signed_url(blob, minutes): - if cached := get_cached_signed_url(blob.name): - return cached - - signed_url = blob.generate_signed_url(version="v4", expiration=datetime.timedelta(minutes=minutes), method="GET") - cache_signed_url(blob.name, signed_url, minutes * 60) - return signed_url +def _get_signed_url(blob, expiration_minutes=60): + try: + if isinstance(blob, MockBlob): + return generate_mock_signed_url(blob.name) + + # Check if the URL is already cached + cached_url = get_cached_signed_url(blob.name) + if cached_url: + return cached_url + + # Generate a new signed URL + expiration = datetime.timedelta(minutes=expiration_minutes) + signed_url = blob.generate_signed_url( + version="v4", + expiration=expiration, + method="GET" + ) + + # Cache the URL + cache_signed_url(blob.name, signed_url, int(expiration.total_seconds())) + + return signed_url + except Exception as e: + print(f"Error generating signed URL: {e}") + return generate_mock_signed_url(blob.name) def upload_plugin_logo(file_path: str, plugin_id: str): diff --git a/backend/utils/stt/streaming.py b/backend/utils/stt/streaming.py index 16799482c9..d50dcb27f4 100644 --- a/backend/utils/stt/streaming.py +++ b/backend/utils/stt/streaming.py @@ -88,29 +88,58 @@ async def send_initial_file(data: List[List[int]], transcript_socket): # Initialize Deepgram client based on environment configuration +deepgram_api_key = os.getenv('DEEPGRAM_API_KEY') is_dg_self_hosted = os.getenv('DEEPGRAM_SELF_HOSTED_ENABLED', '').lower() == 'true' -deepgram_options = DeepgramClientOptions(options={"keepalive": "true", "termination_exception_connect": "true"}) +# Initialize client options +deepgram_options = DeepgramClientOptions(options={"keepalive": "true", "termination_exception_connect": "true"}) deepgram_beta_options = DeepgramClientOptions(options={"keepalive": "true", "termination_exception_connect": "true"}) deepgram_beta_options.url = "https://api.beta.deepgram.com" +# Configure self-hosted URL if enabled if is_dg_self_hosted: dg_self_hosted_url = os.getenv('DEEPGRAM_SELF_HOSTED_URL') if not dg_self_hosted_url: - raise ValueError("DEEPGRAM_SELF_HOSTED_URL must be set when DEEPGRAM_SELF_HOSTED_ENABLED is true") - # Override only the URL while keeping all other options - deepgram_options.url = dg_self_hosted_url - deepgram_beta_options.url = dg_self_hosted_url - print(f"Using Deepgram self-hosted at: {dg_self_hosted_url}") - -deepgram = DeepgramClient(os.getenv('DEEPGRAM_API_KEY'), deepgram_options) -deepgram_beta = DeepgramClient(os.getenv('DEEPGRAM_API_KEY'), deepgram_beta_options) + print("WARNING: DEEPGRAM_SELF_HOSTED_URL must be set when DEEPGRAM_SELF_HOSTED_ENABLED is true. Using default Deepgram URL.") + else: + # Override only the URL while keeping all other options + deepgram_options.url = dg_self_hosted_url + deepgram_beta_options.url = dg_self_hosted_url + print(f"Using Deepgram self-hosted at: {dg_self_hosted_url}") + +# Initialize Deepgram clients if API key is available +if deepgram_api_key: + try: + deepgram = DeepgramClient(deepgram_api_key, deepgram_options) + deepgram_beta = DeepgramClient(deepgram_api_key, deepgram_beta_options) + print("Deepgram clients initialized successfully") + except Exception as e: + print(f"WARNING: Failed to initialize Deepgram clients: {e}") + deepgram = None + deepgram_beta = None +else: + print("WARNING: DEEPGRAM_API_KEY is not set. Deepgram transcription will not be available.") + deepgram = None + deepgram_beta = None async def process_audio_dg( stream_transcript, language: str, sample_rate: int, channels: int, preseconds: int = 0, model: str = 'nova-2-general', ): print('process_audio_dg', language, sample_rate, channels, preseconds) + # Check if Deepgram is available + if not deepgram and not deepgram_beta: + print("ERROR: Deepgram is not configured. Cannot transcribe audio.") + stream_transcript([{ + 'speaker': "SPEAKER_0", + 'start': 0, + 'end': 1, + 'text': "[Transcription unavailable - Deepgram not configured]", + 'is_user': True, + 'person_id': None, + }]) + return None + def on_message(self, result, **kwargs): # print(f"Received message from Deepgram") # Log when message is received sentence = result.channel.alternatives[0].transcript @@ -153,6 +182,15 @@ def on_message(self, result, **kwargs): def on_error(self, error, **kwargs): print(f"Error: {error}") + # Send a notification to the client that an error occurred + stream_transcript([{ + 'speaker': "SPEAKER_0", + 'start': 0, + 'end': 1, + 'text': f"[Transcription error: {error}]", + 'is_user': True, + 'person_id': None, + }]) print("Connecting to Deepgram") # Log before connection attempt return connect_to_deepgram_with_backoff(on_message, on_error, language, sample_rate, channels, model) @@ -185,8 +223,12 @@ def connect_to_deepgram(on_message, on_error, language: str, sample_rate: int, c try: # get connection by model if model == "nova-3": + if not deepgram_beta: + raise Exception("Deepgram beta client is not available") dg_connection = deepgram_beta.listen.websocket.v("1") else: + if not deepgram: + raise Exception("Deepgram client is not available") dg_connection = deepgram.listen.websocket.v("1") dg_connection.on(LiveTranscriptionEvents.Transcript, on_message) @@ -244,7 +286,16 @@ async def process_audio_soniox(stream_transcript, sample_rate: int, language: st # Soniox supports diarization primarily for English api_key = os.getenv('SONIOX_API_KEY') if not api_key: - raise ValueError("API key is not set. Please set the SONIOX_API_KEY environment variable.") + print("ERROR: SONIOX_API_KEY is not set. Cannot transcribe audio with Soniox.") + stream_transcript([{ + 'speaker': "SPEAKER_0", + 'start': 0, + 'end': 1, + 'text': "[Transcription unavailable - Soniox not configured]", + 'is_user': True, + 'person_id': None, + }]) + return None uri = 'wss://stt-rt.soniox.com/transcribe-websocket' @@ -408,7 +459,20 @@ async def on_message(): async def process_audio_speechmatics(stream_transcript, sample_rate: int, language: str, preseconds: int = 0): + # Check if Speechmatics API key is available api_key = os.getenv('SPEECHMATICS_API_KEY') + if not api_key: + print("ERROR: SPEECHMATICS_API_KEY is not set. Cannot transcribe audio with Speechmatics.") + stream_transcript([{ + 'speaker': "SPEAKER_0", + 'start': 0, + 'end': 1, + 'text': "[Transcription unavailable - Speechmatics not configured]", + 'is_user': True, + 'person_id': None, + }]) + return None + uri = 'wss://eu2.rt.speechmatics.com/v2' request = { From 6d2cc7c89467a0a93cd4a95836a8ddd4746c96a6 Mon Sep 17 00:00:00 2001 From: Petr Korolev Date: Thu, 10 Apr 2025 15:13:15 +0200 Subject: [PATCH 23/23] update .env.template remove unused var and add mised bucket variables and admin marketplace --- backend/.env.template | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/.env.template b/backend/.env.template index 65708e8a6d..89b9f8e29a 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -35,8 +35,12 @@ HUME_CALLBACK_URL= # Google Cloud Storage Buckets BUCKET_SPEECH_PROFILES= -BUCKET_BACKUPS= BUCKET_PLUGINS_LOGOS= +BUCKET_POSTPROCESSING= +BUCKET_MEMORIES_RECORDINGS= +BUCKET_TEMPORAL_SYNC_LOCAL= +BUCKET_APP_THUMBNAILS= +BUCKET_CHAT_FILES= # Vector Database (for advanced retrieval) PINECONE_API_KEY= @@ -63,4 +67,7 @@ HTTP_DELETE_TIMEOUT=30 # Developer Options LOCAL_DEVELOPMENT=false -NO_SOCKET_TIMEOUT=false \ No newline at end of file +NO_SOCKET_TIMEOUT=false + +# Admin Options +MARKETPLACE_APP_REVIEWERS= \ No newline at end of file