Merge pull request #559 from phospho-app/fred-disable-analytics

feat: disable analytics for given project

backend/app/services/mongo/projects.py

@@ -160,6 +160,12 @@ async def delete_project_related_resources(project_id: str):
 
 
 async def update_project(project: Project, **kwargs) -> Project:
+    """
+    Update a project with the provided kwargs.
+
+    TODO: Update only the fields that are provided in the kwargs and not the whole project
+    otherwise creates mongoDB timeouts
+    """
     mongo_db = await get_mongo_db()
 
     # Construct the payload of the update
@@ -189,85 +195,87 @@ async def update_project(project: Project, **kwargs) -> Project:
                     "Error updating sentiment threshold: missing score or magnitude"
                 )
 
-        # Create a new recipe for each event in the payload
-        for event_name, event_definition in (
-            payload.get("settings", {}).get("events", {}).items()
-        ):
-            try:
-                event_definition_model = EventDefinition.model_validate(
-                    event_definition
-                )
-                recipe = Recipe(
-                    org_id=project.org_id,
-                    project_id=project.id,
-                    recipe_type="event_detection",
-                    parameters=event_definition_model.model_dump(),
-                )
-                mongo_db["recipes"].insert_one(recipe.model_dump())
-                updated_project.settings.events[event_name].recipe_id = recipe.id
-                event_definition_model.recipe_id = recipe.id
-                # update event_definition with event_id
-                mongo_db["event_definitions"].update_one(
-                    {"project_id": project.id, "id": event_definition_model.id},
-                    {"$set": event_definition_model.model_dump()},
-                    upsert=True,
-                )
-            except Exception as e:
-                logger.error(f"Error creating recipe for event {event_name}: {e}")
-
-        # Detect if an event has been removed
-        # Create a set of events that are NOT in the payload, but are in the current project, based on their id
-
-        updated_project_events_ids = [
-            event_definition.id
-            for event_definition in updated_project.settings.events.values()
-        ]
-
-        for event_name, event_definition in project.settings.events.items():
-            if event_definition.id not in updated_project_events_ids:
-                # Event has been removed
-                try:
-                    event_definition.removed = True
-                    mongo_db["event_definitions"].update_one(
-                        {"project_id": project.id, "id": event_definition.id},
-                        {"$set": event_definition.model_dump()},
-                    )
-                    logger.info(f"Event {event_definition.id} has been removed")
-                except Exception as e:
-                    logger.error(f"Error removing event {event_definition.id}: {e}")
-
-                # Disable the recipe
+        if "events" in payload.get("settings", {}):
+            # Create a new recipe for each event in the payload
+            for event_name, event_definition in (
+                payload.get("settings", {}).get("events", {}).items()
+            ):
                 try:
-                    recipe = await mongo_db["recipes"].find_one(
-                        {"id": event_definition.recipe_id}
+                    event_definition_model = EventDefinition.model_validate(
+                        event_definition
                     )
-                    recipe = Recipe.model_validate(recipe)
-                    recipe.status = "deleted"
-                    mongo_db["recipes"].update_one(
-                        {"id": event_definition.recipe_id},
-                        {"$set": recipe.model_dump()},
+                    recipe = Recipe(
+                        org_id=project.org_id,
+                        project_id=project.id,
+                        recipe_type="event_detection",
+                        parameters=event_definition_model.model_dump(),
                     )
-                except Exception as e:
-                    logger.error(f"Error disabling recipe for event {event_name}: {e}")
-                # Remove all historical events
-                try:
-                    mongo_db["events"].update_many(
-                        {
-                            "project_id": project.id,
-                            "event_definition.id": event_definition.id,
-                        },
-                        {"$set": {"removed": True}},
-                    )
-                    logger.debug(
-                        f"Removing all historical events for event {event_definition.id}"
+                    mongo_db["recipes"].insert_one(recipe.model_dump())
+                    updated_project.settings.events[event_name].recipe_id = recipe.id
+                    event_definition_model.recipe_id = recipe.id
+                    # update event_definition with event_id
+                    mongo_db["event_definitions"].update_one(
+                        {"project_id": project.id, "id": event_definition_model.id},
+                        {"$set": event_definition_model.model_dump()},
+                        upsert=True,
                     )
                 except Exception as e:
-                    logger.error(
-                        f"Error removing all historical events for event {event_definition.id}: {e}"
-                    )
+                    logger.error(f"Error creating recipe for event {event_name}: {e}")
+
+            # Detect if an event has been removed
+            # Create a set of events that are NOT in the payload, but are in the current project, based on their id
+            updated_project_events_ids = [
+                event_definition.id
+                for event_definition in updated_project.settings.events.values()
+            ]
+
+            for event_name, event_definition in project.settings.events.items():
+                if event_definition.id not in updated_project_events_ids:
+                    # Event has been removed
+                    try:
+                        event_definition.removed = True
+                        mongo_db["event_definitions"].update_one(
+                            {"project_id": project.id, "id": event_definition.id},
+                            {"$set": event_definition.model_dump()},
+                        )
+                        logger.info(f"Event {event_definition.id} has been removed")
+                    except Exception as e:
+                        logger.error(f"Error removing event {event_definition.id}: {e}")
+
+                    # Disable the recipe
+                    try:
+                        recipe = await mongo_db["recipes"].find_one(
+                            {"id": event_definition.recipe_id}
+                        )
+                        recipe = Recipe.model_validate(recipe)
+                        recipe.status = "deleted"
+                        mongo_db["recipes"].update_one(
+                            {"id": event_definition.recipe_id},
+                            {"$set": recipe.model_dump()},
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Error disabling recipe for event {event_name}: {e}"
+                        )
+                    # Remove all historical events
+                    try:
+                        mongo_db["events"].update_many(
+                            {
+                                "project_id": project.id,
+                                "event_definition.id": event_definition.id,
+                            },
+                            {"$set": {"removed": True}},
+                        )
+                        logger.debug(
+                            f"Removing all historical events for event {event_definition.id}"
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Error removing all historical events for event {event_definition.id}: {e}"
+                        )
 
         # Update the database
-        _ = await mongo_db["projects"].update_one(
+        await mongo_db["projects"].update_one(
             {"id": project.id}, {"$set": updated_project.model_dump()}
         )
 

extractor/app/services/pipelines.py

@@ -183,6 +183,13 @@ async def task_event_detection_pipeline(
     if project.settings is None:
         logger.warning(f"Project with id {project_id} has no settings")
         return []
+    # We check if the event detection is enabled in the project settings
+    if (
+        project.settings.run_event_detection is not None
+        and not project.settings.run_event_detection
+    ):
+        logger.info(f"Event detection is disabled for project {project_id}")
+        return []
     # Convert to the proper lab project object
     # TODO : Normalize the project definition by storing all db models in the phospho module
     # and importing models from the phospho module
@@ -280,6 +287,18 @@ async def task_scoring_pipeline(
     logger.debug(f"Run the task scoring pipeline for task {task.id}")
     mongo_db = await get_mongo_db()
 
+    # Fetch run_evals variable in settings in the project, if it doesn't exist, return None
+    project = await mongo_db["projects"].find_one(
+        {"id": task.project_id},
+        {"settings.run_evals": 1},
+    )
+
+    run_evals = project.get("settings", {}).get("run_evals", None)
+    logger.debug(run_evals)
+    if run_evals is not None and not run_evals:
+        logger.info(f"run_evals is disabled for project {task.project_id}")
+        return None
+
     # We want 50/50 success and failure examples
     nb_success = int(config.FEW_SHOT_MAX_NUMBER_OF_EXAMPLES / 2)
     nb_failure = int(config.FEW_SHOT_MAX_NUMBER_OF_EXAMPLES / 2)
@@ -461,8 +480,8 @@ async def task_main_pipeline(task: Task, save_task: bool = True) -> PipelineResu
     start_time = time.time()
     logger.info(f"Starting main pipeline for task {task.id}")
 
-    # For now, do things sequentially
-
+    # Do the session scoring -> success, failure
+    mongo_db = await get_mongo_db()
     # Do the event detection
     if task.test_id is None:
         # Run the event detection pipeline
@@ -472,9 +491,6 @@ async def task_main_pipeline(task: Task, save_task: bool = True) -> PipelineResu
             task
         )
 
-    # Do the session scoring -> success, failure
-    mongo_db = await get_mongo_db()
-
     if save_task:
         task_in_db = await mongo_db["tasks"].find_one({"id": task.id})
         if task_in_db.get("flag") is None:
@@ -579,6 +595,14 @@ async def sentiment_and_language_analysis_pipeline(
     mongo_db = await get_mongo_db()
     project = await get_project_by_id(task.project_id)
 
+    if (
+        project.settings is not None
+        and project.settings.run_sentiment_language is not None
+        and not project.settings.run_sentiment_language
+    ):
+        logger.info(f"Sentiment analysis is disabled for project {task.project_id}")
+        return None, None
+
     # Default values
     score_threshold = 0.3
     magnitude_threshold = 0.6

phospho-python/phospho/models.py

@@ -193,6 +193,9 @@ class ProjectSettings(BaseModel):
     sentiment_threshold: Optional[Threshold] = Field(default_factory=Threshold)
     last_langsmith_extract: Optional[str] = None
     last_langfuse_extract: Optional[str] = None
+    run_evals: Optional[bool] = True
+    run_sentiment_language: Optional[bool] = True
+    run_event_detection: Optional[bool] = True
 
 
 class Project(DatedBaseModel):

platform/app/org/settings/project/page.tsx

@@ -2,6 +2,7 @@
 
 import CreateProjectDialog from "@/components/projects/create-project-form";
 import AlertDialogDeleteProject from "@/components/projects/delete-project-popup";
+import DisableAnalytics from "@/components/settings/disable-analytics";
 import TaskProgress from "@/components/settings/tasks-quota";
 import {
   AlertDialog,
@@ -88,6 +89,7 @@ export default function Page() {
           </Link>
           <AlertDialogDeleteProject />
         </div>
+        <DisableAnalytics selectedProject={selectedProject} />
         <div>
           {plan === "usage_based" && (
             <div>

platform/components/insights/events/create-event.tsx

@@ -21,11 +21,7 @@ import { Separator } from "@/components/ui/separator";
 import { SheetFooter, SheetHeader, SheetTitle } from "@/components/ui/sheet";
 import { Textarea } from "@/components/ui/textarea";
 import { useToast } from "@/components/ui/use-toast";
-import {
-  DetectionEngine,
-  DetectionScope,
-  ScoreRangeType,
-} from "@/models/models";
+import { DetectionEngine, DetectionScope } from "@/models/models";
 import { ScoreRangeSettings } from "@/models/models";
 import { dataStateStore, navigationStateStore } from "@/store/store";
 import { zodResolver } from "@hookform/resolvers/zod";