feat: add manual retry for failed async operations (#537)

- API: POST /v1/default/banks/{bank_id}/operations/{operation_id}/retry
  resets status to pending so the worker re-executes the task
- UI: Retry button on failed operations in the operations view
- Control plane proxy route + ControlPlaneClient.retryOperation()
- Updated OpenAPI spec, all generated clients, and operations docs

Author: nicoloboschi

hindsight-api/hindsight_api/api/http.py

@@ -1558,6 +1558,24 @@ class CancelOperationResponse(BaseModel):
     operation_id: str
 
 
+class RetryOperationResponse(BaseModel):
+    """Response model for retry operation endpoint."""
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "success": True,
+                "message": "Operation 550e8400-e29b-41d4-a716-446655440000 queued for retry",
+                "operation_id": "550e8400-e29b-41d4-a716-446655440000",
+            }
+        }
+    )
+
+    success: bool
+    message: str
+    operation_id: str
+
+
 class ChildOperationStatus(BaseModel):
     """Status of a child operation (for batch operations)."""
 
@@ -3532,6 +3550,39 @@ async def api_cancel_operation(
             logger.error(f"Error in /v1/default/banks/{bank_id}/operations/{operation_id}: {error_detail}")
             raise HTTPException(status_code=500, detail=str(e))
 
+    @app.post(
+        "/v1/default/banks/{bank_id}/operations/{operation_id}/retry",
+        response_model=RetryOperationResponse,
+        summary="Retry a failed async operation",
+        description="Re-queue a failed async operation so the worker picks it up again",
+        operation_id="retry_operation",
+        tags=["Operations"],
+    )
+    async def api_retry_operation(
+        bank_id: str, operation_id: str, request_context: RequestContext = Depends(get_request_context)
+    ):
+        """Retry a failed async operation."""
+        try:
+            try:
+                uuid.UUID(operation_id)
+            except ValueError:
+                raise HTTPException(status_code=400, detail=f"Invalid operation_id format: {operation_id}")
+
+            result = await app.state.memory.retry_operation(bank_id, operation_id, request_context=request_context)
+            return RetryOperationResponse(**result)
+        except ValueError as e:
+            raise HTTPException(status_code=404, detail=str(e))
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in POST /v1/default/banks/{bank_id}/operations/{operation_id}/retry: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
     @app.get(
         "/v1/default/banks/{bank_id}/profile",
         response_model=BankProfileResponse,

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -7118,6 +7118,64 @@ async def cancel_operation(
                 "bank_id": bank_id,
             }
 
+    async def retry_operation(
+        self,
+        bank_id: str,
+        operation_id: str,
+        *,
+        request_context: "RequestContext",
+    ) -> dict[str, Any]:
+        """Re-queue a failed async operation."""
+        await self._authenticate_tenant(request_context)
+        from hindsight_api.extensions import OperationValidationError
+
+        if self._operation_validator:
+            from hindsight_api.extensions import BankWriteContext
+
+            ctx = BankWriteContext(bank_id=bank_id, operation="retry_operation", request_context=request_context)
+            await self._validate_operation(self._operation_validator.validate_bank_write(ctx))
+        pool = await self._get_pool()
+
+        op_uuid = uuid.UUID(operation_id)
+
+        async with acquire_with_retry(pool) as conn:
+            row = await conn.fetchrow(
+                f"SELECT bank_id, status FROM {fq_table('async_operations')} WHERE operation_id = $1 AND bank_id = $2",
+                op_uuid,
+                bank_id,
+            )
+
+            if not row:
+                raise ValueError(f"Operation {operation_id} not found for bank {bank_id}")
+
+            if row["status"] != "failed":
+                raise OperationValidationError(
+                    f"Operation {operation_id} cannot be retried: status is '{row['status']}', expected 'failed'",
+                    409,
+                )
+
+            await conn.execute(
+                f"""
+                UPDATE {fq_table("async_operations")}
+                SET status = 'pending',
+                    error_message = NULL,
+                    completed_at = NULL,
+                    next_retry_at = NULL,
+                    worker_id = NULL,
+                    claimed_at = NULL,
+                    retry_count = 0,
+                    updated_at = NOW()
+                WHERE operation_id = $1
+                """,
+                op_uuid,
+            )
+
+            return {
+                "success": True,
+                "message": f"Operation {operation_id} queued for retry",
+                "operation_id": operation_id,
+            }
+
     async def update_bank(
         self,
         bank_id: str,

hindsight-clients/go/api/openapi.yaml

@@ -1771,6 +1771,51 @@ paths:
       summary: Get operation status
       tags:
       - Operations
+  /v1/default/banks/{bank_id}/operations/{operation_id}/retry:
+    post:
+      description: Re-queue a failed async operation so the worker picks it up again
+      operationId: retry_operation
+      parameters:
+      - explode: false
+        in: path
+        name: bank_id
+        required: true
+        schema:
+          title: Bank Id
+          type: string
+        style: simple
+      - explode: false
+        in: path
+        name: operation_id
+        required: true
+        schema:
+          title: Operation Id
+          type: string
+        style: simple
+      - explode: false
+        in: header
+        name: authorization
+        required: false
+        schema:
+          nullable: true
+          type: string
+        style: simple
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RetryOperationResponse'
+          description: Successful Response
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+          description: Validation Error
+      summary: Retry a failed async operation
+      tags:
+      - Operations
   /v1/default/banks/{bank_id}/profile:
     get:
       deprecated: true
@@ -4748,6 +4793,27 @@ components:
       - items_count
       - success
       title: RetainResponse
+    RetryOperationResponse:
+      description: Response model for retry operation endpoint.
+      example:
+        message: Operation 550e8400-e29b-41d4-a716-446655440000 queued for retry
+        operation_id: 550e8400-e29b-41d4-a716-446655440000
+        success: true
+      properties:
+        success:
+          title: Success
+          type: boolean
+        message:
+          title: Message
+          type: string
+        operation_id:
+          title: Operation Id
+          type: string
+      required:
+      - message
+      - operation_id
+      - success
+      title: RetryOperationResponse
     SourceFactsIncludeOptions:
       description: Options for including source facts for observation-type results.
       properties: