diff --git a/application/single_app/config.py b/application/single_app/config.py index a69d7c20c..2dc13e9a2 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.071" +VERSION = "0.250.073" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_assistant_table_exports.py b/application/single_app/functions_assistant_table_exports.py index 30b148bd6..b7cab0165 100644 --- a/application/single_app/functions_assistant_table_exports.py +++ b/application/single_app/functions_assistant_table_exports.py @@ -154,7 +154,6 @@ re.compile(r'\b(?:include|with|using)\s+(?:the\s+)?(?:columns?|fields?)\b'), ) - def assistant_table_export_requested(user_question: str) -> bool: """Return True when the user asked for table-shaped output or a CSV export.""" normalized_question = re.sub(r'\s+', ' ', str(user_question or '').strip().casefold()) @@ -261,20 +260,6 @@ def has_generated_tabular_csv_output(generated_outputs: List[Dict[str, Any]]) -> return False -def get_assistant_csv_export_content(assistant_result: Any) -> str: - """Return the structured document-action reply when it supersedes a concise artifact reply.""" - if not isinstance(assistant_result, dict): - return str(assistant_result or '') - - analysis_result = assistant_result.get('analysis_result') - if isinstance(analysis_result, dict): - analysis_reply = str(analysis_result.get('analysis_reply') or '').strip() - if analysis_reply: - return analysis_reply - - return str(assistant_result.get('reply') or '') - - def extract_assistant_table_entries(assistant_content: str) -> List[Dict[str, str]]: """Extract table rows from Markdown, tab-separated, or CSV assistant output.""" normalized_content = str(assistant_content or '').replace('\r\n', '\n').replace('\r', '\n') @@ -310,14 +295,16 @@ def build_assistant_table_csv(table_rows: List[Dict[str, Any]]) -> str: if not ordered_columns: ordered_columns = ['value'] + safe_columns = build_safe_csv_headers(ordered_columns) + output_buffer = io.StringIO() - writer = csv.DictWriter(output_buffer, fieldnames=ordered_columns, extrasaction='ignore') + writer = csv.DictWriter(output_buffer, fieldnames=safe_columns, extrasaction='ignore') writer.writeheader() for table_row in table_rows or []: serialized_row = {} if isinstance(table_row, dict): - for column_name in ordered_columns: - serialized_row[column_name] = _serialize_table_cell(table_row.get(column_name)) + for source_column, safe_column in zip(ordered_columns, safe_columns): + serialized_row[safe_column] = _serialize_table_cell(table_row.get(source_column)) writer.writerow(serialized_row) return output_buffer.getvalue() diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index 85a0ae5c2..811db3ae4 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -4395,7 +4395,7 @@ def extract_document_metadata(document_id, user_id, group_id=None, public_worksp json.dumps(meta_data), user_id, document_id=document_id, - top_n=25, + top_n=50, doc_scope=document_scope ) elif document_scope == "group": @@ -4403,7 +4403,7 @@ def extract_document_metadata(document_id, user_id, group_id=None, public_worksp json.dumps(meta_data), user_id, document_id=document_id, - top_n=25, + top_n=50, doc_scope=document_scope, active_group_id=scope_id ) @@ -4412,7 +4412,7 @@ def extract_document_metadata(document_id, user_id, group_id=None, public_worksp json.dumps(meta_data), user_id, document_id=document_id, - top_n=25, + top_n=50, doc_scope=document_scope, active_public_workspace_id=scope_id ) diff --git a/application/single_app/functions_generated_file_exports.py b/application/single_app/functions_generated_file_exports.py new file mode 100644 index 000000000..623a90c67 --- /dev/null +++ b/application/single_app/functions_generated_file_exports.py @@ -0,0 +1,609 @@ +# functions_generated_file_exports.py +"""Format-neutral planning and rendering for generated chat file exports.""" + +import html +import io +import json +import os +import re +import tempfile +from datetime import datetime +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from functions_assistant_table_exports import ( + assistant_table_export_requested, + build_assistant_table_csv, + build_csv_output_clarification_guidance, + extract_assistant_table_entries, +) + + +GENERATED_FILE_FORMAT_CSV = 'csv' +GENERATED_FILE_FORMAT_DOCX = 'docx' +GENERATED_FILE_FORMAT_PDF = 'pdf' +GENERATED_FILE_FORMATS = { + GENERATED_FILE_FORMAT_CSV, + GENERATED_FILE_FORMAT_DOCX, + GENERATED_FILE_FORMAT_PDF, +} +GENERATED_FILE_PREVIEW_ROWS = 3 +FUNCTION_RESULT_ROW_KEYS = ( + 'rows', + 'data', + 'items', + 'results', + 'records', + 'value', + 'values', + 'result', + 'body', + 'output', + 'payload', +) +FUNCTION_RESULT_CONTROL_KEYS = { + 'count', + 'detail', + 'error', + 'errormessage', + 'hasmore', + 'message', + 'metadata', + 'meta', + 'nextlink', + 'nextpage', + 'pagination', + 'returnedrows', + 'status', + 'statuscode', + 'success', + 'summary', + 'total', + 'totalcount', + 'totalmatches', +} +FUNCTION_RESULT_SENSITIVE_KEY_FRAGMENTS = ( + 'accesstoken', + 'apikey', + 'authorization', + 'clientsecret', + 'connectionstring', + 'credential', + 'password', + 'privatekey', + 'secret', + 'sharedaccesssignature', + 'subscriptionkey', + 'token', +) +TABULAR_FUNCTION_RESULT_PLUGIN_NAMES = {'tabularprocessingplugin'} +DOCX_OUTPUT_REQUEST_PATTERNS = ( + re.compile( + r'\b(?:build|create|download|export|generate|make|prepare|save|turn|convert)\b' + r'.{0,120}\b(?:a\s+)?(?:word|docx)(?:\s+(?:document|file|output|report))?\b' + ), + re.compile(r'\b(?:word|docx)\s+(?:document|file|output|report|version)\b'), + re.compile(r'\b(?:get|give)\s+(?:me\s+)?(?:a|the|one)\s+(?:word|docx)\b'), + re.compile(r'\b(?:need|want)\s+(?:(?:a|the|one)\s+)?(?:word|docx)\b'), + re.compile(r'\b(?:in|as)\s+(?:a\s+)?(?:word|docx)(?:\s+(?:document|file|report))?\b'), +) +PDF_OUTPUT_REQUEST_PATTERNS = ( + re.compile( + r'\b(?:build|create|download|export|generate|make|prepare|save|turn|convert)\b' + r'.{0,120}\b(?:a\s+)?pdf(?:\s+(?:document|file|output|report))?\b' + ), + re.compile(r'\bpdf\s+(?:document|file|output|report|version)\b'), + re.compile(r'\b(?:get|give)\s+(?:me\s+)?(?:a|the|one)\s+pdf\b'), + re.compile(r'\b(?:need|want)\s+(?:(?:a|the|one)\s+)?pdf\b'), + re.compile(r'\b(?:in|as)\s+(?:a\s+)?pdf(?:\s+(?:document|file|report))?\b'), +) +PDF_EXPORT_CSS = """ +body { font-family: sans-serif; font-size: 10pt; color: #172033; } +h1 { font-size: 20pt; color: #173b5f; margin-bottom: 10pt; } +h2 { font-size: 14pt; color: #173b5f; margin-top: 16pt; } +p { line-height: 1.35; margin-bottom: 8pt; } +table { border-collapse: collapse; width: 100%; margin-top: 8pt; } +th { background-color: #e8eef5; font-weight: bold; } +th, td { border: 0.6pt solid #aab7c4; padding: 4pt; vertical-align: top; } +""" + + +def get_requested_generated_file_format(user_question: str) -> Optional[str]: + """Return the requested generated file format, if any.""" + if assistant_table_export_requested(user_question): + return GENERATED_FILE_FORMAT_CSV + + normalized_question = re.sub(r'\s+', ' ', str(user_question or '').strip().casefold()) + if not normalized_question: + return None + if any(pattern.search(normalized_question) for pattern in DOCX_OUTPUT_REQUEST_PATTERNS): + return GENERATED_FILE_FORMAT_DOCX + if any(pattern.search(normalized_question) for pattern in PDF_OUTPUT_REQUEST_PATTERNS): + return GENERATED_FILE_FORMAT_PDF + return None + + +def generated_file_export_requested(user_question: str) -> bool: + """Return whether the user asked for a supported generated file artifact.""" + return get_requested_generated_file_format(user_question) is not None + + +def build_generated_file_output_guidance(user_question: str) -> str: + """Return shared model guidance for a requested generated output format.""" + output_format = get_requested_generated_file_format(user_question) + if output_format == GENERATED_FILE_FORMAT_CSV: + return build_csv_output_clarification_guidance(user_question) + if output_format in {GENERATED_FILE_FORMAT_DOCX, GENERATED_FILE_FORMAT_PDF}: + return ( + f'The user requested a downloadable {output_format.upper()} artifact. Provide a clear final ' + 'response grounded in the available evidence. Structured function results from this turn may ' + 'be included as labeled tables in the generated file; do not invent rows or claim an attachment ' + 'exists before the file-output finalizer publishes it.' + ) + return '' + + +def get_generated_file_export_content(assistant_result: Any) -> str: + """Return the structured document-action reply when it supersedes a concise artifact reply.""" + if not isinstance(assistant_result, dict): + return str(assistant_result or '') + + analysis_result = assistant_result.get('analysis_result') + if isinstance(analysis_result, dict): + analysis_reply = str(analysis_result.get('analysis_reply') or '').strip() + if analysis_reply: + return analysis_reply + + return str(assistant_result.get('reply') or '') + + +def build_generated_file_export( + user_question: str, + assistant_content: str, + function_results: Optional[List[Dict[str, Any]]] = None, +) -> Optional[Dict[str, Any]]: + """Build a generated file payload from final assistant content and function-result evidence.""" + output_format = get_requested_generated_file_format(user_question) + if output_format not in GENERATED_FILE_FORMATS: + return None + + assistant_text = str(assistant_content or '').strip() + assistant_rows = extract_assistant_table_entries(assistant_text) + function_rows = extract_authorized_function_result_rows(function_results) + + if output_format == GENERATED_FILE_FORMAT_CSV: + rows = assistant_rows or function_rows + if not rows: + return None + row_source = 'assistant response' if assistant_rows else 'structured function result' + return _build_generated_file_payload( + output_format=output_format, + file_content=build_assistant_table_csv(rows), + rows=rows, + row_source=row_source, + assistant_content=assistant_text, + ) + + if not assistant_text and not assistant_rows and not function_rows: + return None + rows = function_rows or assistant_rows + row_source = 'structured function result' if function_rows else 'assistant response' + title = _build_generated_file_title(output_format) + if output_format == GENERATED_FILE_FORMAT_DOCX: + file_content = _render_docx_file_export(title, assistant_text, rows, row_source) + else: + file_content = _render_pdf_file_export(title, assistant_text, rows, row_source) + return _build_generated_file_payload( + output_format=output_format, + file_content=file_content, + rows=rows, + row_source=row_source, + assistant_content=assistant_text, + title=title, + ) + + +def extract_authorized_function_result_rows(function_results: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: + """Return structured rows from successful current-turn non-tabular function results.""" + function_row_groups: List[Tuple[str, List[Dict[str, Any]]]] = [] + for function_result in function_results or []: + if not isinstance(function_result, dict): + continue + if function_result.get('success') is False or _is_tabular_function_result(function_result): + continue + + structured_rows = _extract_function_result_rows( + _parse_function_result_payload(function_result.get('function_result')), + ) + if not structured_rows: + continue + function_row_groups.append(( + _get_function_result_label(function_result), + structured_rows, + )) + + if not function_row_groups: + return [] + if len(function_row_groups) == 1: + return function_row_groups[0][1] + + source_column = _get_function_result_source_column(function_row_groups) + combined_rows = [] + for function_label, rows in function_row_groups: + for row in rows: + normalized_row = dict(row) + normalized_row[source_column] = function_label + combined_rows.append(normalized_row) + return combined_rows + + +def has_generated_file_output(existing_outputs: Optional[List[Dict[str, Any]]], output_format: str) -> bool: + """Return whether an existing generated artifact already covers an output format.""" + normalized_output_format = str(output_format or '').strip().lower() + if not normalized_output_format: + return False + for output in existing_outputs or []: + if not isinstance(output, dict): + continue + existing_output_format = str(output.get('output_format') or '').strip().lower() + existing_file_name = str(output.get('file_name') or '').strip().lower() + if existing_output_format == normalized_output_format or existing_file_name.endswith(f'.{normalized_output_format}'): + return True + return False + + +def build_generated_file_artifact_metadata( + export_payload: Dict[str, Any], + upload_result: Dict[str, Any], + conversation_id: str, +) -> Optional[Dict[str, Any]]: + """Build public artifact metadata after an authorized generated-file upload.""" + uploaded_message = upload_result.get('message') if isinstance(upload_result, dict) else {} + uploaded_message = uploaded_message if isinstance(uploaded_message, dict) else {} + artifact_message_id = str(uploaded_message.get('id') or '').strip() + if not artifact_message_id: + return None + + generated_file_name = str(export_payload.get('file_name') or '').strip() + artifact_metadata = { + 'capability': str(export_payload.get('capability') or 'file_export').strip().lower() or 'file_export', + 'artifact_message_id': artifact_message_id, + 'conversation_id': str(conversation_id or '').strip(), + 'storage_scope': 'chat', + 'file_name': uploaded_message.get('file_name') or generated_file_name, + 'output_format': str(export_payload.get('output_format') or '').strip().lower(), + 'summary': str(export_payload.get('summary') or '').strip(), + } + row_count = export_payload.get('row_count') + if isinstance(row_count, int) and row_count > 0: + artifact_metadata['row_count'] = row_count + preview_rows = export_payload.get('preview_rows') + if isinstance(preview_rows, list) and preview_rows: + artifact_metadata['preview_rows'] = preview_rows + preview_lines = export_payload.get('preview_lines') + if isinstance(preview_lines, list) and preview_lines: + artifact_metadata['preview_lines'] = preview_lines + row_source = str(export_payload.get('row_source') or '').strip() + if row_source: + artifact_metadata['row_source'] = row_source + return artifact_metadata + + +def _build_generated_file_payload( + output_format: str, + file_content: Any, + rows: Sequence[Dict[str, Any]], + row_source: str, + assistant_content: str, + title: str = '', +) -> Dict[str, Any]: + normalized_output_format = str(output_format or '').strip().lower() + row_count = len(rows or []) + normalized_title = str(title or _build_generated_file_title(normalized_output_format)).strip() + return { + 'capability': 'file_export', + 'file_name': _build_generated_file_name(normalized_output_format), + 'file_content': file_content, + 'output_format': normalized_output_format, + 'row_count': row_count, + 'preview_rows': list(rows or [])[:GENERATED_FILE_PREVIEW_ROWS], + 'preview_lines': _build_preview_lines(assistant_content), + 'row_source': row_source, + '_structured_rows': list(rows or []), + 'summary': _build_generated_file_summary( + normalized_output_format, + row_count, + row_source, + normalized_title, + ), + } + + +def _build_generated_file_name(output_format: str) -> str: + timestamp_suffix = datetime.utcnow().strftime('%Y%m%d_%H%M%S') + return f'generated_output_{timestamp_suffix}.{output_format}' + + +def _build_generated_file_title(output_format: str) -> str: + return f'Generated {str(output_format or "file").upper()} export' + + +def _build_generated_file_summary( + output_format: str, + row_count: int, + row_source: str, + title: str, +) -> str: + row_detail = f' with {row_count} structured row(s)' if row_count else '' + return f'Prepared {title}{row_detail} from the {row_source}.' + + +def _build_preview_lines(assistant_content: str) -> List[str]: + normalized_lines = [ + line.strip() + for line in str(assistant_content or '').splitlines() + if line.strip() + ] + return normalized_lines[:3] + + +def _parse_function_result_payload(value: Any) -> Any: + if not isinstance(value, str): + return value + + normalized_value = value.strip() + if not normalized_value or normalized_value[0] not in '[{': + return value + try: + return json.loads(normalized_value) + except (TypeError, ValueError, json.JSONDecodeError): + return value + + +def _extract_function_result_rows( + payload: Any, + depth: int = 0, + is_data_row: bool = False, +) -> List[Dict[str, Any]]: + if depth > 4: + return [] + if isinstance(payload, list): + rows = [] + for item in payload: + if isinstance(item, dict): + normalized_row = _normalize_function_result_row(item, is_data_row=True) + if normalized_row: + rows.append(normalized_row) + elif item not in (None, ''): + rows.append({'value': _sanitize_function_result_value(item)}) + return rows + if not isinstance(payload, dict): + return [] + + normalized_keys = { + _normalize_function_result_key(key): key + for key in payload + } + for row_key in FUNCTION_RESULT_ROW_KEYS: + matching_key = normalized_keys.get(_normalize_function_result_key(row_key)) + if matching_key is None: + continue + rows = _extract_function_result_rows( + payload.get(matching_key), + depth + 1, + is_data_row=True, + ) + if rows: + return rows + + normalized_row = _normalize_function_result_row(payload, is_data_row=is_data_row) + return [normalized_row] if normalized_row else [] + + +def _normalize_function_result_row(row: Dict[str, Any], is_data_row: bool) -> Dict[str, Any]: + normalized_row = {} + for raw_key, raw_value in row.items(): + key = str(raw_key or '').strip() + normalized_key = _normalize_function_result_key(key) + if not key or _is_sensitive_function_result_key(key): + continue + if not is_data_row and normalized_key in FUNCTION_RESULT_CONTROL_KEYS: + continue + + value = _sanitize_function_result_value(raw_value) + if value in (None, '', [], {}) or value == '***REDACTED***': + continue + normalized_row[key] = value + return normalized_row + + +def _sanitize_function_result_value(value: Any, depth: int = 0) -> Any: + if depth > 4: + return '[truncated]' + if isinstance(value, dict): + return { + str(key): _sanitize_function_result_value(item, depth + 1) + for key, item in value.items() + if not _is_sensitive_function_result_key(key) + } + if isinstance(value, (list, tuple, set)): + return [ + _sanitize_function_result_value(item, depth + 1) + for item in value + ] + return value + + +def _normalize_function_result_key(key: Any) -> str: + return re.sub(r'[^a-z0-9]', '', str(key or '').casefold()) + + +def _is_sensitive_function_result_key(key: Any) -> bool: + normalized_key = _normalize_function_result_key(key) + if not normalized_key: + return False + return any(fragment in normalized_key for fragment in FUNCTION_RESULT_SENSITIVE_KEY_FRAGMENTS) + + +def _is_tabular_function_result(function_result: Dict[str, Any]) -> bool: + plugin_name = str(function_result.get('plugin_name') or '').strip().casefold() + return plugin_name in TABULAR_FUNCTION_RESULT_PLUGIN_NAMES + + +def _get_function_result_label(function_result: Dict[str, Any]) -> str: + return ( + str(function_result.get('function_name') or '').strip() + or str(function_result.get('plugin_name') or '').strip() + or 'function result' + ) + + +def _get_function_result_source_column( + function_row_groups: Sequence[Tuple[str, Sequence[Dict[str, Any]]]], +) -> str: + existing_columns = { + str(column_name).casefold() + for _, rows in function_row_groups + for row in rows + for column_name in row + } + source_column = 'Source action' + suffix = 2 + while source_column.casefold() in existing_columns: + source_column = f'Source action {suffix}' + suffix += 1 + return source_column + + +def _render_docx_file_export( + title: str, + assistant_content: str, + rows: Sequence[Dict[str, Any]], + row_source: str, +) -> bytes: + from docx import Document as DocxDocument + + document = DocxDocument() + document.add_heading(title, level=1) + _append_docx_text(document, assistant_content) + if rows: + document.add_heading(_build_structured_rows_heading(row_source), level=2) + _append_docx_table(document, rows) + + output_buffer = io.BytesIO() + document.save(output_buffer) + return output_buffer.getvalue() + + +def _append_docx_text(document: Any, assistant_content: str) -> None: + normalized_content = str(assistant_content or '').strip() + if not normalized_content: + return + for paragraph_text in re.split(r'\n\s*\n', normalized_content): + cleaned_paragraph = paragraph_text.strip() + if cleaned_paragraph: + document.add_paragraph(cleaned_paragraph) + + +def _append_docx_table(document: Any, rows: Sequence[Dict[str, Any]]) -> None: + columns = _collect_structured_row_columns(rows) + if not columns: + return + table = document.add_table(rows=1, cols=len(columns)) + table.style = 'Table Grid' + for index, column_name in enumerate(columns): + table.rows[0].cells[index].text = str(column_name) + for row in rows: + cells = table.add_row().cells + for index, column_name in enumerate(columns): + cells[index].text = _format_structured_cell(row.get(column_name)) + + +def _render_pdf_file_export( + title: str, + assistant_content: str, + rows: Sequence[Dict[str, Any]], + row_source: str, +) -> bytes: + import fitz + + html_parts = [f'

{html.escape(title)}

'] + normalized_content = str(assistant_content or '').strip() + if normalized_content: + html_parts.append('

Response

') + for paragraph_text in re.split(r'\n\s*\n', normalized_content): + cleaned_paragraph = paragraph_text.strip() + if cleaned_paragraph: + html_parts.append(f'

{html.escape(cleaned_paragraph).replace(chr(10), "
")}

') + if rows: + html_parts.append(f'

{html.escape(_build_structured_rows_heading(row_source))}

') + html_parts.append(_build_structured_rows_html(rows)) + + media_box = fitz.paper_rect('letter') + content_box = media_box + (36, 36, -36, -36) + story = fitz.Story(html='\n'.join(html_parts), user_css=PDF_EXPORT_CSS) + temporary_path = None + try: + with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as temporary_file: + temporary_path = temporary_file.name + + writer = fitz.DocumentWriter(temporary_path) + has_more = True + while has_more: + device = writer.begin_page(media_box) + has_more, _ = story.place(content_box) + story.draw(device) + writer.end_page() + writer.close() + with open(temporary_path, 'rb') as generated_file: + return generated_file.read() + finally: + if temporary_path: + try: + os.unlink(temporary_path) + except OSError: + pass + + +def _build_structured_rows_html(rows: Sequence[Dict[str, Any]]) -> str: + columns = _collect_structured_row_columns(rows) + if not columns: + return '

No structured rows were available.

' + table_parts = [''] + table_parts.extend(f'' for column_name in columns) + table_parts.append('') + for row in rows: + table_parts.append('') + for column_name in columns: + table_parts.append(f'') + table_parts.append('') + table_parts.append('
{html.escape(str(column_name))}
{html.escape(_format_structured_cell(row.get(column_name))).replace(chr(10), "
")}
') + return ''.join(table_parts) + + +def _collect_structured_row_columns(rows: Sequence[Dict[str, Any]]) -> List[str]: + columns = [] + seen_columns = set() + for row in rows or []: + if not isinstance(row, dict): + continue + for raw_column_name in row: + column_name = str(raw_column_name or '').strip() + if not column_name or column_name.casefold() in seen_columns: + continue + seen_columns.add(column_name.casefold()) + columns.append(column_name) + return columns + + +def _format_structured_cell(value: Any) -> str: + if value is None: + return '' + if isinstance(value, (dict, list, tuple, set)): + return json.dumps(value, default=str, ensure_ascii=False) + return str(value) + + +def _build_structured_rows_heading(row_source: str) -> str: + if row_source == 'structured function result': + return 'Structured function results' + return 'Structured response rows' diff --git a/application/single_app/functions_group.py b/application/single_app/functions_group.py index 94ce8a075..df59c1c01 100644 --- a/application/single_app/functions_group.py +++ b/application/single_app/functions_group.py @@ -101,7 +101,7 @@ def search_all_groups(search_query, limit=10): parameters=params, enable_cross_partition_query=True )) - return results[:max(1, min(int(limit or 10), 25))] + return results[:max(1, min(int(limit or 10), 50))] def get_user_groups(user_id): """ diff --git a/application/single_app/functions_search.py b/application/single_app/functions_search.py index 34b7a7e02..8b8bc38da 100644 --- a/application/single_app/functions_search.py +++ b/application/single_app/functions_search.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) -SEARCH_DEFAULT_TOP_N = 25 +SEARCH_DEFAULT_TOP_N = 50 SEARCH_MAX_TOP_N = 500 VALID_SEARCH_SCOPES = {"all", "personal", "group", "public"} BASE_SEARCH_SELECT_FIELDS = [ @@ -259,7 +259,7 @@ def _build_odata_any_eq(collection_field: str, iterator_name: str, value: Any) - escaped_value = _escape_odata_literal(value) return f"{collection_field}/any({iterator_name}: {iterator_name} eq '{escaped_value}')" -def hybrid_search(query, user_id, document_id=None, document_ids=None, top_n=25, doc_scope="all", active_group_id=None, active_group_ids=None, active_public_workspace_id=None, enable_file_sharing=True, tags_filter=None, document_filter_mode="intersection", enforce_public_workspace_visibility=True): +def hybrid_search(query, user_id, document_id=None, document_ids=None, top_n=50, doc_scope="all", active_group_id=None, active_group_ids=None, active_public_workspace_id=None, enable_file_sharing=True, tags_filter=None, document_filter_mode="intersection", enforce_public_workspace_visibility=True): """ Hybrid search that queries the user doc index, group doc index, or public doc index depending on doc type. diff --git a/application/single_app/functions_search_service.py b/application/single_app/functions_search_service.py index 17e116947..2dc18b832 100644 --- a/application/single_app/functions_search_service.py +++ b/application/single_app/functions_search_service.py @@ -45,8 +45,8 @@ SUMMARY_MAX_WINDOW_SIZE = 50 CHAT_UPLOAD_CHUNK_WORD_SIZE = 400 CHAT_UPLOAD_CHUNK_WORD_OVERLAP = 40 -MIXED_SOURCE_TABULAR_CANDIDATE_TOP_N = 36 -MIXED_SOURCE_TABULAR_CANDIDATE_LIMIT = 6 +MIXED_SOURCE_TABULAR_CANDIDATE_TOP_N = 50 +MIXED_SOURCE_TABULAR_CANDIDATE_LIMIT = 100 MIXED_SOURCE_TABULAR_EXTENSIONS = frozenset({".csv", ".xls", ".xlsx", ".xlsm"}) diff --git a/application/single_app/functions_simplechat_operations.py b/application/single_app/functions_simplechat_operations.py index e7dd3c368..e89a24e6f 100644 --- a/application/single_app/functions_simplechat_operations.py +++ b/application/single_app/functions_simplechat_operations.py @@ -1343,7 +1343,7 @@ def search_directory_users(query: str, limit: int = 10) -> List[Dict[str, str]]: f"or startswith(mail, '{escaped_query}') " f"or startswith(userPrincipalName, '{escaped_query}')" ), - "$top": max(1, min(int(limit or 10), 25)), + "$top": max(1, min(int(limit or 10), 50)), "$select": "id,displayName,mail,userPrincipalName", }, ) diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 30b874926..701309d8b 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -46,14 +46,18 @@ from functions_activity_logging import log_conversation_creation, log_token_usage, log_workflow_run from functions_appinsights import log_event from functions_assistant_table_exports import ( - build_assistant_table_csv_export, - build_csv_output_clarification_guidance, build_safe_csv_headers, - extract_assistant_table_entries, - get_assistant_csv_export_content, has_generated_tabular_csv_output, neutralize_csv_spreadsheet_formula, ) +from functions_generated_file_exports import ( + build_generated_file_artifact_metadata, + build_generated_file_export, + build_generated_file_output_guidance, + get_generated_file_export_content, + get_requested_generated_file_format, + has_generated_file_output, +) from functions_chart_operations import append_proactive_chart_guidance from functions_collaboration import ( create_collaboration_message_notifications, @@ -2636,9 +2640,9 @@ def _build_workflow_chat_messages(prompt_text, url_access_context=None, apply_ge source_review_content = _get_workflow_url_access_system_content(url_access_context) if source_review_content: messages.append({'role': 'system', 'content': source_review_content}) - csv_output_guidance = build_csv_output_clarification_guidance(prompt_text) - if csv_output_guidance: - messages.append({'role': 'system', 'content': csv_output_guidance}) + generated_file_output_guidance = build_generated_file_output_guidance(prompt_text) + if generated_file_output_guidance: + messages.append({'role': 'system', 'content': generated_file_output_guidance}) messages.append({'role': 'user', 'content': user_content}) return messages @@ -2646,10 +2650,10 @@ def _build_workflow_chat_messages(prompt_text, url_access_context=None, apply_ge def _build_workflow_agent_messages(prompt_text, url_access_context=None, apply_generation_guidance=False): user_content = _build_workflow_generation_prompt(prompt_text) if apply_generation_guidance else str(prompt_text or '').strip() source_review_content = _get_workflow_url_access_system_content(url_access_context) - csv_output_guidance = build_csv_output_clarification_guidance(prompt_text) + generated_file_output_guidance = build_generated_file_output_guidance(prompt_text) content_sections = [ content - for content in (csv_output_guidance, source_review_content) + for content in (generated_file_output_guidance, source_review_content) if content ] content_sections.append(f'[Workflow Task]\n{user_content}') @@ -4347,18 +4351,28 @@ def _add_workflow_activity_thought( ) -def _maybe_create_workflow_assistant_table_generated_output( +def _maybe_create_workflow_generated_file_output( workflow, conversation_id, user_question, assistant_content, + function_results=None, existing_outputs=None, ): - """Persist a workflow CSV artifact from a valid structured assistant response.""" - if has_generated_tabular_csv_output(existing_outputs): + """Persist a requested workflow CSV, DOCX, or PDF artifact.""" + output_format = get_requested_generated_file_format(user_question) + if not output_format: + return None + if output_format == 'csv' and has_generated_tabular_csv_output(existing_outputs): + return None + if has_generated_file_output(existing_outputs, output_format): return None - export_payload = build_assistant_table_csv_export(user_question, assistant_content) + export_payload = build_generated_file_export( + user_question, + assistant_content, + function_results=function_results, + ) if not export_payload: return None @@ -4370,13 +4384,19 @@ def _maybe_create_workflow_assistant_table_generated_output( generated_file_name = str(export_payload.get('file_name') or '').strip() row_count = int(export_payload.get('row_count') or 0) - table_rows = extract_assistant_table_entries(assistant_content) settings = get_settings() - row_batches = build_tabular_generated_output_row_batches(table_rows, settings=settings) - if not generated_file_name or row_count <= 0 or not row_batches: + structured_rows = export_payload.get('_structured_rows') or [] + row_batches = [] + if output_format == 'csv': + row_batches = build_tabular_generated_output_row_batches(structured_rows, settings=settings) + if not generated_file_name: return None - if should_queue_tabular_generated_output_background(row_count, len(row_batches), settings): + if output_format == 'csv' and should_queue_tabular_generated_output_background( + row_count, + len(row_batches), + settings, + ): try: background_run = queue_tabular_generated_output_run( user_id=user_id, @@ -4389,7 +4409,7 @@ def _maybe_create_workflow_assistant_table_generated_output( 'source': 'chat', }, }, - output_format='csv', + output_format=output_format, row_batches=row_batches, gpt_model='', settings=settings, @@ -4398,11 +4418,12 @@ def _maybe_create_workflow_assistant_table_generated_output( return build_background_tabular_generated_output_metadata(background_run) except Exception as exc: log_event( - '[Workflow Assistant Table Export] Failed to queue large CSV export', + '[Workflow Generated File Export] Failed to queue large CSV export', { 'workflow_id': normalized_workflow.get('id'), 'conversation_id': normalized_conversation_id, 'row_count': row_count, + 'output_format': output_format, 'error': str(exc), }, level=logging.ERROR, @@ -4416,50 +4437,49 @@ def _maybe_create_workflow_assistant_table_generated_output( conversation_id=normalized_conversation_id, file_name=generated_file_name, file_content=export_payload.get('file_content'), - capability='tabular', - output_format='csv', + capability=export_payload.get('capability') or 'file_export', + output_format=output_format, summary=export_payload.get('summary'), ) except Exception as exc: log_event( - '[Workflow Assistant Table Export] Failed to save assistant table CSV artifact', + '[Workflow Generated File Export] Failed to save generated file artifact', { 'workflow_id': normalized_workflow.get('id'), 'conversation_id': normalized_conversation_id, 'row_count': row_count, + 'output_format': output_format, 'error': str(exc), }, debug_only=True, ) return None - uploaded_message = upload_result.get('message') or {} - artifact_message_id = uploaded_message.get('id') - if not artifact_message_id: + artifact_metadata = build_generated_file_artifact_metadata( + export_payload, + upload_result, + normalized_conversation_id, + ) + if not artifact_metadata: return None - uploaded_file_name = uploaded_message.get('file_name') or generated_file_name log_event( - '[Workflow Assistant Table Export] Saved assistant table CSV artifact', + '[Workflow Generated File Export] Saved generated file artifact', { 'workflow_id': normalized_workflow.get('id'), 'conversation_id': normalized_conversation_id, - 'artifact_message_id': artifact_message_id, + 'artifact_message_id': artifact_metadata.get('artifact_message_id'), 'row_count': row_count, + 'output_format': output_format, }, debug_only=True, ) - return { - 'capability': 'tabular', - 'artifact_message_id': artifact_message_id, - 'conversation_id': normalized_conversation_id, - 'storage_scope': 'chat', - 'file_name': uploaded_file_name, - 'output_format': 'csv', - 'row_count': row_count, - 'preview_rows': export_payload.get('preview_rows') or [], - 'summary': export_payload.get('summary'), - } + return artifact_metadata + + +def _maybe_create_workflow_assistant_table_generated_output(*args, **kwargs): + """Backward-compatible wrapper for the generic workflow file-output finalizer.""" + return _maybe_create_workflow_generated_file_output(*args, **kwargs) def _create_assistant_message(conversation, workflow, result, trigger_source, run_id, user_message_doc, assistant_message_id=None): @@ -4471,17 +4491,19 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru group_id = _get_workflow_group_id(workflow) generated_analysis_artifacts = list(result.get('generated_analysis_artifacts') or []) generated_tabular_outputs = list(result.get('generated_tabular_outputs') or []) - assistant_table_generated_output = _maybe_create_workflow_assistant_table_generated_output( + raw_agent_citations = list(result.get('agent_citations') or []) + generated_file_output = _maybe_create_workflow_generated_file_output( workflow=workflow, conversation_id=conversation.get('id'), user_question=workflow.get('task_prompt', ''), - assistant_content=get_assistant_csv_export_content(result), + assistant_content=get_generated_file_export_content(result), + function_results=raw_agent_citations, existing_outputs=generated_analysis_artifacts + generated_tabular_outputs, ) - if assistant_table_generated_output: - generated_analysis_artifacts.append(assistant_table_generated_output) - generated_tabular_outputs.append(assistant_table_generated_output) - raw_agent_citations = list(result.get('agent_citations') or []) + if generated_file_output: + generated_analysis_artifacts.append(generated_file_output) + if generated_file_output.get('output_format') == 'csv': + generated_tabular_outputs.append(generated_file_output) web_search_citations = list(result.get('web_search_citations') or []) source_review_metadata = result.get('source_review') if isinstance(result.get('source_review'), dict) else {} url_access_metadata = result.get('url_access') if isinstance(result.get('url_access'), dict) else {} @@ -5218,7 +5240,7 @@ def _prepare_workflow_search_context( scoped_action['active_public_workspace_id'] = manifest_public_workspace_ids if not is_mixed_source_chat_search_enabled(settings): - search_top_n = normalize_search_top_n(max(12, len(document_ids) * 3 if document_ids else 12)) + search_top_n = normalize_search_top_n(max(50, len(document_ids) * 3 if document_ids else 50)) search_result = search_documents( query=query, user_id=user_id, @@ -5297,7 +5319,7 @@ def _prepare_workflow_search_context( manifest_public_workspace_ids.append(public_workspace_id) search_top_n = normalize_search_top_n( - max(12, len(narrative_document_ids) * 3 if narrative_document_ids else 12) + max(50, len(narrative_document_ids) * 3 if narrative_document_ids else 50) ) search_result = { 'results': [], diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 529572201..2b1d9f737 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -120,14 +120,18 @@ from functions_content import generate_embedding, generate_embeddings_batch from functions_assistant_table_exports import ( assistant_table_export_requested, - build_csv_output_clarification_guidance, build_safe_csv_headers, - build_assistant_table_csv_export, - extract_assistant_table_entries, - get_assistant_csv_export_content, has_generated_tabular_csv_output, neutralize_csv_spreadsheet_formula, ) +from functions_generated_file_exports import ( + build_generated_file_artifact_metadata, + build_generated_file_export, + build_generated_file_output_guidance, + get_generated_file_export_content, + get_requested_generated_file_format, + has_generated_file_output, +) from functions_chart_operations import ( CORE_CHART_PLUGIN_NAME, INLINE_CHART_BLOCK_LANGUAGE, @@ -207,7 +211,7 @@ DOCUMENT_ACTION_TYPE_ANALYZE: ASSIGNED_KNOWLEDGE_USER_ACTION_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON: ASSIGNED_KNOWLEDGE_USER_ACTION_COMPARE, } -ASSIGNED_KNOWLEDGE_CONTEXT_TOP_N = 12 +ASSIGNED_KNOWLEDGE_CONTEXT_TOP_N = 50 ASSIGNED_KNOWLEDGE_CONTEXT_EXCERPT_MAX_CHARS = 1800 MIXED_SOURCE_CHAT_RELEVANCE_SOURCE_LIMIT = 48 FOUNDRY_SELECTED_AGENT_TYPES = {'aifoundry', 'new_foundry', 'foundry_workflow'} @@ -2050,36 +2054,54 @@ def _has_generated_tabular_csv_output(generated_outputs): return has_generated_tabular_csv_output(generated_outputs) -def maybe_create_assistant_table_generated_output( +def maybe_create_generated_file_output( user_question, assistant_content, conversation_id, + function_results=None, existing_outputs=None, cancel_requested=None, request_correlation_id=None, ): - """Save a CSV artifact when a table-request answer contains a parseable table.""" + """Save a requested CSV, DOCX, or PDF artifact from response and action evidence.""" raise_if_mixed_source_cancelled( cancel_requested, 'artifact_publication', request_correlation_id=request_correlation_id, ) - if _has_generated_tabular_csv_output(existing_outputs): + output_format = get_requested_generated_file_format(user_question) + if not output_format: + return None + if output_format == 'csv' and _has_generated_tabular_csv_output(existing_outputs): + return None + if has_generated_file_output(existing_outputs, output_format): return None - export_payload = build_assistant_table_csv_export(user_question, assistant_content) + export_payload = build_generated_file_export( + user_question, + assistant_content, + function_results=function_results, + ) if not export_payload: return None - generated_file_name = export_payload.get('file_name') + generated_file_name = str(export_payload.get('file_name') or '').strip() + if not generated_file_name: + return None row_count = _safe_int(export_payload.get('row_count')) settings = get_settings() - table_rows = extract_assistant_table_entries(assistant_content) - row_batches = _build_tabular_generated_output_row_batches( - table_rows, - settings=settings, - ) - if should_queue_tabular_generated_output_background(row_count, len(row_batches), settings): + structured_rows = export_payload.get('_structured_rows') or [] + row_batches = [] + if output_format == 'csv': + row_batches = _build_tabular_generated_output_row_batches( + structured_rows, + settings=settings, + ) + if output_format == 'csv' and should_queue_tabular_generated_output_background( + row_count, + len(row_batches), + settings, + ): try: background_run = queue_tabular_generated_output_run( user_id=get_current_user_id(), @@ -2092,7 +2114,7 @@ def maybe_create_assistant_table_generated_output( 'source': 'chat', }, }, - output_format='csv', + output_format=output_format, row_batches=row_batches, gpt_model='', settings=settings, @@ -2116,11 +2138,12 @@ def maybe_create_assistant_table_generated_output( raise except Exception as exc: log_event( - '[Assistant Table Export] Failed to queue large CSV export', + '[Generated File Export] Failed to queue large CSV export', { 'conversation_id': conversation_id, 'generated_file_name': generated_file_name, 'row_count': row_count, + 'output_format': output_format, 'error': str(exc), }, level=logging.ERROR, @@ -2138,8 +2161,8 @@ def maybe_create_assistant_table_generated_output( conversation_id=conversation_id, file_name=generated_file_name, file_content=export_payload.get('file_content'), - capability='tabular', - output_format='csv', + capability=export_payload.get('capability') or 'file_export', + output_format=output_format, summary=export_payload.get('summary'), ) try: @@ -2158,43 +2181,43 @@ def maybe_create_assistant_table_generated_output( raise except Exception as exc: log_event( - '[Assistant Table Export] Failed to save assistant table CSV artifact', + '[Generated File Export] Failed to save generated file artifact', { 'conversation_id': conversation_id, 'generated_file_name': generated_file_name, 'row_count': row_count, + 'output_format': output_format, 'error': str(exc), }, debug_only=True, ) return None - artifact_message_id = upload_result.get('message', {}).get('id') - if not artifact_message_id: + artifact_metadata = build_generated_file_artifact_metadata( + export_payload, + upload_result, + conversation_id, + ) + if not artifact_metadata: return None - uploaded_file_name = upload_result.get('message', {}).get('file_name') or generated_file_name log_event( - '[Assistant Table Export] Saved assistant table CSV artifact', + '[Generated File Export] Saved generated file artifact', { 'conversation_id': conversation_id, - 'artifact_message_id': artifact_message_id, - 'generated_file_name': uploaded_file_name, + 'artifact_message_id': artifact_metadata.get('artifact_message_id'), + 'generated_file_name': artifact_metadata.get('file_name'), 'row_count': row_count, + 'output_format': output_format, }, debug_only=True, ) - return { - 'capability': 'tabular', - 'artifact_message_id': artifact_message_id, - 'conversation_id': conversation_id, - 'storage_scope': 'chat', - 'file_name': uploaded_file_name, - 'output_format': 'csv', - 'row_count': row_count, - 'preview_rows': export_payload.get('preview_rows') or [], - 'summary': export_payload.get('summary'), - } + return artifact_metadata + + +def maybe_create_assistant_table_generated_output(*args, **kwargs): + """Backward-compatible wrapper for the generic generated-file finalizer.""" + return maybe_create_generated_file_output(*args, **kwargs) def _safe_int(value, default=0): @@ -4292,7 +4315,7 @@ def _resolve_tabular_related_document_evidence(document_match, user_question, us search_payload = search_documents( query=search_query, user_id=user_id, - top_n=2, + top_n=10, doc_scope=doc_scope, document_ids=[document_id], active_group_ids=active_group_ids, @@ -13815,17 +13838,19 @@ def execute_document_action_chat_request( 'artifact_publication', request_correlation_id=request_correlation_id, ) - assistant_table_generated_output = maybe_create_assistant_table_generated_output( + generated_file_output = maybe_create_generated_file_output( user_question=user_message, - assistant_content=get_assistant_csv_export_content(execution_result), + assistant_content=get_generated_file_export_content(execution_result), conversation_id=conversation_id, + function_results=execution_result.get('agent_citations') or [], existing_outputs=document_generated_analysis_artifacts + document_generated_tabular_outputs, cancel_requested=cancel_requested, request_correlation_id=request_correlation_id, ) - if assistant_table_generated_output: - document_generated_analysis_artifacts.append(assistant_table_generated_output) - document_generated_tabular_outputs.append(assistant_table_generated_output) + if generated_file_output: + document_generated_analysis_artifacts.append(generated_file_output) + if generated_file_output.get('output_format') == 'csv': + document_generated_tabular_outputs.append(generated_file_output) _reauthorize_document_action_finalization( normalized_action, execution_result, @@ -14468,11 +14493,11 @@ def result_requires_message_reload(result: Any) -> bool: generated_tabular_outputs_list = [] generated_analysis_artifacts_list = [] system_messages_for_augmentation = [] # Collect system messages from search - csv_output_guidance = build_csv_output_clarification_guidance(user_message) - if csv_output_guidance: + generated_file_output_guidance = build_generated_file_output_guidance(user_message) + if generated_file_output_guidance: system_messages_for_augmentation.append({ 'role': 'system', - 'content': csv_output_guidance, + 'content': generated_file_output_guidance, }) search_results = [] mixed_source_narrative_retrieval_failed = False @@ -17995,15 +18020,17 @@ def gpt_error(e): created_timestamp=assistant_timestamp, user_info=user_info_for_assistant, ) - assistant_table_generated_output = maybe_create_assistant_table_generated_output( + generated_file_output = maybe_create_generated_file_output( user_question=user_message, assistant_content=ai_message, conversation_id=conversation_id, + function_results=agent_citations_list, existing_outputs=generated_analysis_artifacts_list + generated_tabular_outputs_list, ) - if assistant_table_generated_output: - generated_analysis_artifacts_list.append(assistant_table_generated_output) - generated_tabular_outputs_list.append(assistant_table_generated_output) + if generated_file_output: + generated_analysis_artifacts_list.append(generated_file_output) + if generated_file_output.get('output_format') == 'csv': + generated_tabular_outputs_list.append(generated_file_output) generated_analysis_metadata = _build_generated_analysis_metadata( generated_analysis_artifacts=generated_analysis_artifacts_list, generated_tabular_outputs=generated_tabular_outputs_list, @@ -18594,11 +18621,11 @@ def stream_cancel_requested(): generated_tabular_outputs_list = [] generated_analysis_artifacts_list = [] system_messages_for_augmentation = [] - csv_output_guidance = build_csv_output_clarification_guidance(user_message) - if csv_output_guidance: + generated_file_output_guidance = build_generated_file_output_guidance(user_message) + if generated_file_output_guidance: system_messages_for_augmentation.append({ 'role': 'system', - 'content': csv_output_guidance, + 'content': generated_file_output_guidance, }) search_results = [] mixed_source_narrative_retrieval_failed = False @@ -19760,7 +19787,7 @@ def record_and_publish_streaming_thought(thought_payload): search_args = { "query": search_query, "user_id": user_id, - "top_n": 12, + "top_n": 50, "doc_scope": effective_document_scope, } @@ -21614,17 +21641,19 @@ def finalize_cancelled_agent_stream_response(): 'artifact_publication', request_correlation_id=mixed_source_request_correlation_id, ) - assistant_table_generated_output = maybe_create_assistant_table_generated_output( + generated_file_output = maybe_create_generated_file_output( user_question=user_message, assistant_content=accumulated_content, conversation_id=conversation_id, + function_results=agent_citations_list, existing_outputs=generated_analysis_artifacts_list + generated_tabular_outputs_list, cancel_requested=stream_cancel_requested, request_correlation_id=mixed_source_request_correlation_id, ) - if assistant_table_generated_output: - generated_analysis_artifacts_list.append(assistant_table_generated_output) - generated_tabular_outputs_list.append(assistant_table_generated_output) + if generated_file_output: + generated_analysis_artifacts_list.append(generated_file_output) + if generated_file_output.get('output_format') == 'csv': + generated_tabular_outputs_list.append(generated_file_output) if mixed_source_manifest: fresh_finalization_manifest = resolve_authorized_source_manifest( [source.get('document_id') for source in mixed_source_manifest], diff --git a/application/single_app/static/json/schemas/document_search_plugin.additional_settings.schema.json b/application/single_app/static/json/schemas/document_search_plugin.additional_settings.schema.json index d147ddd5f..a8c8a3384 100644 --- a/application/single_app/static/json/schemas/document_search_plugin.additional_settings.schema.json +++ b/application/single_app/static/json/schemas/document_search_plugin.additional_settings.schema.json @@ -17,7 +17,7 @@ "type": "integer", "minimum": 1, "maximum": 500, - "default": 12 + "default": 50 }, "default_window_unit": { "type": "string", diff --git a/application/single_app/utils_cache.py b/application/single_app/utils_cache.py index fcd46fdca..230f247d6 100644 --- a/application/single_app/utils_cache.py +++ b/application/single_app/utils_cache.py @@ -303,7 +303,7 @@ def generate_search_cache_key( active_group_id: Optional[str] = None, active_group_ids: Optional[List[str]] = None, active_public_workspace_id: Optional[str] = None, - top_n: int = 25, + top_n: int = 50, enable_file_sharing: bool = True, tags_filter: Optional[List[str]] = None, document_filter_mode: str = "intersection" diff --git a/docs/explanation/features/GENERATED_FILE_EXPORT_FRAMEWORK.md b/docs/explanation/features/GENERATED_FILE_EXPORT_FRAMEWORK.md new file mode 100644 index 000000000..7b4d6fc9f --- /dev/null +++ b/docs/explanation/features/GENERATED_FILE_EXPORT_FRAMEWORK.md @@ -0,0 +1,94 @@ +# Generated File Export Framework + +Implemented in version: **0.250.072** + +GitHub issue: [#1071](https://github.com/microsoft/simplechat/issues/1071) + +Related config.py update: `VERSION = "0.250.072"` + +## Overview + +Generated file output is a first-class response capability. The framework accepts the completed assistant response and the successful structured function results produced during the same turn, selects a requested renderer, and publishes one authorized downloadable chat artifact. + +CSV, Word (`.docx`), and PDF are separate renderer capabilities. They share source normalization, output intent detection, artifact metadata, authorization-safe publication, downloads, and workspace-promotion behavior. + +## Purpose + +Function results previously remained available as citations, while downloadable output depended on the model reproducing those rows in its final response. That made an action that returned structured data less reliable as an export source than a manually formatted assistant table. + +The framework normalizes current-turn structured function results once and makes them available to every supported renderer. CSV remains the first durable renderer; DOCX and PDF provide immediate generated artifacts for supported response-sized outputs. + +## Dependencies + +- `functions_generated_file_exports.py` for output intent, structured function-result normalization, renderer dispatch, and artifact metadata +- `functions_assistant_table_exports.py` for CSV intent, table parsing, safe headers, and formula-injection protection +- `functions_simplechat_operations.py` for authorized generated chat-artifact upload, download, promotion, and rollback +- `functions_tabular_generated_exports.py` for durable CSV batching, checkpoints, cancellation, reauthorization, and publication +- `python-docx` for DOCX rendering and PyMuPDF for PDF rendering + +## Technical Specifications + +### Supported Renderers + +- **CSV**: Renders structured rows with safe headers, formula neutralization, quoted/multiline values, and durable background execution when the existing row or batch threshold is exceeded. +- **DOCX**: Renders a titled document with final assistant content and, when present, a structured function-result table. +- **PDF**: Renders a titled PDF with final assistant content and, when present, a structured function-result table. + +The response request selects the format through natural language such as `create a CSV`, `create a Word document`, or `export to PDF`. + +### Function Result Source Contract + +Only function results from the current completed response are considered. The adapter: + +- accepts successful citation payloads in conventional `rows`, `data`, `items`, `results`, `records`, `value`, `values`, `result`, `body`, `output`, or `payload` envelopes +- supports a row-like result object when no envelope is present +- parses JSON-string payloads when they contain structured values +- defensively excludes sensitive key names and secret-like fields even after plugin invocation sanitization +- labels merged rows with their originating action when more than one action contributes rows +- ignores `TabularProcessingPlugin` results so CSV/XLSX rows continue through the existing coverage-aware, revision-aware tabular export path + +A valid assistant-rendered table takes precedence over function-result rows for CSV. For DOCX and PDF, the final assistant response is included alongside normalized function-result tables. + +### Response Paths + +The same finalizer is invoked after: + +- standard Chat and streaming Chat +- selected agents and action/tool calls +- Chat Search +- Analyze and Compare document actions +- direct-model and agent workflows +- source-free model responses + +Each path supplies the final assistant content plus its current-turn function citations. The framework does not read arbitrary historical citations or externally supplied action identifiers. + +### Artifact Publication + +The existing generated chat-artifact uploader remains the sole publication mechanism. It validates conversation ownership, allowed output extension, content size, and artifact metadata before creating a blob-backed file message. + +Generated artifacts retain their format, capability, summary, preview metadata, and source provenance. The existing authorized download and workspace-promotion routes work without a new browser transport or external runtime asset. + +## Usage + +Examples: + +- `Ask the billing action for invoices and save the action results as one CSV.` +- `Create a Word document from the action results.` +- `Export the agent's findings to PDF.` +- `Create a PDF report from this response.` + +When an action returns structured data and the assistant summarizes it instead of reprinting a table, the requested generated file still receives the normalized rows. If a request is ambiguous only for CSV row granularity or columns, the assistant asks the existing single conversation clarification before finalization. + +## Testing and Validation + +- `functional_tests/test_assistant_table_csv_artifact.py` covers CSV, DOCX, PDF, structured function-result normalization, sensitive-field exclusion, multi-action provenance, assistant-table precedence, and tabular-plugin exclusion. +- `functional_tests/test_mixed_source_hardening.py` covers cancellation and artifact rollback through the generic finalizer. +- `functional_tests/test_document_action_token_usage_aggregation.py` covers workflow assistant-message persistence with the shared finalizer. +- Existing durable CSV, document action, workflow, and generated-artifact tests remain part of validation. + +## Performance and Limitations + +- CSV retains the existing durable background path for large row sets. +- DOCX and PDF render immediately for response-sized content; durable long-form DOCX work is tracked separately in [#1072](https://github.com/microsoft/simplechat/issues/1072). +- The framework deliberately does not route tabular-plugin rows around source coverage, authorization, or source-version checks. +- Unsupported, failed, unresolved, canceled, or partial source states remain visible through their existing evidence and export contracts; the framework does not fabricate missing rows. diff --git a/docs/explanation/features/UNIVERSAL_CSV_GENERATION.md b/docs/explanation/features/UNIVERSAL_CSV_GENERATION.md index 364f87d61..52eb7d24d 100644 --- a/docs/explanation/features/UNIVERSAL_CSV_GENERATION.md +++ b/docs/explanation/features/UNIVERSAL_CSV_GENERATION.md @@ -1,15 +1,17 @@ # Universal CSV Generation -Implemented in version: **0.250.071** +Implemented in version: **0.250.072** GitHub issue: [#1071](https://github.com/microsoft/simplechat/issues/1071) -Related config.py update: `VERSION = "0.250.071"` +Related config.py update: `VERSION = "0.250.072"` ## Overview CSV is a shared response-output capability rather than a CSV/XLSX-only feature. When a user requests CSV and a response contains valid structured rows, SimpleChat creates one downloadable CSV artifact through the existing authorized chat-artifact contract. +CSV is the durable tabular renderer in the broader [Generated File Export Framework](GENERATED_FILE_EXPORT_FRAMEWORK.md). The framework can also render DOCX and PDF artifacts from final responses and current-turn structured function results. + The finalizer is source-neutral. Native evidence adapters continue to handle PDF, Office, text, image/media-derived, CSV, XLSX, and mixed-source evidence; once a response has a valid Markdown table, tab-separated table, or CSV-shaped result, the same CSV artifact path is used. ## Purpose @@ -41,6 +43,8 @@ The exporter accepts valid structured response forms: Every generated CSV uses safe headers and neutralizes spreadsheet formula-like values while preserving signed numeric text. +Successful current-turn structured function results are also accepted through the generated file export framework when the assistant summarizes an action rather than reproducing its rows. Sensitive fields are excluded, merged action rows carry source provenance, and tabular-plugin results remain on the existing coverage-aware tabular export path. + ### Row and Schema Clarification For an ambiguous CSV request, Chat and workflow model/agent prompts direct the assistant to ask exactly one concise question before generating a file: whether each row represents files, documents, or extracted records, and which columns to include. The assistant response is persisted in the conversation, so the next user turn can answer the clarification without a separate temporary state store. diff --git a/docs/explanation/features/index.md b/docs/explanation/features/index.md index be01c00ec..48736bec6 100644 --- a/docs/explanation/features/index.md +++ b/docs/explanation/features/index.md @@ -40,6 +40,7 @@ category: Version History ## Versioned Features +- [Generated File Export Framework](GENERATED_FILE_EXPORT_FRAMEWORK.md) - [Microsoft Teams App SSO](v0.242.072/TEAMS_APP_SSO.md) - [Tabular SK Large Result Pagination](v0.242.067/TABULAR_SK_LARGE_RESULT_PAGINATION.md) - [Universal CSV Generation](UNIVERSAL_CSV_GENERATION.md) diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 8664c444c..6e3fdba58 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,24 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.073)** + +#### New Features + +* **Expanded Search Retrieval Capacity** + * Increased default search, workflow, assigned-knowledge, directory, and document-metadata retrieval limits so broader authorized result sets can reach Chat, agents, and workflows. + * Expanded mixed-source tabular candidate discovery while retaining authorization and existing maximum bounds. + * (Ref: `functions_search.py`, `functions_search_service.py`, `route_backend_chats.py`, `functions_workflow_runner.py`, `test_document_search_api_and_plugin.py`, `test_mixed_source_manifest_contracts.py`) + +### **(v0.250.072)** + +#### New Features + +* **Generated File Export Framework** + * Added one first-class output framework for CSV, DOCX, and PDF artifacts across Chat, streaming Chat, agents, actions, Analyze, Compare, workflows, and source-free responses. + * Successful structured function results now feed the same authorized artifact pipeline as final assistant content, while tabular-plugin results retain their coverage-aware export path. + * (Ref: microsoft/simplechat#1071, `functions_generated_file_exports.py`, `route_backend_chats.py`, `functions_workflow_runner.py`, `GENERATED_FILE_EXPORT_FRAMEWORK.md`) + ### **(v0.250.071)** #### New Features diff --git a/functional_tests/test_assistant_table_csv_artifact.py b/functional_tests/test_assistant_table_csv_artifact.py index 875d687d0..0eb85923a 100644 --- a/functional_tests/test_assistant_table_csv_artifact.py +++ b/functional_tests/test_assistant_table_csv_artifact.py @@ -2,8 +2,8 @@ #!/usr/bin/env python3 """ Functional test for assistant-rendered table CSV artifacts. -Version: 0.250.071 -Implemented in: 0.241.050; non-tabular document CSV parsing in 0.250.065; universal CSV intent in 0.250.071 +Version: 0.250.073 +Implemented in: 0.241.050; non-tabular document CSV parsing in 0.250.065; generated file export framework in 0.250.072; updated in 0.250.073 This test ensures that explicit table-format requests with assistant-rendered tables, including CSV rows extracted from non-tabular documents, are converted @@ -25,7 +25,7 @@ CHAT_ROUTE_FILE = APP_DIR / 'route_backend_chats.py' BACKGROUND_EXPORT_FILE = APP_DIR / 'functions_tabular_generated_exports.py' WORKFLOW_RUNNER_FILE = APP_DIR / 'functions_workflow_runner.py' -EXPECTED_VERSION = '0.250.071' +EXPECTED_VERSION = '0.250.073' sys.path.append(str(APP_DIR)) @@ -35,9 +35,15 @@ build_safe_csv_headers, build_assistant_table_csv_export, extract_assistant_table_entries, - get_assistant_csv_export_content, neutralize_csv_spreadsheet_formula, ) +from functions_generated_file_exports import ( # noqa: E402 + build_generated_file_artifact_metadata, + build_generated_file_export, + get_generated_file_export_content, + get_requested_generated_file_format, + has_generated_file_output, +) def read_text(path: Path) -> str: @@ -83,20 +89,20 @@ def load_csv_writer_helpers(source_file, function_names): return {function_name: namespace[function_name] for function_name in function_names} -def load_workflow_assistant_table_export_helper(namespace): +def load_workflow_generated_file_export_helper(namespace): module_tree = ast.parse(read_text(WORKFLOW_RUNNER_FILE), filename=str(WORKFLOW_RUNNER_FILE)) selected_nodes = [ node for node in module_tree.body if isinstance(node, ast.FunctionDef) - and node.name == '_maybe_create_workflow_assistant_table_generated_output' + and node.name == '_maybe_create_workflow_generated_file_output' ] if len(selected_nodes) != 1: - raise AssertionError('Expected workflow assistant-table CSV artifact helper.') + raise AssertionError('Expected workflow generated-file artifact helper.') extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) exec(compile(extracted_module, str(WORKFLOW_RUNNER_FILE), 'exec'), namespace) - return namespace['_maybe_create_workflow_assistant_table_generated_output'] + return namespace['_maybe_create_workflow_generated_file_output'] def test_markdown_table_response_builds_csv_export(): @@ -185,7 +191,7 @@ def test_document_action_analysis_reply_builds_csv_export(): ```''', }, } - selected_content = get_assistant_csv_export_content(assistant_result) + selected_content = get_generated_file_export_content(assistant_result) export_payload = build_assistant_table_csv_export( 'turn these into a single CSV', selected_content, @@ -199,6 +205,126 @@ def test_document_action_analysis_reply_builds_csv_export(): assert_true(csv_rows[0]['Invoice Number'] == 'DCAW1366188', 'Expected the analysis reply row to be exported.') +def test_structured_action_result_builds_csv_when_assistant_summarizes(): + print('Testing structured action-result CSV export fallback...') + + action_results = [{ + 'plugin_name': 'BillingPlugin', + 'function_name': 'list_invoices', + 'success': True, + 'function_result': { + 'rows': [ + {'Invoice Number': 'DCAW1366188', 'Amount': '=42.50', 'api_key': 'must-not-export'}, + {'Invoice Number': 'DCAW1366189', 'Amount': '-10.00', 'api_key': 'must-not-export'}, + ], + }, + }] + export_payload = build_generated_file_export( + 'save the action results as one CSV', + 'The billing action returned two invoices.', + function_results=action_results, + ) + + assert_true(export_payload is not None, 'Expected structured action data to produce a CSV when the assistant summarizes it.') + assert_true(export_payload.get('row_source') == 'structured function result', 'Expected function-result CSV provenance.') + csv_rows = parse_csv_rows(export_payload.get('file_content')) + assert_true(len(csv_rows) == 2, 'Expected both action result rows in the CSV artifact.') + assert_true(csv_rows[0]['Invoice Number'] == 'DCAW1366188', 'Expected action result fields to be preserved.') + assert_true(csv_rows[0]['Amount'].startswith("'="), 'Expected action result formulas to be neutralized.') + assert_true('api_key' not in csv_rows[0], 'Expected sensitive action result fields to be omitted.') + + +def test_structured_action_results_combine_and_preserve_assistant_priority(): + print('Testing combined action-result CSV rows and assistant table priority...') + + action_results = [ + { + 'plugin_name': 'DirectoryPlugin', + 'function_name': 'list_people', + 'success': True, + 'function_result': '{"value":[{"Name":"Ada","Department":"Engineering"}]}', + }, + { + 'plugin_name': 'DirectoryPlugin', + 'function_name': 'list_contractors', + 'success': True, + 'function_result': {'items': [{'Name': 'Grace', 'Department': 'Operations'}]}, + }, + ] + action_export = build_generated_file_export( + 'create a combined CSV', + 'The directory actions completed.', + function_results=action_results, + ) + action_rows = parse_csv_rows(action_export.get('file_content')) + assert_true(len(action_rows) == 2, 'Expected data rows from both action results.') + assert_true( + {row['Source action'] for row in action_rows} == {'list_people', 'list_contractors'}, + 'Expected combined action rows to retain their source action.', + ) + + assistant_export = build_generated_file_export( + 'create a combined CSV', + '''| Name | Department | +| --- | --- | +| Assistant-selected | Finance | +''', + function_results=action_results, + ) + assistant_rows = parse_csv_rows(assistant_export.get('file_content')) + assert_true(len(assistant_rows) == 1, 'Expected a valid assistant table to take priority over action rows.') + assert_true(assistant_rows[0]['Name'] == 'Assistant-selected', 'Expected assistant-selected table data to remain authoritative.') + + +def test_tabular_action_result_does_not_bypass_coverage_aware_exports(): + print('Testing tabular action-result exclusion...') + + export_payload = build_generated_file_export( + 'download CSV', + 'The table query returned a partial page.', + function_results=[{ + 'plugin_name': 'TabularProcessingPlugin', + 'function_name': 'query_tabular_data', + 'success': True, + 'function_result': {'data': [{'Case ID': 'SC-1'}]}, + }], + ) + assert_true( + export_payload is None, + 'Expected tabular action rows to remain on their coverage-aware export path.', + ) + + +def test_function_results_render_docx_and_pdf_capabilities(): + print('Testing DOCX and PDF function-result export capabilities...') + + function_results = [{ + 'plugin_name': 'DirectoryPlugin', + 'function_name': 'list_people', + 'success': True, + 'function_result': {'value': [{'Name': 'Ada', 'Department': 'Engineering'}]}, + }] + docx_export = build_generated_file_export( + 'create a Word document from the action results', + 'The directory action completed successfully.', + function_results=function_results, + ) + pdf_export = build_generated_file_export( + 'export the action results to PDF', + 'The directory action completed successfully.', + function_results=function_results, + ) + + assert_true(get_requested_generated_file_format('create a Word document') == 'docx', 'Expected DOCX output intent.') + assert_true(get_requested_generated_file_format('export to PDF') == 'pdf', 'Expected PDF output intent.') + assert_true(get_requested_generated_file_format('I need a DOCX') == 'docx', 'Expected natural DOCX output intent.') + assert_true(get_requested_generated_file_format('Give me a PDF') == 'pdf', 'Expected natural PDF output intent.') + assert_true(docx_export is not None and docx_export['file_content'].startswith(b'PK'), 'Expected a DOCX file export.') + assert_true(pdf_export is not None and pdf_export['file_content'].startswith(b'%PDF'), 'Expected a PDF file export.') + assert_true(docx_export['row_source'] == 'structured function result', 'Expected DOCX to include function-result rows.') + assert_true(pdf_export['row_source'] == 'structured function result', 'Expected PDF to include function-result rows.') + + def test_plain_document_csv_response_excludes_surrounding_prose_and_citation(): print('Testing plain document CSV response boundary detection...') @@ -726,14 +852,16 @@ def test_csv_schema_clarification_guidance_is_specific_and_resumable(): ) -def test_workflow_assistant_table_csv_artifacts_reuse_shared_contract(): - print('Testing workflow assistant-table CSV artifact finalization...') +def test_workflow_generated_file_artifacts_reuse_shared_contract(): + print('Testing workflow generated-file artifact finalization...') uploaded_requests = [] queue_requests = [] shared_namespace = { - 'build_assistant_table_csv_export': build_assistant_table_csv_export, - 'extract_assistant_table_entries': extract_assistant_table_entries, + 'build_generated_file_artifact_metadata': build_generated_file_artifact_metadata, + 'build_generated_file_export': build_generated_file_export, + 'get_requested_generated_file_format': get_requested_generated_file_format, + 'has_generated_file_output': has_generated_file_output, 'has_generated_tabular_csv_output': lambda outputs: any( output.get('output_format') == 'csv' for output in outputs or [] @@ -753,7 +881,7 @@ def test_workflow_assistant_table_csv_artifacts_reuse_shared_contract(): 'logging': type('Logging', (), {'ERROR': 'ERROR'}), 'storage_account_personal_chat_container_name': 'personal-chat', } - helper = load_workflow_assistant_table_export_helper(shared_namespace) + helper = load_workflow_generated_file_export_helper(shared_namespace) workflow = { 'id': 'workflow-1', 'user_id': 'user-1', @@ -777,13 +905,34 @@ def test_workflow_assistant_table_csv_artifacts_reuse_shared_contract(): assert_true(len(uploaded_requests) == 1, 'Expected one authorized artifact upload.') assert_true(uploaded_requests[0]['current_user_id'] == 'user-1', 'Expected upload to use the workflow owner.') assert_true(uploaded_requests[0]['output_format'] == 'csv', 'Expected a CSV artifact upload.') + + word_workflow = { + **workflow, + 'task_prompt': 'create a Word document from the action results', + } + word_artifact = helper( + word_workflow, + 'conversation-1', + word_workflow['task_prompt'], + 'The directory action completed successfully.', + function_results=[{ + 'plugin_name': 'DirectoryPlugin', + 'function_name': 'list_people', + 'success': True, + 'function_result': {'rows': [{'Name': 'Ada', 'Department': 'Engineering'}]}, + }], + ) + assert_true(word_artifact is not None, 'Expected a workflow DOCX artifact from structured function results.') + assert_true(uploaded_requests[-1]['output_format'] == 'docx', 'Expected workflow DOCX artifact metadata.') + assert_true(uploaded_requests[-1]['capability'] == 'file_export', 'Expected generic file-export capability metadata.') + assert_true(uploaded_requests[-1]['file_content'].startswith(b'PK'), 'Expected a rendered DOCX upload payload.') assert_true( helper( workflow, 'conversation-1', workflow['task_prompt'], assistant_content, - [{'capability': 'tabular', 'output_format': 'csv'}], + existing_outputs=[{'capability': 'tabular', 'output_format': 'csv'}], ) is None, 'Expected existing tabular CSV output to suppress a duplicate workflow artifact.', ) @@ -802,7 +951,7 @@ def test_workflow_assistant_table_csv_artifacts_reuse_shared_contract(): } ), }) - background_helper = load_workflow_assistant_table_export_helper(background_namespace) + background_helper = load_workflow_generated_file_export_helper(background_namespace) background_artifact = background_helper( workflow, 'conversation-1', @@ -818,12 +967,12 @@ def test_workflow_assistant_table_csv_artifacts_reuse_shared_contract(): workflow_runner_content = read_text(WORKFLOW_RUNNER_FILE) assert_true( - 'assistant_table_generated_output = _maybe_create_workflow_assistant_table_generated_output(' in workflow_runner_content, - 'Expected workflow assistant messages to finalize shared CSV artifacts.', + 'generated_file_output = _maybe_create_workflow_generated_file_output(' in workflow_runner_content, + 'Expected workflow assistant messages to finalize shared file artifacts.', ) assert_true( - 'generated_analysis_artifacts.append(assistant_table_generated_output)' in workflow_runner_content, - 'Expected workflow CSV metadata to reach the generic artifact UI.', + 'generated_analysis_artifacts.append(generated_file_output)' in workflow_runner_content, + 'Expected workflow generated-file metadata to reach the generic artifact UI.', ) @@ -839,12 +988,12 @@ def test_chat_route_wires_assistant_table_artifacts(): 'Expected route_backend_chats.py to reuse the shared assistant table export intent predicate.', ) assert_true( - 'def maybe_create_assistant_table_generated_output(' in chat_route_content, - 'Expected route_backend_chats.py to expose assistant table artifact creation.', + 'def maybe_create_generated_file_output(' in chat_route_content, + 'Expected route_backend_chats.py to expose generic generated-file artifact creation.', ) assert_true( - 'should_queue_tabular_generated_output_background(row_count, len(row_batches), settings)' in chat_route_content, - 'Expected large assistant-derived CSV artifacts to use the durable background export threshold.', + "output_format == 'csv' and should_queue_tabular_generated_output_background(" in chat_route_content, + 'Expected large generated CSV artifacts to use the durable background export threshold.', ) assert_true( 'queue_tabular_generated_output_run(' in chat_route_content, @@ -859,29 +1008,41 @@ def test_chat_route_wires_assistant_table_artifacts(): 'Expected queued assistant exports to reuse standard background-export metadata.', ) assert_true( - 'document_generated_analysis_artifacts.append(assistant_table_generated_output)' in chat_route_content, - 'Expected document-action assistant messages to include assistant table CSV artifacts.', + 'document_generated_analysis_artifacts.append(generated_file_output)' in chat_route_content, + 'Expected document-action assistant messages to include generated file artifacts.', ) assert_true( - 'generated_analysis_artifacts_list.append(assistant_table_generated_output)' in chat_route_content, - 'Expected normal and streaming assistant messages to include assistant table CSV artifacts.', + 'generated_analysis_artifacts_list.append(generated_file_output)' in chat_route_content, + 'Expected normal and streaming assistant messages to include generated file artifacts.', ) assert_true( - 'assistant_content=get_assistant_csv_export_content(execution_result)' in chat_route_content, - 'Expected document-action CSV exports to use the structured analysis reply when available.', + 'assistant_content=get_generated_file_export_content(execution_result)' in chat_route_content, + 'Expected document-action file exports to use the structured analysis reply when available.', ) assert_true( - 'assistant_content=get_assistant_csv_export_content(result)' in read_text(WORKFLOW_RUNNER_FILE), - 'Expected workflow CSV exports to use the structured analysis reply when available.', + 'assistant_content=get_generated_file_export_content(result)' in read_text(WORKFLOW_RUNNER_FILE), + 'Expected workflow file exports to use the structured analysis reply when available.', ) assert_true( - chat_route_content.count('build_csv_output_clarification_guidance(user_message)') == 2, - 'Expected normal and streaming Chat to apply the same CSV clarification guidance.', + chat_route_content.count('build_generated_file_output_guidance(user_message)') == 2, + 'Expected normal and streaming Chat to apply the same file-output guidance.', ) workflow_runner_content = read_text(WORKFLOW_RUNNER_FILE) assert_true( - workflow_runner_content.count('build_csv_output_clarification_guidance(prompt_text)') == 2, - 'Expected workflow model and agent execution to apply CSV clarification guidance.', + workflow_runner_content.count('build_generated_file_output_guidance(prompt_text)') == 2, + 'Expected workflow model and agent execution to apply the same file-output guidance.', + ) + assert_true( + chat_route_content.count('function_results=agent_citations_list') == 2, + 'Expected normal and streaming Chat to pass current-turn action results to generated-file exports.', + ) + assert_true( + 'function_results=execution_result.get(\'agent_citations\') or []' in chat_route_content, + 'Expected document actions to pass current-turn action results to generated-file exports.', + ) + assert_true( + 'function_results=raw_agent_citations' in workflow_runner_content, + 'Expected workflows to pass current-turn action results to generated-file exports.', ) assert_true( 'if assistant_table_export_requested(user_question):' in chat_route_content, @@ -899,6 +1060,10 @@ def run_tests() -> bool: test_tab_separated_table_response_builds_rows, test_non_tabular_document_csv_response_builds_export, test_document_action_analysis_reply_builds_csv_export, + test_structured_action_result_builds_csv_when_assistant_summarizes, + test_structured_action_results_combine_and_preserve_assistant_priority, + test_tabular_action_result_does_not_bypass_coverage_aware_exports, + test_function_results_render_docx_and_pdf_capabilities, test_plain_document_csv_response_excludes_surrounding_prose_and_citation, test_document_csv_response_preserves_multiline_and_escaped_quotes, test_fenced_document_csv_preserves_sentence_shaped_rows, @@ -920,7 +1085,7 @@ def run_tests() -> bool: test_natural_csv_and_create_table_phrases_are_recognized, test_universal_csv_request_variants_are_recognized, test_csv_schema_clarification_guidance_is_specific_and_resumable, - test_workflow_assistant_table_csv_artifacts_reuse_shared_contract, + test_workflow_generated_file_artifacts_reuse_shared_contract, test_chat_route_wires_assistant_table_artifacts, ] diff --git a/functional_tests/test_document_action_conversation_scope_metadata.py b/functional_tests/test_document_action_conversation_scope_metadata.py index 39b82dc01..3e33bb52f 100644 --- a/functional_tests/test_document_action_conversation_scope_metadata.py +++ b/functional_tests/test_document_action_conversation_scope_metadata.py @@ -2,8 +2,8 @@ # test_document_action_conversation_scope_metadata.py """ Functional test for document-action conversation scope metadata. -Version: 0.250.071 -Implemented in: 0.241.124; Updated in: 0.250.071 +Version: 0.250.073 +Implemented in: 0.241.124; Updated in: 0.250.073 This test ensures Analyze and tabular document-action results can assign conversation workspace metadata from selected document summaries when no @@ -20,7 +20,7 @@ METADATA_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'functions_conversation_metadata.py') ROUTE_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'route_backend_chats.py') CONFIG_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'config.py') -FIX_VERSION = '0.250.071' +FIX_VERSION = '0.250.073' TEST_USER_ID = 'scope-user-1' CRIMSON_GROUP_ID = 'crimson-group-1' PUBLIC_WORKSPACE_ID = 'public-workspace-1' diff --git a/functional_tests/test_document_action_token_usage_aggregation.py b/functional_tests/test_document_action_token_usage_aggregation.py index cfae3866f..f4bed695e 100644 --- a/functional_tests/test_document_action_token_usage_aggregation.py +++ b/functional_tests/test_document_action_token_usage_aggregation.py @@ -1,8 +1,8 @@ # test_document_action_token_usage_aggregation.py """ Functional test for document action token usage aggregation. -Version: 0.250.071 -Implemented in: 0.241.116; updated for universal CSV artifacts in 0.250.071 +Version: 0.250.073 +Implemented in: 0.241.116; updated for generated file exports in 0.250.072; updated in 0.250.073 This test ensures analysis and comparison aggregate tokens across all internal model calls and persist the aggregate usage on assistant metadata. @@ -299,8 +299,8 @@ def test_workflow_assistant_persists_token_usage(): '_get_document_action_config': lambda workflow: workflow.get('document_action', {}), '_get_workflow_scope': lambda workflow: 'personal', '_get_workflow_group_id': lambda workflow: '', - '_maybe_create_workflow_assistant_table_generated_output': lambda **kwargs: None, - 'get_assistant_csv_export_content': lambda result: result.get('reply', ''), + '_maybe_create_workflow_generated_file_output': lambda **kwargs: None, + 'get_generated_file_export_content': lambda result: result.get('reply', ''), '_persist_agent_citation_artifacts': lambda **kwargs: [], 'cosmos_messages_container': message_container, 'cosmos_conversations_container': conversation_container, @@ -373,7 +373,7 @@ def test_version_update(): with open(CONFIG_PATH, 'r', encoding='utf-8') as handle: content = handle.read() - assert_in('VERSION = "0.250.071"', content, 'config version update') + assert_in('VERSION = "0.250.073"', content, 'config version update') print('Version update passed.') return True diff --git a/functional_tests/test_document_search_api_and_plugin.py b/functional_tests/test_document_search_api_and_plugin.py index e6fe84a02..ed4281ea8 100644 --- a/functional_tests/test_document_search_api_and_plugin.py +++ b/functional_tests/test_document_search_api_and_plugin.py @@ -66,7 +66,7 @@ def test_functions_search_contract(): return False required_snippets = [ - 'SEARCH_DEFAULT_TOP_N = 12', + 'SEARCH_DEFAULT_TOP_N = 50', 'SEARCH_MAX_TOP_N = 500', '"document_id": r.get("document_id")', 'select=get_search_select_fields("personal")', diff --git a/functional_tests/test_mixed_source_analyze_workflow.py b/functional_tests/test_mixed_source_analyze_workflow.py index 21133220f..9e40414a2 100644 --- a/functional_tests/test_mixed_source_analyze_workflow.py +++ b/functional_tests/test_mixed_source_analyze_workflow.py @@ -2,8 +2,8 @@ # test_mixed_source_analyze_workflow.py """ Functional test for Phase 3 mixed-source combined Analyze. -Version: 0.250.071 -Implemented in: 0.250.071 +Version: 0.250.073 +Implemented in: 0.250.072; updated in 0.250.073 This test ensures #1058 composes native narrative and tabular analysis behind automatic combined Analyze routing, retains terminal coverage after either diff --git a/functional_tests/test_mixed_source_conversation_continuity.py b/functional_tests/test_mixed_source_conversation_continuity.py index 3acc40715..cce3cc75a 100644 --- a/functional_tests/test_mixed_source_conversation_continuity.py +++ b/functional_tests/test_mixed_source_conversation_continuity.py @@ -2,8 +2,8 @@ # test_mixed_source_conversation_continuity.py """ Functional test for Phase 5 mixed-source conversation continuity. -Version: 0.250.071 -Implemented in: 0.250.068; updated in 0.250.071 +Version: 0.250.073 +Implemented in: 0.250.068; updated in 0.250.073 This test ensures #1060 preserves compact source continuity only as a reauthorization hint for #1055 and prerequisite phases #1056, #1057, #1058, @@ -118,7 +118,7 @@ def test_flag_and_standard_streaming_wiring_are_present(): assert route_source.count("'history',") >= 2 assert 'source_continuity_refs=None' in metadata_source assert 'source_continuity_refs=source_continuity_refs' in metadata_source - assert 'VERSION = "0.250.071"' in config_source + assert 'VERSION = "0.250.073"' in config_source print('PASS: flag and Chat parity wiring') diff --git a/functional_tests/test_mixed_source_hardening.py b/functional_tests/test_mixed_source_hardening.py index b1c9921ed..b154413af 100644 --- a/functional_tests/test_mixed_source_hardening.py +++ b/functional_tests/test_mixed_source_hardening.py @@ -574,19 +574,30 @@ def test_export_cancellation_rolls_back_queued_and_uploaded_artifacts(): def load_export_helpers(queue_background): return _load_route_helpers( - {"_has_generated_tabular_csv_output", "maybe_create_assistant_table_generated_output"}, + {"_has_generated_tabular_csv_output", "maybe_create_generated_file_output"}, namespace={ "MixedSourceCancellationError": orchestration.MixedSourceCancellationError, "raise_if_mixed_source_cancelled": orchestration.raise_if_mixed_source_cancelled, - "build_assistant_table_csv_export": lambda question, content: { + "has_generated_tabular_csv_output": lambda outputs: False, + "get_requested_generated_file_format": lambda question: "csv", + "has_generated_file_output": lambda outputs, output_format: False, + "build_generated_file_export": lambda *args, **kwargs: { + "capability": "file_export", "file_name": "answer.csv", "file_content": "name,value\nalpha,1\n", + "output_format": "csv", "row_count": 1, "summary": "One row.", + "_structured_rows": [{"name": "alpha", "value": 1}], + }, + "build_generated_file_artifact_metadata": lambda export_payload, upload_result, conversation_id: { + "artifact_message_id": upload_result["message"]["id"], + "conversation_id": conversation_id, + "file_name": upload_result["message"]["file_name"], + "output_format": export_payload["output_format"], }, "_safe_int": lambda value: int(value or 0), "get_settings": lambda: {}, - "extract_assistant_table_entries": lambda content: [{"name": "alpha", "value": 1}], "_build_tabular_generated_output_row_batches": lambda rows, settings=None: [rows], "should_queue_tabular_generated_output_background": lambda *args: queue_background, "queue_tabular_generated_output_run": lambda **kwargs: ( @@ -607,7 +618,7 @@ def load_export_helpers(queue_background): "log_event": lambda *args, **kwargs: None, "logging": __import__("logging"), }, - )["maybe_create_assistant_table_generated_output"] + )["maybe_create_generated_file_output"] queued_helper = load_export_helpers(queue_background=True) queued_checks = iter([False, True]) diff --git a/functional_tests/test_mixed_source_manifest_contracts.py b/functional_tests/test_mixed_source_manifest_contracts.py index 7c156941d..6ccfbc937 100644 --- a/functional_tests/test_mixed_source_manifest_contracts.py +++ b/functional_tests/test_mixed_source_manifest_contracts.py @@ -117,7 +117,7 @@ def load_isolated_search_service(): public_stub.get_user_visible_public_workspace_ids_from_settings = lambda user_id: [] search_stub = types.ModuleType("functions_search") - search_stub.SEARCH_DEFAULT_TOP_N = 12 + search_stub.SEARCH_DEFAULT_TOP_N = 50 search_stub.SEARCH_MAX_TOP_N = 500 search_stub.hybrid_search = lambda **kwargs: [] search_stub.normalize_search_id_list = _normalize_id_list diff --git a/functional_tests/test_tabular_row_orchestration_scale.py b/functional_tests/test_tabular_row_orchestration_scale.py index f826f3e72..d8a7289ad 100644 --- a/functional_tests/test_tabular_row_orchestration_scale.py +++ b/functional_tests/test_tabular_row_orchestration_scale.py @@ -1,8 +1,8 @@ # test_tabular_row_orchestration_scale.py """ Functional test for scalable per-row tabular orchestration. -Version: 0.250.071 -Implemented in: 0.250.060; generated CSV formula safety in 0.250.065; shared CSV queue authorization in 0.250.071 +Version: 0.250.073 +Implemented in: 0.250.060; generated CSV formula safety in 0.250.065; generated file export routing in 0.250.072; updated in 0.250.073 This test ensures generated exports preserve source identity and row order while enforcing one stable output schema across independently generated batches. @@ -1217,15 +1217,15 @@ def test_route_queues_replayable_pages_and_suppresses_summary_fallback(): assert '_has_generated_tabular_csv_output(existing_outputs)' in route_source route_module = ast.parse(route_source, filename=str(CHAT_ROUTE)) - assistant_table_export = next( + generated_file_export = next( node for node in route_module.body if isinstance(node, ast.FunctionDef) - and node.name == 'maybe_create_assistant_table_generated_output' + and node.name == 'maybe_create_generated_file_output' ) - assistant_table_export_source = ast.get_source_segment(route_source, assistant_table_export) - assert "'source': 'chat'," in assistant_table_export_source - assert "'container': storage_account_personal_chat_container_name" not in assistant_table_export_source + generated_file_export_source = ast.get_source_segment(route_source, generated_file_export) + assert "'source': 'chat'," in generated_file_export_source + assert "'container': storage_account_personal_chat_container_name" not in generated_file_export_source helpers = _load_failed_export_helpers() failed_output = helpers['_build_failed_tabular_generated_output_metadata'](