\s*
', '', text_content) # Remove empty paragraphs + + return BeautifulSoup(text_content, 'html.parser') + +def clean_title_text(title: str) -> str: + """Clean title text by removing unwanted characters and symbols.""" + # Remove paragraph symbols and other unwanted characters + title = re.sub(r'¶', '', title) # Remove paragraph symbols + title = re.sub(r'¶', '', title) # Remove HTML paragraph entities + title = re.sub(r'¶', '', title) # Remove numeric paragraph entities + + # Clean up extra whitespace + title = re.sub(r'\s+', ' ', title) # Collapse multiple spaces + title = title.strip() + + return title + +def extract_page_title(soup: BeautifulSoup) -> str: + """Extract the main page title.""" + # Try different selectors for the main title + title_selectors = [ + 'h1', '.document h1', 'main h1', + '.content h1', 'article h1', + '.section h1', '#content h1' + ] + + for selector in title_selectors: + title_elem = soup.select_one(selector) + if title_elem: + title = title_elem.get_text().strip() + return clean_title_text(title) + + # Fallback to HTML title + title_elem = soup.find('title') + if title_elem: + title = title_elem.get_text().strip() + # Clean up Sphinx-style titles + if ' — ' in title: + title = title.split(' — ')[0] + return clean_title_text(title) + + return "Untitled" + +def analyze_document_structure(html_files: list, build_dir: Path) -> list: + """Analyze document structure to create hierarchical TOC with page estimates.""" + toc_items = [] + page_counter = 3 # Start after cover page and TOC + + current_section = None + section_items = [] + + for i, html_file in enumerate(html_files): + try: + with open(html_file, 'r', encoding='utf-8') as f: + content = f.read() + soup = BeautifulSoup(content, 'html.parser') + + title = extract_page_title(soup) + rel_path = html_file.relative_to(build_dir / "html") + section = rel_path.parts[0] if rel_path.parts else 'General' + + # Estimate pages based on content length (rough approximation) + text_length = len(soup.get_text()) + estimated_pages = max(1, text_length // 3000) # ~3000 chars per page + + anchor_id = f"section_{i}" + + item = { + 'title': title, + 'path': str(rel_path), + 'section': section, + 'anchor': anchor_id, + 'page': page_counter, + 'estimated_pages': estimated_pages, + 'level': 1 # Default level + } + + # Determine hierarchy level based on file path depth and title + path_depth = len(rel_path.parts) - 1 # Subtract 1 for filename + if title.lower().startswith(('overview', 'introduction', 'getting started')): + item['level'] = 1 + elif path_depth > 1 or any(word in title.lower() for word in ['configuration', 'settings', 'advanced']): + item['level'] = 2 + else: + item['level'] = 1 + + toc_items.append(item) + page_counter += estimated_pages + + except Exception as e: + logger.warning(f"Could not analyze {html_file}: {e}") + + return toc_items + +def create_table_of_contents(html_files: list, build_dir: Path) -> tuple: + """Generate a hierarchical table of contents with page numbers.""" + toc_items = analyze_document_structure(html_files, build_dir) + + # Generate enhanced TOC HTML + toc_html = """ ++ Generated from Mattermost Documentation +
+""" + + # Add Table of Contents + toc_html, toc_items = create_table_of_contents(html_files, build_dir) + html_content += toc_html + + # Process each HTML file + for i, html_file in enumerate(html_files): + try: + with open(html_file, 'r', encoding='utf-8') as f: + soup = BeautifulSoup(f.read(), 'html.parser') + + # Clean the content + soup = clean_html_content(soup, build_dir) + + # Extract the main content + main_content = None + for selector in ['main', '.main', '#main', 'article', '.document', '.content']: + main_content = soup.select_one(selector) + if main_content: + break + + if not main_content: + main_content = soup.find('body') + + if main_content: + # Add section break for all but the first file + if i > 0: + html_content += '\n' + + # Add anchor for TOC linking + anchor_id = toc_items[i]['anchor'] + page_title = clean_title_text(toc_items[i]['title']) # Clean the title + html_content += f'