From b3d746b4e35b18598446b054128311d176402bad Mon Sep 17 00:00:00 2001 From: Alan Mond Date: Thu, 17 Jul 2025 23:11:28 -0700 Subject: [PATCH 1/8] added better structure --- Dockerfile | 7 +- config.yaml | 114 +++++++++++++++++++++------- devsite_to_hugo_converter.py | 25 +++++- templates/hugo_config.yaml.jinja2 | 25 ++++-- utils/hugo_generator.py | 122 +++++++++++++++++++++++++++--- 5 files changed, 247 insertions(+), 46 deletions(-) diff --git a/Dockerfile b/Dockerfile index 210c0bf..2cd4aa1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,4 +66,9 @@ RUN hugo --destination /workspace/public EXPOSE 1313 -CMD ["hugo", "server", "--bind", "0.0.0.0", "--baseURL", "https://bazel-docs-68tmf.ondigitalocean.app/", "--disableFastRender"] +# Default base URL can be overridden at build or run time +ARG BASE_URL=https://bazel-docs-68tmf.ondigitalocean.app/ +ENV HUGO_BASEURL=${BASE_URL} + +# Use shell form so the environment variable is expanded correctly +CMD ["sh","-c", "hugo server --bind 0.0.0.0 --baseURL \"$HUGO_BASEURL\" --disableFastRender"] diff --git a/config.yaml b/config.yaml index f690cc5..2ba7753 100644 --- a/config.yaml +++ b/config.yaml @@ -15,47 +15,103 @@ hugo: languageCode: "en-us" theme: "docsy" -# Content mapping - map Devsite sections to Hugo content types +# Content mapping - organized into 4 main categories content_mapping: - concepts: - type: "docs" + # TUTORIALS - Learning-oriented content + tutorials: + type: "tutorials" weight: 10 - basics: - type: "docs" + category: "Tutorials" + start: + type: "tutorials" weight: 20 - tutorials: - type: "docs" + category: "Tutorials" + basics: + type: "tutorials" weight: 30 + category: "Tutorials" + + # HOW-TO GUIDES - Problem-solving oriented + install: + type: "how-to-guides" + weight: 10 + category: "How-To Guides" + configure: + type: "how-to-guides" + weight: 20 + category: "How-To Guides" build: - type: "docs" + type: "how-to-guides" + weight: 30 + category: "How-To Guides" + run: + type: "how-to-guides" weight: 40 - configure: - type: "docs" + category: "How-To Guides" + remote: + type: "how-to-guides" weight: 50 - extending: - type: "docs" + category: "How-To Guides" + migrate: + type: "how-to-guides" weight: 60 - external: - type: "docs" + category: "How-To Guides" + rules: + type: "how-to-guides" weight: 70 - remote: - type: "docs" + category: "How-To Guides" + docs: + type: "how-to-guides" weight: 80 - query: - type: "docs" - weight: 90 - reference: - type: "docs" - weight: 100 + category: "How-To Guides" + + # EXPLANATIONS - Understanding-oriented content + concepts: + type: "explanations" + weight: 10 + category: "Explanations" + extending: + type: "explanations" + weight: 20 + category: "Explanations" + external: + type: "explanations" + weight: 30 + category: "Explanations" + about: + type: "explanations" + weight: 40 + category: "Explanations" community: - type: "community" - weight: 200 + type: "explanations" + weight: 50 + category: "Explanations" contribute: - type: "community" - weight: 210 - about: - type: "about" - weight: 300 + type: "explanations" + weight: 60 + category: "Explanations" + release: + type: "explanations" + weight: 70 + category: "Explanations" + + # REFERENCE - Information-oriented content + reference: + type: "reference" + weight: 10 + category: "Reference" + query: + type: "reference" + weight: 20 + category: "Reference" + versions: + type: "reference" + weight: 30 + category: "Reference" + advanced: + type: "reference" + weight: 40 + category: "Reference" # CSS conversion settings css_conversion: diff --git a/devsite_to_hugo_converter.py b/devsite_to_hugo_converter.py index f544692..ec473ca 100644 --- a/devsite_to_hugo_converter.py +++ b/devsite_to_hugo_converter.py @@ -172,9 +172,12 @@ def _convert_content_files(self, devsite_structure: Dict, source_path: str, conversion_stats['skipped_files'] += 1 continue + # Determine category for this file + category_path = self._get_category_path(relative_path) + # Convert file if self._convert_single_file( - md_file, output_dir / 'content' / relative_path, + md_file, output_dir / 'content' / category_path, devsite_structure, dry_run): conversion_stats['converted_files'] += 1 else: @@ -186,6 +189,26 @@ def _convert_content_files(self, devsite_structure: Dict, source_path: str, return conversion_stats + def _get_category_path(self, relative_path: Path) -> Path: + """Get the category path for a file based on its section""" + # Get the top-level directory (section) + parts = relative_path.parts + if not parts: + return relative_path + + section_name = parts[0] + + # Look up the category for this section + if section_name in self.config['content_mapping']: + mapping = self.config['content_mapping'][section_name] + category_type = mapping['type'] + + # Return path with category prefix + return Path(category_type) / relative_path + + # Default to original path if no mapping found + return relative_path + def _convert_single_file(self, source_file: Path, output_file: Path, devsite_structure: Dict, dry_run: bool) -> bool: """Convert a single markdown file from Devsite to Hugo format""" diff --git a/templates/hugo_config.yaml.jinja2 b/templates/hugo_config.yaml.jinja2 index 009eece..0c87c9c 100644 --- a/templates/hugo_config.yaml.jinja2 +++ b/templates/hugo_config.yaml.jinja2 @@ -50,15 +50,24 @@ markup: # Menu configuration menu: main: - - name: "Documentation" - url: "/docs/" + - name: "Tutorials" + url: "/tutorials/" weight: 10 - - name: "Community" - url: "/community/" + - name: "How-To Guides" + url: "/how-to-guides/" weight: 20 - - name: "About" - url: "/about/" + - name: "Explanations" + url: "/explanations/" weight: 30 + - name: "Reference" + url: "/reference/" + weight: 40 + - name: "Community" + url: "/explanations/community/" + weight: 50 + - name: "About" + url: "/explanations/about/" + weight: 60 # Parameters for Docsy theme params: @@ -101,6 +110,10 @@ params: ui: showLightDarkModeMenu: true mode: dark # Set the default mode to dark + sidebar_menu_compact: true + ul_show: 1 + sidebar_menu_foldable: true + sidebar_cache_limit: 1000 # Taxonomies taxonomies: diff --git a/utils/hugo_generator.py b/utils/hugo_generator.py index 9995c82..b05506c 100644 --- a/utils/hugo_generator.py +++ b/utils/hugo_generator.py @@ -128,6 +128,34 @@ def _generate_main_index(self, content_dir: Path, devsite_structure: Dict) -> No 'weight': 1 } + # Create the 4 main categories + categories = [ + { + 'title': 'Tutorials', + 'path': '/tutorials/', + 'description': 'Tutorials to guide you through Bazel specific examples', + 'weight': 10 + }, + { + 'title': 'How-To Guides', + 'path': '/how-to-guides/', + 'description': 'Guides for specific tasks and issues your will encounter', + 'weight': 20 + }, + { + 'title': 'Explanations', + 'path': '/explanations/', + 'description': 'Understanding Bazel concepts and features', + 'weight': 30 + }, + { + 'title': 'Reference', + 'path': '/reference/', + 'description': 'Reference materials, API documentation, and good information for rules authors', + 'weight': 40 + } + ] + # Render main index template = self.template_env.get_template('section_index.jinja2') index_content = template.render({ @@ -136,14 +164,7 @@ def _generate_main_index(self, content_dir: Path, devsite_structure: Dict) -> No 'description': context['description'], 'type': 'docs', 'weight': 1, - 'subsections': [ - { - 'title': section['title'], - 'path': f"/{section['name']}/", - 'description': f"{section['title']} documentation" - } - for section in devsite_structure['sections'] - ] + 'subsections': categories } }) @@ -153,11 +174,94 @@ def _generate_main_index(self, content_dir: Path, devsite_structure: Dict) -> No with open(index_file, 'w', encoding='utf-8') as f: f.write(index_content) + # Generate category index files + self._generate_category_indices(content_dir, devsite_structure) + logger.debug(f"Generated main index: {index_file}") + def _generate_category_indices(self, content_dir: Path, devsite_structure: Dict) -> None: + """Generate _index.md files for the 4 main categories""" + categories = { + 'tutorials': { + 'title': 'Tutorials', + 'description': 'Learning-oriented content to get you started with Bazel', + 'weight': 10, + 'sections': [] + }, + 'how-to-guides': { + 'title': 'How-To Guides', + 'description': 'Problem-solving oriented guides for specific tasks', + 'weight': 20, + 'sections': [] + }, + 'explanations': { + 'title': 'Explanations', + 'description': 'Understanding-oriented content about Bazel concepts', + 'weight': 30, + 'sections': [] + }, + 'reference': { + 'title': 'Reference', + 'description': 'Information-oriented reference materials', + 'weight': 40, + 'sections': [] + } + } + + # Group sections by category + for section in devsite_structure['sections']: + section_name = section['name'] + if section_name in self.config['content_mapping']: + mapping = self.config['content_mapping'][section_name] + category_type = mapping['type'] + if category_type in categories: + categories[category_type]['sections'].append(section) + + # Generate index file for each category + for category_type, category_info in categories.items(): + category_dir = content_dir / category_type + category_dir.mkdir(parents=True, exist_ok=True) + + # Prepare subsections + subsections = [] + for section in category_info['sections']: + subsections.append({ + 'title': section['title'], + 'path': f"/{category_type}/{section['name']}/", + 'description': f"{section['title']} documentation" + }) + + # Render category index + template = self.template_env.get_template('section_index.jinja2') + index_content = template.render({ + 'section': { + 'title': category_info['title'], + 'description': category_info['description'], + 'type': 'docs', + 'weight': category_info['weight'], + 'subsections': subsections + } + }) + + # Write category index file + index_file = category_dir / '_index.md' + with open(index_file, 'w', encoding='utf-8') as f: + f.write(index_content) + + logger.debug(f"Generated category index: {index_file}") + def _generate_section_index(self, content_dir: Path, section: Dict) -> None: """Generate _index.md file for a section""" - section_dir = content_dir / section['name'] + # Determine the category for this section + section_name = section['name'] + category_type = 'docs' # default + + if section_name in self.config['content_mapping']: + mapping = self.config['content_mapping'][section_name] + category_type = mapping['type'] + + # Create section directory under its category + section_dir = content_dir / category_type / section['name'] section_dir.mkdir(parents=True, exist_ok=True) # Prepare subsections list From 0dd22b25666647374651b115516161c6d3ed1c35 Mon Sep 17 00:00:00 2001 From: Alan Mond Date: Thu, 17 Jul 2025 23:33:52 -0700 Subject: [PATCH 2/8] use constants --- utils/hugo_generator.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/utils/hugo_generator.py b/utils/hugo_generator.py index b05506c..58a37e4 100644 --- a/utils/hugo_generator.py +++ b/utils/hugo_generator.py @@ -12,6 +12,11 @@ logger = logging.getLogger(__name__) +TUTORIALS_DESCRIPTION = 'Tutorials to guide you through Bazel specific examples' +HOW_TO_GUIDES_DESCRIPTION = 'Guides for specific tasks and issues your will encounter' +EXPLANATIONS_DESCRIPTION = 'Understanding Bazel concepts and features' +REFERENCE_DESCRIPTION = 'Reference materials, API documentation, and good information for rules authors' + class HugoGenerator: """Generator for Hugo site structure and configuration""" @@ -133,25 +138,25 @@ def _generate_main_index(self, content_dir: Path, devsite_structure: Dict) -> No { 'title': 'Tutorials', 'path': '/tutorials/', - 'description': 'Tutorials to guide you through Bazel specific examples', + 'description': TUTORIALS_DESCRIPTION, 'weight': 10 }, { 'title': 'How-To Guides', 'path': '/how-to-guides/', - 'description': 'Guides for specific tasks and issues your will encounter', + 'description': HOW_TO_GUIDES_DESCRIPTION, 'weight': 20 }, { 'title': 'Explanations', 'path': '/explanations/', - 'description': 'Understanding Bazel concepts and features', + 'description': EXPLANATIONS_DESCRIPTION, 'weight': 30 }, { 'title': 'Reference', 'path': '/reference/', - 'description': 'Reference materials, API documentation, and good information for rules authors', + 'description': REFERENCE_DESCRIPTION, 'weight': 40 } ] @@ -184,25 +189,25 @@ def _generate_category_indices(self, content_dir: Path, devsite_structure: Dict) categories = { 'tutorials': { 'title': 'Tutorials', - 'description': 'Learning-oriented content to get you started with Bazel', + 'description': TUTORIALS_DESCRIPTION, 'weight': 10, 'sections': [] }, 'how-to-guides': { 'title': 'How-To Guides', - 'description': 'Problem-solving oriented guides for specific tasks', + 'description': HOW_TO_GUIDES_DESCRIPTION, 'weight': 20, 'sections': [] }, 'explanations': { 'title': 'Explanations', - 'description': 'Understanding-oriented content about Bazel concepts', + 'description': EXPLANATIONS_DESCRIPTION, 'weight': 30, 'sections': [] }, 'reference': { 'title': 'Reference', - 'description': 'Information-oriented reference materials', + 'description': REFERENCE_DESCRIPTION, 'weight': 40, 'sections': [] } From 7651ed19711d20f500f85af1e92d003a925349d0 Mon Sep 17 00:00:00 2001 From: Alan Mond Date: Sun, 27 Jul 2025 18:05:20 -0700 Subject: [PATCH 3/8] improved code blocks --- .github/workflows/hugo-render-gh-pages.yml | 114 +++++++++++++++ config.yaml | 151 ++++++++++++++++++- devsite_to_hugo_converter.py | 162 +++++++++++++++------ templates/hugo_config.yaml.jinja2 | 9 +- utils/css_converter.py | 3 +- 5 files changed, 387 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/hugo-render-gh-pages.yml diff --git a/.github/workflows/hugo-render-gh-pages.yml b/.github/workflows/hugo-render-gh-pages.yml new file mode 100644 index 0000000..f4d6218 --- /dev/null +++ b/.github/workflows/hugo-render-gh-pages.yml @@ -0,0 +1,114 @@ +# Sample workflow for building and deploying a Hugo site to GitHub Pages +name: Deploy Hugo site to Pages + +on: + push: + # prevent this workflow from running on pushes to the main branch + branches-ignore: [main] + # Run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + # allow write access to add a comment to the PR + pull-requests: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +# Default to bash +defaults: + run: + # GitHub-hosted runners automatically enable `set -eo pipefail` for Bash shells. + shell: bash + +jobs: + # Build job + build: + runs-on: ubuntu-latest + env: + DART_SASS_VERSION: 1.89.2 + HUGO_VERSION: 0.146.0 + HUGO_ENVIRONMENT: production + TZ: America/Los_Angeles + steps: + - name: Install Hugo CLI + run: | + wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb + sudo dpkg -i ${{ runner.temp }}/hugo.deb + - name: Install Dart Sass + run: | + wget -O ${{ runner.temp }}/dart-sass.tar.gz https://github.com/sass/dart-sass/releases/download/${DART_SASS_VERSION}/dart-sass-${DART_SASS_VERSION}-linux-x64.tar.gz + tar -xf ${{ runner.temp }}/dart-sass.tar.gz --directory ${{ runner.temp }} + mv ${{ runner.temp }}/dart-sass/ /usr/local/bin + echo "/usr/local/bin/dart-sass" >> $GITHUB_PATH + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + - name: Setup Pages + id: pages + uses: actions/configure-pages@v5 + - name: Install Node.js dependencies + run: "[[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true" + - name: Cache Restore + id: cache-restore + uses: actions/cache/restore@v4 + with: + path: | + ${{ runner.temp }}/hugo_cache + key: hugo-${{ github.run_id }} + restore-keys: + hugo- + - name: Configure Git + run: git config core.quotepath false + - name: Build with Hugo + run: | + hugo \ + --gc \ + --minify \ + --baseURL "${{ steps.pages.outputs.base_url }}/" \ + --cacheDir "${{ runner.temp }}/hugo_cache" + - name: Cache Save + id: cache-save + uses: actions/cache/save@v4 + with: + path: | + ${{ runner.temp }}/hugo_cache + key: ${{ steps.cache-restore.outputs.cache-primary-key }} + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./public + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + +# Create a comment on the pull request with a link to the deployed site + comment: + runs-on: ubuntu-latest + needs: deploy + if: github.event_name == 'pull_request' + steps: + - name: Create PR comment with deployment link + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + :rocket: Preview Deployed to GitHub Pages! You can view the site at [${{ steps.deployment.outputs.page_url }}](${{ steps.deployment.outputs.page_url }}) :rocket: \ No newline at end of file diff --git a/config.yaml b/config.yaml index 2ba7753..67642dc 100644 --- a/config.yaml +++ b/config.yaml @@ -17,7 +17,7 @@ hugo: # Content mapping - organized into 4 main categories content_mapping: - # TUTORIALS - Learning-oriented content + # TUTORIALS - Step-by-step guides for users tutorials: type: "tutorials" weight: 10 @@ -31,7 +31,7 @@ content_mapping: weight: 30 category: "Tutorials" - # HOW-TO GUIDES - Problem-solving oriented + # HOW-TO GUIDES - Practical instructions for specific tasks install: type: "how-to-guides" weight: 10 @@ -65,7 +65,7 @@ content_mapping: weight: 80 category: "How-To Guides" - # EXPLANATIONS - Understanding-oriented content + # EXPLANATIONS - In-depth articles explaining concepts and features concepts: type: "explanations" weight: 10 @@ -95,7 +95,7 @@ content_mapping: weight: 70 category: "Explanations" - # REFERENCE - Information-oriented content + # REFERENCE - Detailed reference material for advanced users reference: type: "reference" weight: 10 @@ -155,3 +155,146 @@ file_patterns: - "*.bzl" - ".git/**" - "node_modules/**" + +# Used to automatically detect language for code blocks without explicit language identifiers +code_language_patterns: + starlark: + - "cc_binary(" + - "cc_library(" + - "java_library(" + - "py_binary(" + - "py_library(" + - "load(" + - "BUILD" + - "WORKSPACE" + - "bazel-" + - "name =" + - "srcs =" + - "deps =" + bash: + - "bazel build" + - "bazel test" + - "bazel run" + - "bazel query" + - "#!/bin/bash" + - "#!/bin/sh" + - "echo " + - "export " + - "source " + - "$(" + - "${" + - "npm " + - "yarn " + - "git " + - "cd " + - "mkdir " + - "rm " + - "cp " + python: + - "def " + - "class " + - "import " + - "from " + - "print(" + - "__init__" + - "self." + - "@" + - "lambda " + - "return " + - "if __name__" + - "try:" + - "except:" + cpp: + - "#include" + - "int main" + - "std::" + - "namespace " + - "cout <<" + - "cin >>" + - "void " + - "template<" + - "nullptr" + - "using namespace" + - "printf(" + - "scanf(" + java: + - "public class" + - "private " + - "protected " + - "import java" + - "package " + - "extends " + - "implements " + - "@Override" + - "System.out.println" + - "new " + - "throw new" + - "catch (" + javascript: + - "function " + - "const " + - "let " + - "var " + - "=>" + - "console.log" + - "require(" + - "export " + - "import " + - "async " + - "await " + - "document." + - "window." + - "addEventListener" + typescript: + - "interface " + - "type " + - ": string" + - ": number" + - ": boolean" + - "enum " + - "readonly " + - "as " + yaml: + - "name:" + - "type:" + - "- name:" + - "kind:" + - "apiVersion:" + - "metadata:" + - "spec:" + json: + - '{"' + - '":' + - '": ' + - '[' + - '],' + xml: + - "" + - "xmlns" + html: + - "" + - " str: - """Add language identifiers to code blocks without them to prevent KaTeX rendering issues""" + """Use defaults in case config.yaml is not set""" + + language_patterns = self.config.get('code_language_patterns', { + 'starlark': [ + 'cc_binary(', 'cc_library(', 'java_library(', 'load(', + 'py_binary(', 'py_library(', 'BUILD', 'WORKSPACE', + 'cc_toolchain(', 'toolchain(', 'filegroup(', 'exports_files(' + ], + 'bash': [ + 'bazel build', 'bazel test', 'bazel run', '#!/bin/bash', + 'echo ', 'export ', 'source ', '$', 'npm ', 'yarn ' + ], + 'python': [ + 'def ', 'class ', 'import ', 'from ', 'print(', '__init__', + 'self.', '@', 'lambda ', 'return ' + ], + 'cpp': [ + '#include', 'int main', 'std::', 'namespace ', 'cout <<', + 'void ', 'template<', 'nullptr', '.cc', '.cpp', '.h' + ], + 'java': [ + 'public class', 'private ', 'protected ', 'import java', + 'package ', 'extends ', 'implements ', '@Override' + ], + 'javascript': [ + 'function ', 'const ', 'let ', 'var ', '=>', 'console.log', + 'require(', 'export ', 'import ', 'async ', 'await ' + ], + 'yaml': [ + 'name:', 'type:', '- name:', 'kind:', 'apiVersion:' + ], + 'json': [ + '{"', '":', '": ' + ] + }) def determine_language(code_content): - """Determine appropriate language identifier based on code content""" - code_lower = code_content.lower().strip() - - # Check for common patterns - if 'load(' in code_content or 'cc_library(' in code_content or 'java_library(' in code_content: - return 'starlark' # Bazel/Starlark (was incorrectly 'python') - elif code_content.startswith('/') or 'BUILD' in code_content: - return 'text' # File paths and directory structures - elif any(keyword in code_lower - for keyword in ['def ', 'class ', 'import ', 'from ']): - return 'python' - elif any(keyword in code_lower - for keyword in ['function', 'var ', 'const ', 'let ']): - return 'javascript' - elif any(keyword in code_lower - for keyword in ['#include', 'int main', 'std::']): - return 'cpp' - elif any(keyword in code_lower - for keyword in ['public class', 'import java']): - return 'java' - elif '$' in code_content or 'echo' in code_lower or code_content.startswith( - '#!/'): - return 'bash' + if not code_content or not code_content.strip(): + return 'text' + + # Check for directory structure indicators + if (code_content.strip().startswith('/') or + any(char in code_content for char in ['└', '├', '│'])): + return 'text' + + # Check language patterns + for language, patterns in language_patterns.items(): + for pattern in patterns: + if pattern in code_content: + return language + + return 'text' + + lines = content.splitlines(keepends=True) # Keep line endings + result = [] + in_code_block = False + in_unlabeled_block = False + code_lines = [] + + i = 0 + while i < len(lines): + line = lines[i] + stripped_line = line.rstrip() + + if not in_code_block: + # Not in a code block, check if this starts one + if stripped_line == '```': + # Starting an unlabeled code block + in_code_block = True + in_unlabeled_block = True + code_lines = [] + # Don't add this line yet, we'll reconstruct it with language + elif re.match(r'^```[\w-]', stripped_line): + # Starting a labeled code block, just pass it through + in_code_block = True + in_unlabeled_block = False + result.append(line) + else: + # Regular line + result.append(line) else: - return 'text' # Default for unknown content - - # Clean up any text that appears after closing backticks - # This ensures nothing ever appears after ``` - content = re.sub(r'```[^\n\r]*(\n|$)', '```\n', content) - - # Pattern to match code blocks that start with ``` followed by only whitespace and newline - # This ensures we only match code blocks WITHOUT language identifiers - pattern = r'```\s*\n(.*?)\n```' - - def replace_code_block(match): - code_content = match.group(1) + # We're in a code block + if stripped_line == '```': + # End of code block + in_code_block = False + + if in_unlabeled_block: + # This was an unlabeled block, process it + code_content = ''.join(code_lines).rstrip('\n\r') + language = determine_language(code_content) + + # Add the processed block + result.append(f'```{language}\n') + if code_content: + result.append(code_content + '\n') + result.append('```\n') + + code_lines = [] + in_unlabeled_block = False + else: + # This was a labeled block, just pass through the closing + result.append(line) + else: + # Content line within code block + if in_unlabeled_block: + # Collect this line for processing + code_lines.append(line) + else: + # Labeled block, just pass through + result.append(line) + + i += 1 + + # Handle case where file ends in middle of code block + if in_code_block and in_unlabeled_block: + code_content = ''.join(code_lines).rstrip('\n\r') language = determine_language(code_content) - return f'```{language}\n{code_content}\n```' - - # Apply the replacement using multiline and dotall flags - result = re.sub(pattern, - replace_code_block, - content, - flags=re.MULTILINE | re.DOTALL) + result.append(f'```{language}\n') + if code_content: + result.append(code_content + '\n') + result.append('```\n') - return result + return ''.join(result) def _generate_hugo_markdown(self, frontmatter: Dict, body: str) -> str: """Generate Hugo markdown file with frontmatter and body""" diff --git a/templates/hugo_config.yaml.jinja2 b/templates/hugo_config.yaml.jinja2 index 0c87c9c..98705bc 100644 --- a/templates/hugo_config.yaml.jinja2 +++ b/templates/hugo_config.yaml.jinja2 @@ -13,6 +13,13 @@ layoutDir: "layouts" staticDir: "static" archetypeDir: "archetypes" +# Cache configuration +caches: + images: + dir: :cacheDir/images + +math: false + # Build configuration buildDrafts: false buildFuture: false @@ -94,8 +101,6 @@ params: # Disable math rendering katex: enable: false - math: - enable: false # Disable MathJax mathjax: enable: false diff --git a/utils/css_converter.py b/utils/css_converter.py index 357eecc..467b27c 100644 --- a/utils/css_converter.py +++ b/utils/css_converter.py @@ -3,11 +3,10 @@ Handles conversion of CSS files to Hugo-compatible SCSS """ -import os import re import logging from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List logger = logging.getLogger(__name__) From 2530894717cc6234c7a1e98ac7595172adf5b1b6 Mon Sep 17 00:00:00 2001 From: Alan Mond Date: Sun, 27 Jul 2025 18:46:39 -0700 Subject: [PATCH 4/8] use config.yaml and remove defaults --- config.yaml | 18 ++- devsite_to_hugo_converter.py | 251 ++++++++++++++++++----------------- 2 files changed, 142 insertions(+), 127 deletions(-) diff --git a/config.yaml b/config.yaml index 67642dc..5f35ef1 100644 --- a/config.yaml +++ b/config.yaml @@ -171,7 +171,13 @@ code_language_patterns: - "name =" - "srcs =" - "deps =" + - "cc_toolchain(" + - "toolchain(" + - "filegroup(" + - "exports_files(" bash: + - "bazel " + - "bazel help" - "bazel build" - "bazel test" - "bazel run" @@ -182,7 +188,8 @@ code_language_patterns: - "export " - "source " - "$(" - - "${" + - "${" + - "$" - "npm " - "yarn " - "git " @@ -210,10 +217,13 @@ code_language_patterns: - "std::" - "namespace " - "cout <<" - - "cin >>" - "void " - "template<" - "nullptr" + - ".cc" + - ".cpp" + - ".h" + - "cin >>" - "using namespace" - "printf(" - "scanf(" @@ -267,7 +277,7 @@ code_language_patterns: - '":' - '": ' - '[' - - '],' + - '],' xml: - " str: - """Use defaults in case config.yaml is not set""" - - language_patterns = self.config.get('code_language_patterns', { - 'starlark': [ - 'cc_binary(', 'cc_library(', 'java_library(', 'load(', - 'py_binary(', 'py_library(', 'BUILD', 'WORKSPACE', - 'cc_toolchain(', 'toolchain(', 'filegroup(', 'exports_files(' - ], - 'bash': [ - 'bazel build', 'bazel test', 'bazel run', '#!/bin/bash', - 'echo ', 'export ', 'source ', '$', 'npm ', 'yarn ' - ], - 'python': [ - 'def ', 'class ', 'import ', 'from ', 'print(', '__init__', - 'self.', '@', 'lambda ', 'return ' - ], - 'cpp': [ - '#include', 'int main', 'std::', 'namespace ', 'cout <<', - 'void ', 'template<', 'nullptr', '.cc', '.cpp', '.h' - ], - 'java': [ - 'public class', 'private ', 'protected ', 'import java', - 'package ', 'extends ', 'implements ', '@Override' - ], - 'javascript': [ - 'function ', 'const ', 'let ', 'var ', '=>', 'console.log', - 'require(', 'export ', 'import ', 'async ', 'await ' - ], - 'yaml': [ - 'name:', 'type:', '- name:', 'kind:', 'apiVersion:' - ], - 'json': [ - '{"', '":', '": ' - ] - }) - - def determine_language(code_content): - if not code_content or not code_content.strip(): - return 'text' - - # Check for directory structure indicators - if (code_content.strip().startswith('/') or - any(char in code_content for char in ['└', '├', '│'])): - return 'text' - - # Check language patterns - for language, patterns in language_patterns.items(): - for pattern in patterns: - if pattern in code_content: - return language - - return 'text' - - lines = content.splitlines(keepends=True) # Keep line endings - result = [] - in_code_block = False - in_unlabeled_block = False - code_lines = [] - - i = 0 - while i < len(lines): - line = lines[i] - stripped_line = line.rstrip() - - if not in_code_block: - # Not in a code block, check if this starts one - if stripped_line == '```': - # Starting an unlabeled code block - in_code_block = True - in_unlabeled_block = True - code_lines = [] - # Don't add this line yet, we'll reconstruct it with language - elif re.match(r'^```[\w-]', stripped_line): - # Starting a labeled code block, just pass it through - in_code_block = True - in_unlabeled_block = False - result.append(line) - else: - # Regular line - result.append(line) - else: - # We're in a code block - if stripped_line == '```': - # End of code block - in_code_block = False - - if in_unlabeled_block: - # This was an unlabeled block, process it - code_content = ''.join(code_lines).rstrip('\n\r') - language = determine_language(code_content) - - # Add the processed block - result.append(f'```{language}\n') - if code_content: - result.append(code_content + '\n') - result.append('```\n') - - code_lines = [] - in_unlabeled_block = False - else: - # This was a labeled block, just pass through the closing - result.append(line) - else: - # Content line within code block - if in_unlabeled_block: - # Collect this line for processing - code_lines.append(line) - else: - # Labeled block, just pass through - result.append(line) - - i += 1 - - # Handle case where file ends in middle of code block - if in_code_block and in_unlabeled_block: - code_content = ''.join(code_lines).rstrip('\n\r') - language = determine_language(code_content) - result.append(f'```{language}\n') - if code_content: - result.append(code_content + '\n') - result.append('```\n') - - return ''.join(result) + """Annotate unlabeled code fences with language identifiers. + + + Parameters + ---------- + content : str + The markdown content possibly containing fenced code blocks. + + Returns + ------- + str + Content with unlabeled code blocks annotated with language + identifiers. + """ + # Fetch language detection patterns from configuration, or use + # sensible defaults. Keys are language names, values are + # sequences of substrings indicative of that language. + language_patterns: Dict[str, List[str]] = self.config.get('code_language_patterns') + + def determine_language(code_content: str) -> str: + """Return the best language guess for a code snippet. + + This helper inspects the provided code content for the + presence of known substrings. The first matching language + wins. If no patterns match, ``text`` is returned. + + Parameters + ---------- + code_content : str + Raw code content extracted from a fenced code block. + + Returns + ------- + str + Name of the detected language or ``'text'`` when + uncertain. + """ + if not code_content or not code_content.strip(): + return 'text' + # Treat directory structures as plain text (these may use + # ASCII/Unicode tree characters or start with a leading slash). + stripped_content = code_content.strip() + if stripped_content.startswith('/') or any( + char in code_content for char in ('└', '├', '│') + ): + return 'text' + # Check each configured language pattern in the order they + # appear. The first match determines the language. + for language, patterns in language_patterns.items(): + for pattern in patterns: + if pattern in code_content: + return language + return 'text' + + # Split the content into lines, preserving line endings so we + # can reconstruct the document accurately. + lines: List[str] = content.splitlines(keepends=True) + result: List[str] = [] + in_code_block = False + in_unlabeled_block = False + code_lines: List[str] = [] + fence_indent: str = '' + + idx = 0 + while idx < len(lines): + line = lines[idx] + stripped = line.strip() + # Detect the start of a code fence when not already in a block + if not in_code_block: + if stripped.startswith('```'): + after = stripped[3:].strip() + # Capture the indent of the fence (everything before + # the first non-whitespace character). This indent is + # reused when emitting annotated fences to preserve + # formatting inside lists. + fence_indent = line[: len(line) - len(line.lstrip())] + if after == '': + # Begin an unlabeled fenced code block + in_code_block = True + in_unlabeled_block = True + code_lines = [] + else: + # Begin a labeled fenced code block; copy the opening + # fence verbatim + in_code_block = True + in_unlabeled_block = False + result.append(line) + else: + result.append(line) + else: + # We're inside a fenced block + if stripped.startswith('```'): + # Encountered a closing fence + if in_unlabeled_block: + # Compute the language and emit annotated fence + code_content = ''.join(code_lines).rstrip('\n\r') + language = determine_language(code_content) + result.append(f'{fence_indent}```{language}\n') + if code_content: + result.append(code_content + '\n') + result.append(f'{fence_indent}```\n') + # Reset state + code_lines = [] + in_unlabeled_block = False + in_code_block = False + else: + # Labeled block; preserve closing fence + result.append(line) + in_code_block = False + else: + # Collect lines inside the fenced block + if in_unlabeled_block: + code_lines.append(line) + else: + result.append(line) + idx += 1 + + # Handle unterminated unlabeled blocks at EOF. If the file ends + # inside an unlabeled code block, annotate it anyway. + if in_code_block and in_unlabeled_block: + code_content = ''.join(code_lines).rstrip('\n\r') + language = determine_language(code_content) + result.append(f'{fence_indent}```{language}\n') + if code_content: + result.append(code_content + '\n') + result.append(f'{fence_indent}```\n') + + return ''.join(result) def _generate_hugo_markdown(self, frontmatter: Dict, body: str) -> str: """Generate Hugo markdown file with frontmatter and body""" From a574f4c1f535c960454185efd11ce37ee25429f8 Mon Sep 17 00:00:00 2001 From: Alan Mond Date: Sun, 27 Jul 2025 18:48:46 -0700 Subject: [PATCH 5/8] re-running deployment to gh pages From ec9fd423698e2e7c83d2f3f16d71be0c9666201a Mon Sep 17 00:00:00 2001 From: Alan Mond Date: Sun, 27 Jul 2025 18:50:22 -0700 Subject: [PATCH 6/8] create comment --- .github/workflows/hugo-render-gh-pages.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/hugo-render-gh-pages.yml b/.github/workflows/hugo-render-gh-pages.yml index f4d6218..e38c128 100644 --- a/.github/workflows/hugo-render-gh-pages.yml +++ b/.github/workflows/hugo-render-gh-pages.yml @@ -104,7 +104,6 @@ jobs: comment: runs-on: ubuntu-latest needs: deploy - if: github.event_name == 'pull_request' steps: - name: Create PR comment with deployment link uses: peter-evans/create-or-update-comment@v4 From f21e1e449847f2d64ea81cc446326214dfacd6b5 Mon Sep 17 00:00:00 2001 From: Alan Mond Date: Sun, 27 Jul 2025 18:55:05 -0700 Subject: [PATCH 7/8] on pull request --- .github/workflows/hugo-render-gh-pages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/hugo-render-gh-pages.yml b/.github/workflows/hugo-render-gh-pages.yml index e38c128..bc7ced8 100644 --- a/.github/workflows/hugo-render-gh-pages.yml +++ b/.github/workflows/hugo-render-gh-pages.yml @@ -2,8 +2,8 @@ name: Deploy Hugo site to Pages on: - push: - # prevent this workflow from running on pushes to the main branch + pull_request: + types: [opened, synchronize, reopened] branches-ignore: [main] # Run this workflow manually from the Actions tab workflow_dispatch: From 4f3f188543166e7593b880903f8f8b4fdf65c188 Mon Sep 17 00:00:00 2001 From: Alan Mond Date: Sun, 27 Jul 2025 22:23:48 -0700 Subject: [PATCH 8/8] additional cleanup --- .github/workflows/hugo-render-gh-pages.yml | 113 --------- config.yaml | 9 +- devsite_to_hugo_converter.py | 21 +- templates/hugo_config.yaml.jinja2 | 20 +- utils/css_converter.py | 282 --------------------- utils/hugo_generator.py | 155 +---------- 6 files changed, 23 insertions(+), 577 deletions(-) delete mode 100644 .github/workflows/hugo-render-gh-pages.yml delete mode 100644 utils/css_converter.py diff --git a/.github/workflows/hugo-render-gh-pages.yml b/.github/workflows/hugo-render-gh-pages.yml deleted file mode 100644 index bc7ced8..0000000 --- a/.github/workflows/hugo-render-gh-pages.yml +++ /dev/null @@ -1,113 +0,0 @@ -# Sample workflow for building and deploying a Hugo site to GitHub Pages -name: Deploy Hugo site to Pages - -on: - pull_request: - types: [opened, synchronize, reopened] - branches-ignore: [main] - # Run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - # allow write access to add a comment to the PR - pull-requests: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - -# Default to bash -defaults: - run: - # GitHub-hosted runners automatically enable `set -eo pipefail` for Bash shells. - shell: bash - -jobs: - # Build job - build: - runs-on: ubuntu-latest - env: - DART_SASS_VERSION: 1.89.2 - HUGO_VERSION: 0.146.0 - HUGO_ENVIRONMENT: production - TZ: America/Los_Angeles - steps: - - name: Install Hugo CLI - run: | - wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb - sudo dpkg -i ${{ runner.temp }}/hugo.deb - - name: Install Dart Sass - run: | - wget -O ${{ runner.temp }}/dart-sass.tar.gz https://github.com/sass/dart-sass/releases/download/${DART_SASS_VERSION}/dart-sass-${DART_SASS_VERSION}-linux-x64.tar.gz - tar -xf ${{ runner.temp }}/dart-sass.tar.gz --directory ${{ runner.temp }} - mv ${{ runner.temp }}/dart-sass/ /usr/local/bin - echo "/usr/local/bin/dart-sass" >> $GITHUB_PATH - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - name: Setup Pages - id: pages - uses: actions/configure-pages@v5 - - name: Install Node.js dependencies - run: "[[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true" - - name: Cache Restore - id: cache-restore - uses: actions/cache/restore@v4 - with: - path: | - ${{ runner.temp }}/hugo_cache - key: hugo-${{ github.run_id }} - restore-keys: - hugo- - - name: Configure Git - run: git config core.quotepath false - - name: Build with Hugo - run: | - hugo \ - --gc \ - --minify \ - --baseURL "${{ steps.pages.outputs.base_url }}/" \ - --cacheDir "${{ runner.temp }}/hugo_cache" - - name: Cache Save - id: cache-save - uses: actions/cache/save@v4 - with: - path: | - ${{ runner.temp }}/hugo_cache - key: ${{ steps.cache-restore.outputs.cache-primary-key }} - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: ./public - - # Deployment job - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 - -# Create a comment on the pull request with a link to the deployed site - comment: - runs-on: ubuntu-latest - needs: deploy - steps: - - name: Create PR comment with deployment link - uses: peter-evans/create-or-update-comment@v4 - with: - issue-number: ${{ github.event.pull_request.number }} - body: | - :rocket: Preview Deployed to GitHub Pages! You can view the site at [${{ steps.deployment.outputs.page_url }}](${{ steps.deployment.outputs.page_url }}) :rocket: \ No newline at end of file diff --git a/config.yaml b/config.yaml index 5f35ef1..85e7dee 100644 --- a/config.yaml +++ b/config.yaml @@ -1,11 +1,10 @@ # Configuration for Devsite to Hugo Converter -# Source repository settings +# Docsy Theme Configuration source_repo: - owner: "bazelbuild" - name: "bazel" - branch: "master" - path: "site/en" + owner: "alan707" + name: "bazel-docs" + branch: "main" # Hugo site settings hugo: diff --git a/devsite_to_hugo_converter.py b/devsite_to_hugo_converter.py index 7778981..f0596b0 100644 --- a/devsite_to_hugo_converter.py +++ b/devsite_to_hugo_converter.py @@ -11,7 +11,6 @@ import re from utils.devsite_parser import DevsiteParser from utils.hugo_generator import HugoGenerator -from utils.css_converter import CSSConverter # Configure logging logging.basicConfig( @@ -34,7 +33,6 @@ def __init__(self, config_path: str = "config.yaml"): self.config = self._load_config() self.devsite_parser = DevsiteParser(self.config) self.hugo_generator = HugoGenerator(self.config) - self.css_converter = CSSConverter(self.config) def _load_config(self) -> Dict: """Load configuration from YAML file""" @@ -92,15 +90,13 @@ def convert_documentation(self, devsite_structure, source_path, output_path, dry_run, incremental) - # Convert CSS and static assets + # Convert static assets if not dry_run: self._convert_assets(source_path, output_path) # Generate Hugo configuration if not dry_run: - self._generate_hugo_config(devsite_structure, output_path) - # Skip custom layouts when using Docsy theme - # self._generate_layouts(output_path) + self._generate_hugo_config(output_path) self._generate_section_indices(devsite_structure, output_path) logger.info(f"Conversion completed successfully") @@ -626,16 +622,10 @@ def _file_needs_conversion(self, source_file: Path, return source_file.stat().st_mtime > output_file.stat().st_mtime def _convert_assets(self, source_path: str, output_path: str) -> None: - """Convert CSS and static assets""" + """Convert static assets""" source_dir = Path(source_path) output_dir = Path(output_path) - # Convert CSS files - css_files = list(source_dir.rglob('*.css')) - for css_file in css_files: - self.css_converter.convert_css_file(css_file, - output_dir / 'assets' / 'scss') - # Copy static assets (images, etc.) static_extensions = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico'] for ext in static_extensions: @@ -646,10 +636,9 @@ def _convert_assets(self, source_path: str, output_path: str) -> None: shutil.copy2(asset_file, output_asset) logger.debug(f"Copied asset: {asset_file} -> {output_asset}") - def _generate_hugo_config(self, devsite_structure: Dict, - output_path: str) -> None: + def _generate_hugo_config(self, output_path: str) -> None: """Generate Hugo configuration file""" - self.hugo_generator.generate_config(devsite_structure, output_path) + self.hugo_generator.generate_config(output_path) def _generate_layouts(self, output_path: str) -> None: """Generate Hugo layout templates""" diff --git a/templates/hugo_config.yaml.jinja2 b/templates/hugo_config.yaml.jinja2 index 98705bc..33c3253 100644 --- a/templates/hugo_config.yaml.jinja2 +++ b/templates/hugo_config.yaml.jinja2 @@ -1,10 +1,10 @@ # Hugo configuration for Bazel documentation site # Generated automatically by devsite-to-hugo-converter -baseURL: "{{ config.baseURL }}" -title: "{{ config.title }}" -description: "{{ config.description }}" -languageCode: "{{ config.languageCode }}" +baseURL: "{{ hugo.baseURL }}" +title: "{{ hugo.title }}" +description: "{{ hugo.description }}" +languageCode: "{{ hugo.languageCode }}" # Content configuration contentDir: "content" @@ -81,14 +81,13 @@ params: # Repository information github_repo: "https://github.com/{{ source_repo.owner }}/{{ source_repo.name }}" github_branch: "{{ source_repo.branch }}" - github_subdir: "{{ source_repo.path }}" # Edit page configuration edit_page: true # Search configuration search: - enabled: true + enabled: false # Navigation configuration navigation: @@ -96,7 +95,7 @@ params: # Footer configuration footer: - enable: true + enable: false # Disable math rendering katex: @@ -127,9 +126,10 @@ taxonomies: # Output formats outputs: - home: ["HTML", "RSS", "JSON"] - page: ["HTML"] - section: ["HTML", "RSS"] + section: + - HTML + - RSS + # Security configuration security: diff --git a/utils/css_converter.py b/utils/css_converter.py deleted file mode 100644 index 467b27c..0000000 --- a/utils/css_converter.py +++ /dev/null @@ -1,282 +0,0 @@ -""" -CSS Converter Module -Handles conversion of CSS files to Hugo-compatible SCSS -""" - -import re -import logging -from pathlib import Path -from typing import Dict, List - -logger = logging.getLogger(__name__) - -class CSSConverter: - """Converter for CSS files to Hugo-compatible SCSS""" - - def __init__(self, config: Dict): - """Initialize converter with configuration""" - self.config = config - self.conversion_config = config.get('css_conversion', {}) - - def convert_css_file(self, css_file: Path, output_dir: Path) -> bool: - """ - Convert a CSS file to SCSS format - - Args: - css_file: Path to source CSS file - output_dir: Output directory for SCSS files - - Returns: - True if successful, False otherwise - """ - try: - # Read source CSS - with open(css_file, 'r', encoding='utf-8') as f: - css_content = f.read() - - # Convert to SCSS - scss_content = self._convert_css_to_scss(css_content) - - # Determine output file path - output_file = output_dir / f"_{css_file.stem}.scss" - output_file.parent.mkdir(parents=True, exist_ok=True) - - # Write SCSS file - with open(output_file, 'w', encoding='utf-8') as f: - f.write(scss_content) - - logger.debug(f"Converted CSS to SCSS: {css_file} -> {output_file}") - return True - - except Exception as e: - logger.error(f"Failed to convert CSS file {css_file}: {e}") - return False - - def _convert_css_to_scss(self, css_content: str) -> str: - """Convert CSS content to SCSS format""" - scss_content = css_content - - # Add SCSS header comment - header = """// Converted from Devsite CSS to SCSS for Hugo/Docsy -// Generated automatically by devsite-to-hugo-converter - -""" - - # Convert CSS custom properties to SCSS variables if configured - if self.conversion_config.get('preserve_custom_properties', True): - scss_content = self._convert_custom_properties(scss_content) - - # Add vendor prefixes handling - scss_content = self._add_vendor_prefix_mixins(scss_content) - - # Convert color values to SCSS variables - scss_content = self._extract_color_variables(scss_content) - - # Convert font definitions to SCSS variables - scss_content = self._extract_font_variables(scss_content) - - # Add responsive breakpoint support - scss_content = self._add_responsive_mixins(scss_content) - - return header + scss_content - - def _convert_custom_properties(self, css_content: str) -> str: - """Convert CSS custom properties to SCSS variables""" - # Find all CSS custom properties - custom_prop_pattern = r'--([a-zA-Z0-9-]+):\s*([^;]+);' - custom_props = re.findall(custom_prop_pattern, css_content) - - if not custom_props: - return css_content - - # Generate SCSS variables section - scss_vars = "\n// SCSS Variables (converted from CSS custom properties)\n" - for prop_name, prop_value in custom_props: - scss_var_name = prop_name.replace('-', '_') - scss_vars += f"${scss_var_name}: {prop_value.strip()};\n" - - # Replace CSS custom property usage with SCSS variables - modified_content = css_content - for prop_name, _ in custom_props: - css_var_usage = f"var(--{prop_name})" - scss_var_usage = f"${prop_name.replace('-', '_')}" - modified_content = modified_content.replace(css_var_usage, scss_var_usage) - - return scss_vars + "\n" + modified_content - - def _add_vendor_prefix_mixins(self, scss_content: str) -> str: - """Add SCSS mixins for vendor prefixes""" - mixins = """ -// Vendor prefix mixins -@mixin transform($value) { - -webkit-transform: $value; - -moz-transform: $value; - -ms-transform: $value; - transform: $value; -} - -@mixin transition($value) { - -webkit-transition: $value; - -moz-transition: $value; - -ms-transition: $value; - transition: $value; -} - -@mixin box-shadow($value) { - -webkit-box-shadow: $value; - -moz-box-shadow: $value; - box-shadow: $value; -} - -@mixin border-radius($value) { - -webkit-border-radius: $value; - -moz-border-radius: $value; - border-radius: $value; -} - -""" - return mixins + scss_content - - def _extract_color_variables(self, scss_content: str) -> str: - """Extract color values and convert to SCSS variables""" - # Find color values (hex, rgb, rgba, hsl, hsla) - color_patterns = [ - r'#[0-9a-fA-F]{3,6}', - r'rgb\([^)]+\)', - r'rgba\([^)]+\)', - r'hsl\([^)]+\)', - r'hsla\([^)]+\)' - ] - - colors = set() - for pattern in color_patterns: - colors.update(re.findall(pattern, scss_content)) - - if not colors: - return scss_content - - # Generate color variables - color_vars = "\n// Color variables\n" - color_map = {} - - for i, color in enumerate(sorted(colors)): - var_name = f"$color-{i + 1}" - color_vars += f"{var_name}: {color};\n" - color_map[color] = var_name - - # Replace color values with variables - modified_content = scss_content - for color, var_name in color_map.items(): - modified_content = modified_content.replace(color, var_name) - - return color_vars + "\n" + modified_content - - def _extract_font_variables(self, scss_content: str) -> str: - """Extract font definitions and convert to SCSS variables""" - # Find font-family declarations - font_pattern = r'font-family:\s*([^;]+);' - fonts = set(re.findall(font_pattern, scss_content)) - - if not fonts: - return scss_content - - # Generate font variables - font_vars = "\n// Font variables\n" - font_map = {} - - for i, font in enumerate(sorted(fonts)): - var_name = f"$font-family-{i + 1}" - font_vars += f"{var_name}: {font};\n" - font_map[font] = var_name - - # Replace font declarations with variables - modified_content = scss_content - for font, var_name in font_map.items(): - modified_content = modified_content.replace(f"font-family: {font};", f"font-family: {var_name};") - - return font_vars + "\n" + modified_content - - def _add_responsive_mixins(self, scss_content: str) -> str: - """Add responsive breakpoint mixins""" - responsive_mixins = """ -// Responsive breakpoint mixins -$breakpoints: ( - mobile: 576px, - tablet: 768px, - desktop: 992px, - large: 1200px -); - -@mixin respond-to($breakpoint) { - @if map-has-key($breakpoints, $breakpoint) { - @media (min-width: map-get($breakpoints, $breakpoint)) { - @content; - } - } @else { - @warn "Unknown breakpoint: #{$breakpoint}."; - } -} - -@mixin respond-below($breakpoint) { - @if map-has-key($breakpoints, $breakpoint) { - @media (max-width: map-get($breakpoints, $breakpoint) - 1px) { - @content; - } - } @else { - @warn "Unknown breakpoint: #{$breakpoint}."; - } -} - -""" - return responsive_mixins + scss_content - - def convert_devsite_styles(self, source_dir: Path, output_dir: Path) -> bool: - """ - Convert all Devsite CSS files to Hugo-compatible SCSS - - Args: - source_dir: Source directory containing CSS files - output_dir: Output directory for SCSS files - - Returns: - True if successful, False otherwise - """ - try: - css_files = list(source_dir.rglob('*.css')) - - if not css_files: - logger.info("No CSS files found to convert") - return True - - scss_dir = output_dir / 'scss' - scss_dir.mkdir(parents=True, exist_ok=True) - - success_count = 0 - for css_file in css_files: - if self.convert_css_file(css_file, scss_dir): - success_count += 1 - - # Generate main SCSS file that imports all converted files - self._generate_main_scss(scss_dir, css_files) - - logger.info(f"Converted {success_count}/{len(css_files)} CSS files to SCSS") - return success_count == len(css_files) - - except Exception as e: - logger.error(f"Failed to convert Devsite styles: {e}") - return False - - def _generate_main_scss(self, scss_dir: Path, css_files: List[Path]) -> None: - """Generate main SCSS file that imports all converted files""" - main_scss = "// Main SCSS file for Bazel documentation\n" - main_scss += "// Imports all converted Devsite CSS files\n\n" - - for css_file in css_files: - import_name = css_file.stem - main_scss += f"@import '{import_name}';\n" - - main_scss_file = scss_dir / '_main.scss' - with open(main_scss_file, 'w', encoding='utf-8') as f: - f.write(main_scss) - - logger.debug(f"Generated main SCSS file: {main_scss_file}") diff --git a/utils/hugo_generator.py b/utils/hugo_generator.py index 58a37e4..bd37a45 100644 --- a/utils/hugo_generator.py +++ b/utils/hugo_generator.py @@ -25,7 +25,7 @@ def __init__(self, config: Dict): self.config = config self.template_env = Environment(loader=FileSystemLoader('templates')) - def generate_config(self, devsite_structure: Dict, output_path: str) -> bool: + def generate_config(self, output_path: str) -> bool: """ Generate Hugo configuration file @@ -38,14 +38,9 @@ def generate_config(self, devsite_structure: Dict, output_path: str) -> bool: """ try: output_dir = Path(output_path) - - # Prepare template context - context = { - 'config': self.config['hugo'], - 'source_repo': self.config['source_repo'], - 'devsite_structure': devsite_structure - } - + + with open(Path('config.yaml'), 'r') as f: + context = yaml.safe_load(f) # Render Hugo configuration template = self.template_env.get_template('hugo_config.yaml.jinja2') config_content = template.render(context) @@ -393,8 +388,6 @@ def _generate_base_layout(self, layouts_dir: Path) -> None: {{ .Title }} | {{ .Site.Title }} - - {{ $style := resources.Get "scss/main.scss" | resources.ToCSS | resources.Minify }} @@ -521,127 +514,6 @@ def _generate_shortcodes(self, layouts_dir: Path) -> None: with open(toc_file, 'w', encoding='utf-8') as f: f.write(toc_content) - def _generate_main_scss(self, output_dir: Path) -> None: - """Generate main SCSS file that imports Bazel styles""" - scss_dir = output_dir / 'assets' / 'scss' - scss_dir.mkdir(parents=True, exist_ok=True) - - # Create main.scss that imports the Bazel styles - main_scss_content = '''// Main SCSS file for Hugo site -// Imports Bazel-specific styles - -@import "bazel"; - -// Additional site-wide styles -body { - font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; - line-height: 1.6; - margin: 0; - padding: 0; -} - -.content { - max-width: 1200px; - margin: 0 auto; - padding: 20px; -} - -header { - background-color: $color-2; - color: $color-5; - padding: 1rem 0; -} - -.header-content { - max-width: 1200px; - margin: 0 auto; - padding: 0 20px; -} - -.logo { - color: $color-5; - text-decoration: none; - font-size: 1.5rem; - font-weight: bold; -} - -nav { - background-color: $color-3; - padding: 0.5rem 0; -} - -nav a { - color: $color-5; - text-decoration: none; - margin: 0 1rem; - padding: 0.5rem 0; -} - -nav a:hover { - text-decoration: underline; -} - -footer { - background-color: $color-4; - padding: 2rem 0; - margin-top: 3rem; - text-align: center; -} - -.section-list { - list-style: none; - padding: 0; -} - -.section-list li { - margin: 1rem 0; - padding: 1rem; - border: 1px solid #e1e4e8; - border-radius: 6px; -} - -.section-list a { - color: $color-2; - text-decoration: none; - font-weight: bold; -} - -.section-list a:hover { - text-decoration: underline; -} - -.section-grid { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); - gap: 2rem; - margin: 2rem 0; -} - -.section-card { - padding: 1.5rem; - border: 1px solid #e1e4e8; - border-radius: 6px; - background: $color-5; -} - -.section-card h3 { - margin-top: 0; -} - -.section-card a { - color: $color-2; - text-decoration: none; -} - -.section-card a:hover { - text-decoration: underline; -} -''' - - main_scss_file = scss_dir / 'main.scss' - with open(main_scss_file, 'w', encoding='utf-8') as f: - f.write(main_scss_content) - def generate_menu_configuration(self, devsite_structure: Dict) -> Dict: """Generate Hugo menu configuration from Devsite structure""" menu_config = { @@ -658,22 +530,3 @@ def generate_menu_configuration(self, devsite_structure: Dict) -> Dict: menu_config['main'].append(menu_item) return menu_config - - def generate_params_configuration(self, devsite_structure: Dict) -> Dict: - """Generate Hugo params configuration for Docsy theme""" - params = { - 'github_repo': f"https://github.com/{self.config['source_repo']['owner']}/{self.config['source_repo']['name']}", - 'github_branch': self.config['source_repo']['branch'], - 'github_subdir': self.config['source_repo']['path'], - 'edit_page': True, - 'search': {'enabled': True}, - 'navigation': {'depth': 4}, - 'footer': {'enable': True}, - 'taxonomy': { - 'taxonomyCloud': ['tags', 'categories'], - 'taxonomyCloudTitle': ['Tag Cloud', 'Categories'], - 'taxonomyPageHeader': ['tags', 'categories'] - } - } - - return params