diff --git a/build/components/component.py b/build/components/component.py index 2e895e28e9..23396a792c 100644 --- a/build/components/component.py +++ b/build/components/component.py @@ -273,6 +273,11 @@ def _copy_examples(self): example_metadata['sourceUrl'] = ( f'{ex["git_uri"]}/tree/{default_branch}/{ex["path"]}/{os.path.basename(f)}' ) + + # Add binderId only if it exists + if e.binder_id: + example_metadata['binderId'] = e.binder_id + examples = self._root._examples if example_id not in examples: examples[example_id] = {} diff --git a/build/components/example.py b/build/components/example.py index c64da687df..5b8841067d 100644 --- a/build/components/example.py +++ b/build/components/example.py @@ -8,6 +8,7 @@ STEP_START = 'STEP_START' STEP_END = 'STEP_END' EXAMPLE = 'EXAMPLE:' +BINDER_ID = 'BINDER_ID' GO_OUTPUT = 'Output:' TEST_MARKER = { 'java': '@Test', @@ -45,6 +46,7 @@ class Example(object): hidden = None highlight = None named_steps = None + binder_id = None def __init__(self, language: str, path: str) -> None: logging.debug("ENTERING: ") @@ -59,6 +61,7 @@ def __init__(self, language: str, path: str) -> None: self.hidden = [] self.highlight = [] self.named_steps = {} + self.binder_id = None self.make_ranges() self.persist(self.path) logging.debug("EXITING: ") @@ -88,6 +91,7 @@ def make_ranges(self) -> None: rstart = re.compile(f'{PREFIXES[self.language]}\\s?{REMOVE_START}') rend = re.compile(f'{PREFIXES[self.language]}\\s?{REMOVE_END}') exid = re.compile(f'{PREFIXES[self.language]}\\s?{EXAMPLE}') + binder = re.compile(f'{PREFIXES[self.language]}\\s?{BINDER_ID}\\s+([a-zA-Z0-9_-]+)') go_output = re.compile(f'{PREFIXES[self.language]}\\s?{GO_OUTPUT}') go_comment = re.compile(f'{PREFIXES[self.language]}') test_marker = re.compile(f'{TEST_MARKER.get(self.language)}') @@ -150,6 +154,13 @@ def make_ranges(self) -> None: elif re.search(exid, l): output = False pass + elif re.search(binder, l): + # Extract BINDER_ID hash value + match = re.search(binder, l) + if match: + self.binder_id = match.group(1) + logging.debug(f'Found BINDER_ID: {self.binder_id} in {self.path}:L{curr+1}') + output = False elif self.language == "go" and re.search(go_output, l): if output: logging.error("Nested Go Output anchor in {self.path}:L{curr+1} - aborting.") diff --git a/build/local_examples.py b/build/local_examples.py index 46d50d2e37..9d6bdd3130 100644 --- a/build/local_examples.py +++ b/build/local_examples.py @@ -180,6 +180,10 @@ def process_local_examples(local_examples_dir: str = 'local_examples', 'sourceUrl': None # Local examples don't have source URLs } + # Add binderId only if it exists + if example.binder_id: + example_metadata['binderId'] = example.binder_id + examples_data[example_id][client_name] = example_metadata logging.info(f"Processed {client_name} example for {example_id}") diff --git a/build/tcedocs/README.md b/build/tcedocs/README.md new file mode 100644 index 0000000000..ddc22a6d65 --- /dev/null +++ b/build/tcedocs/README.md @@ -0,0 +1,240 @@ +# How to add a multi-language code examples to redis.io + +## Configure Hugo + +The website redis.io is built from Markdown files using [Hugo](https://gohugo.io). Multi-language code example support is configured in Hugo by adding information to its configuration file, `config.toml`. +There are two sections that need to updated when new languages are added. + +1. In the `[params]` section: + + ```toml + clientsExamples = ["Python", "Node.js", "Java-Sync", "Java-Async", "Java-Reactive", "Go", "C#", "RedisVL", "PHP"] + ``` + + The order of the `clientsExamples` list matters: it's the order in which the language tabs are presented for each code example. +1. In the `[params.clientsConfig]` section: + + ```toml + [params.clientsConfig] + "Python"={quickstartSlug="redis-py"} + "Node.js"={quickstartSlug="nodejs"} + "Java-sync"={quickstartSlug="jedis"} + "Java-async"={quickstartSlug="lettuce"} + "Java-reactive"={quickstartSlug="lettuce"} + "Go"={quickstartSlug="go"} + "C#"={quickstartSlug="dotnet"} + "RedisVL"={quickstartSlug="redis-vl"} + "PHP"={quickstartSlug="php"} + ``` + +This configuration, along with the configuration steps below, is used to control the behavior of the Hugo shortcode that was developed to show tabbed code examples. +A shortcode is a simple snippet inside a content file that Hugo will render using a predefined template. This template can contain HTML and JavaScript. + +### How to add a new programming language + +#### Add the components file + +The folder `data/components` contains one component configuration file for each supported language. These files contain information about the GitHub repos that house the code examples. + +Here is the configuration file for Python, `redis_py.json`: + +```json +{ + "id": "redis_py", + "type": "client", + "name": "redis-py", + "language": "Python", + "label": "Python", + "repository": { + "git_uri": "https://github.com/redis/redis-py" + }, + "examples": { + "git_uri": "https://github.com/redis/redis-py", + "path": "doctests", + "pattern": "*.py" + } +} +``` + +The `language` property needs to match the value that was added to the `config.toml` file in the previous step. The `label` property, while generally the same as `language`, may be set to a string that is different from `language`. For RedisVL, `language` is set to `Python` and `label` is set to `RedisVL`. The `examples` property points to a GitHub repository, a path under which examples should be searched, and a file name pattern. The current logic will scan for examples that fulfill the filename pattern within the given path. + +#### Register the component file + +Register your component file by adding it to the `clients` array in the `index.json` file, which resides in the the same folder as the per-language JSON files. The entry should match the file name prefix and ID of the component. + +Here is an example: +```json +"clients": [ + "nredisstack", + "go_redis", + "node_redis", + "php", + "redis_py", + "jedis", + "lettuce_async", + "lettuce_reactive", + "redis_vl" +] +``` + +Code examples are pulled from the GitHub repo for each supported language at docs site build time. + +### Verify that your language is supported by the source code file parser + +Component handling is implemented in `build/components/component.py`. The example file parser that is used by it is implemented inside `build/components/example.py`. Add any language-specific information you need to have the build code support your language's examples. + +```python +TEST_MARKER = { + 'java': '@Test', + 'java-sync': '@Test', + 'java-async': '@Test', + 'java-reactive': '@Test', + 'c#': r'\[Fact]|\[SkipIfRedis\(.*\)]' +} +PREFIXES = { + 'python': '#', + 'node.js': '//', + 'java': '//', + 'java-sync': '//', + 'java-async': '//', + 'java-reactive': '//', + 'go': '//', + 'c#': '//', + 'redisvl': '#', + 'php': '//' +} +``` + +The `TEST_MARKER` dictionary maps programming languages to test framework annotations, which allows the parser to filter such source code lines out. The `PREFIXES` dictionary maps each language to its comment prefix. Python, for example, uses a hashtag (`#`) to start a comment. + +## Understand special comments in the example source code files + +Each code example uses special comments, such as `HIDE_START` and `REMOVE_START`, to control how the examples are displayed. The following list gives an explanation: + +- `EXAMPLE: id`: Defines the identifier of the source code example file, where `id` is any common string (for example, `cmds_string`). IDs should only contain ASCII alphanumeric characters, underline characters (`_`), or hyphen characters (`-`). Do not use multibyte characters. +- `BINDER_ID id`: Defines the [BinderHub](https://binderhub.readthedocs.io/en/latest/) commit hash for the example. This is used to generate a link to a BinderHub instance that will run the example. +- `HIDE_START`: Starts a code block that should be *hidden* when showing the example. This code block will only become visible if **unhide** (the eye button) is clicked. +- `HIDE_END`: Marks the end a hidden code block. +- `REMOVE_START`: Starts a code block that should be entirely removed when the example is processed by the build code. This is useful for removing lines of code that do not contribute to the example but are needed to embed the code into a proper test case or framework. Good examples of such code blocks are imports of external libraries or test assertions. +- `REMOVE_END`: Marks the end of a code block that should be removed from the example. +- `STEP_START step-name`: Starts a code block that represents a specific step in a set of examples. +- `STEP_END`: Marks the end of a code block that represents a specific step in a set of examples. + +## Add examples to the client library or to the local_examples directory + +Examples are added to either a client repo, or, temporarily, to the `local_examples` directory in the `redis.io/docs` repo. + +### Add examples to the client libraries + +Add a source code file to an appropriate client repo. Consult the /data/components/.json file for the location. + +| Programming Language | GitHub Repo | Default directory | +|----------------------|-----------------------------------------------------|---------------------------------------------------| +| C# | [NRedisStack](https://github.com/redis/NRedisStack) | `tests/Doc` | +| Go | [go-redis](https://github.com/redis/go-redis) | `doctests` | +| Java | [jedis](https://github.com/redis/jedis) | `src/test/java/io/redis/examples` | +| | [Lettuce](https://github.com/redis/lettuce) | `src/test/java/io/redis/examples/async` or | +| | | `src/test/java/io/redis/examples/reactive` | +| Node.js | [node-redis](https://github.com/redis/node-redis) | `doctests` | +| PHP | [Predis](https://github.com/predis/predis) | Examples, for now, are stored in `local_examples` | +| Python | [redis-py](https://github.com/redis/redis-py) | `doctests` | +| | [RedisVL](https://github.com/redis/redis-vl-python) | `doctests` | + +### Add examples to the local_examples directory + +At times, it can take quite a while to get new or updated examples through the review process. To make the examples available immediately on the docs site, you can place examples temporarily in the `local_examples/client-specific` directory. The manner in which files are added isn't terribly important, as the build code will recursively walk the entire directory, so it will find examples in any directory under `local_examples`. + +``` +local_examples +├── client-specific +│   ├── go +│   │   ... +│   ├── jedis +│   │   ... +│   ├── lettuce-async +│   │   ... +│   ├── lettuce-reactive +│   │   ... +│   ├── nodejs +│   │   ... +│   └── redis-py +│   ... +``` + +## Add your example to the content page + +In order to add a multi-language code example to a content page, use the `clients-example` shortcode: + +``` +{{< clients-example id ... />}} +``` + +The ID is the same one you used with `EXAMPLE: id` in the first line of your code example. + +### Named versus positional parameters + +The `clients-example` shortcode supports both positional and named parameters. The lion's share of current examples use positional parameters, but, going forward, names parameters should be used. + +Named parameters: + +- set: Name of the example set (required) +- step: Example step name (required) +- lang_filter: Language filter (optional, default: "") +- max_lines: Maximum number of lines shown by default (optional, default: 100) +- dft_tab_name: Custom first tab name (optional, default: ">_ Redis CLI") +- dft_tab_link_title: Custom first tab footer link title (optional) +- dft_tab_url: Custom first tab footer link URL (optional) +- show_footer: Show footer (optional, default: true) + +Positional parameters (for backward compatibility): + +- 0: example set name +- 1: step name +- 2: language filter +- 3: max lines +- 4: custom first tab name +- 5: custom first tab footer link title +- 6: custom first tab footer link URL + +### Examples + +When converting existing content with redis-cli examples to the new format, you can wrap the existing redis-cli example: + +``` +{{< clients-example set="set_and_get" step="" >}} +> set mykey somevalue +OK +> get mykey +"somevalue" +{{< /clients-example >}} +``` + +If the redis-cli example is too long you can hide some lines by specifying the limit as the fourth argument: + +``` +{{< clients-example set="set_and_get" step="" lang_filter="" max_lines="2" >}} +> set mykey somevalue +OK +> get mykey <-- this line will be hidden +"somevalue" <-- this line will be hidden +{{< /clients-example >}} +``` + +To refer to a particular step placed in between `STEP_START stepname` and `STEP_END` comments in the code example, you should use the second argument to define the name of the step: + +``` +{{< clients-example set="id" step="stepname" />}} +``` + +If you need to embed an example for a specific programming language, the third argument should be defined: + +``` +{{< clients-example set="id" step="stepname" lang_filter="lang" />}} +``` + +The following example shows the `connect` step of a Python example: + +``` +{{< clients-example set="set_and_get" step="connect" lang_filter="Python" />}} +``` +The programming language name should match the value in the Hugo configuration file. diff --git a/build/tcedocs/SPECIFICATION.md b/build/tcedocs/SPECIFICATION.md new file mode 100644 index 0000000000..aacb7dc8b4 --- /dev/null +++ b/build/tcedocs/SPECIFICATION.md @@ -0,0 +1,2140 @@ +# Code Example System - Technical Specification + +> **For Documentation Authors**: See `build/tcedocs/README.md` for user-facing documentation on writing examples. + +## Document Purpose + +This specification is for developers who need to: +- **Understand** how the code example system works +- **Maintain** the build scripts and templates +- **Extend** the system (add new languages, modify UI, etc.) +- **Debug** issues with example processing or rendering + +**Not covered**: Line-by-line code walkthrough, Hugo basics, JavaScript implementation details. + +## Quick Navigation + +**I want to...** +- Understand the system → [System Overview](#system-overview), [Architecture](#architecture) +- Add a new example → [Working with Examples](#working-with-examples) +- Add a new language → [Extension Points](#extension-points), [Appendix: Adding a Language](#adding-a-language) +- Fix a build issue → [Troubleshooting](#troubleshooting) +- Understand the build → [Build Process](#build-process) +- Find configuration → [Configuration](#configuration) + +## Table of Contents + +1. [System Overview](#system-overview) +2. [Architecture](#architecture) +3. [Key Components](#key-components) +4. [File Structure and Conventions](#file-structure-and-conventions) +5. [Configuration](#configuration) +6. [Working with Examples](#working-with-examples) +7. [Extension Points](#extension-points) +8. [Build Process](#build-process) +9. [Troubleshooting](#troubleshooting) +10. [Appendix](#appendix) + +--- + +## System Overview + +### Purpose + +The code example system provides a multi-language, tabbed code example interface for the Redis documentation site. It allows documentation authors to embed **executable, tested code examples** from multiple programming languages in a single, unified interface with language-specific tabs. + +**Critical Design Principle**: All examples are actual test code from client library repositories or local test files. This ensures examples are always valid, executable, and tested against real Redis instances. + +### Key Features + +- **Multi-language support**: Display the same example in multiple programming languages +- **Interactive execution**: "Run in browser" links via BinderHub integration (Jupyter notebooks supporting multiple languages) +- **Tabbed interface**: Users can switch between languages using a dropdown selector +- **Code hiding/highlighting**: Support for hiding boilerplate code and highlighting relevant sections +- **Named steps**: Break examples into logical steps that can be referenced individually +- **Remote and local examples**: Pull examples from client library repositories or use local examples +- **Syntax highlighting**: Automatic syntax highlighting based on language +- **Source linking**: Link back to the original source code in GitHub repositories + +### Remote vs Local Examples + +**Remote Examples** (Preferred): +- Pulled from client library repositories (e.g., `redis-py/doctests/`) +- Automatically updated when client libraries release new versions +- Include GitHub source links for contributions +- **Use when**: Example is stable and part of client library test suite + +**Local Examples** (`local_examples/`): +- Stored directly in the docs repository +- Faster iteration during development +- No GitHub source links +- **Use when**: + - Example is under active development + - Waiting for client library PR approval + - Example is docs-specific and doesn't belong in client library + - Need to quickly fix or update an example + +**Important**: Local examples should eventually migrate to client repositories when stable. + +--- + +## Architecture + +### High-Level Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Build Process │ +│ │ +│ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ Remote Examples │ │ Local Examples │ │ +│ │ (GitHub Repos) │ │ (local_examples/)│ │ +│ └────────┬─────────┘ └────────┬─────────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌────────────────────────────────────────────┐ │ +│ │ build/make.py (Orchestrator) │ │ +│ │ - Calls component.py for remote examples │ │ +│ │ - Calls local_examples.py for local │ │ +│ └────────────────┬───────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────┐ │ +│ │ Example Processing (example.py) │ │ +│ │ - Parse special comments │ │ +│ │ - Extract steps, hide/remove blocks │ │ +│ │ - Generate metadata │ │ +│ └────────────────┬───────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────┐ │ +│ │ Output │ │ +│ │ - examples/ (processed code files) │ │ +│ │ - data/examples.json (metadata) │ │ +│ └────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Hugo Rendering │ +│ │ +│ ┌────────────────────────────────────────────┐ │ +│ │ Documentation Pages (Markdown) │ │ +│ │ {{< clients-example set="..." />}} │ │ +│ └────────────────┬───────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────┐ │ +│ │ Shortcode (clients-example.html) │ │ +│ │ - Parse parameters │ │ +│ │ - Call partial template │ │ +│ └────────────────┬───────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────┐ │ +│ │ Partial (tabbed-clients-example.html) │ │ +│ │ - Load examples.json │ │ +│ │ - Generate tabs for each language │ │ +│ │ - Apply syntax highlighting │ │ +│ └────────────────┬───────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────┐ │ +│ │ HTML Output (Interactive Tabs) │ │ +│ └────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Component Interaction Flow + +The system operates in three distinct phases: + +**1. Build Time (Python)** - Processes example source code: +- Clones remote repositories or reads local files +- Parses special comment markers (`EXAMPLE:`, `HIDE_START`, etc.) +- Removes test framework code and boilerplate +- Generates metadata (line ranges for highlighting, hidden sections, steps) +- Writes processed files to `examples/` directory +- Creates/updates `data/examples.json` with metadata for all examples + +**2. Hugo Build Time (Go Templates)** - Renders HTML: +- Reads `data/examples.json` metadata +- Processes `{{< clients-example >}}` shortcodes in Markdown files +- Loads processed example files from `examples/` directory +- Applies syntax highlighting +- Generates tabbed HTML interface with language selector + +**3. Browser Runtime (JavaScript)** - Handles interactivity: +- Tab switching between languages +- Show/hide hidden code sections +- Copy code to clipboard +- Persist language preference across page loads + +**Key Insight**: The Python build phase does the heavy lifting (parsing, processing), while Hugo simply renders pre-processed files. This separation allows Hugo to remain fast even with hundreds of examples. + +### Important Behaviors + +**Metadata Merging**: +- If the same example ID exists in both remote and local sources, both are included in `examples.json` +- Each language variant is stored separately (e.g., `"Python"`, `"Node.js"`) +- Local examples can supplement or override remote examples for specific languages +- Example: Remote has Python/Node.js, local adds Go → final result has all three + +**Generated Files** (gitignored): +- `examples/` directory - processed code files +- `data/examples.json` - metadata for all examples +- These are regenerated on every build and should not be committed + +**In-Place Processing**: +- The `Example` class modifies files in-place after copying to `examples/` +- Original source files (in repos or `local_examples/`) remain unchanged +- Processed files have test markers removed, REMOVE blocks stripped, etc. + +--- + +## Key Components + +> **Note**: This section provides technical details about each component. For practical usage, see [Working with Examples](#working-with-examples). + +### 1. Build Scripts + +#### `build/make.py` + +**Purpose**: Main orchestrator for the build process + +**Responsibilities**: +- Parse command-line arguments (stack definition, skip-clone, log level, etc.) +- Initialize the build environment +- Invoke component processing for remote examples +- Invoke local example processing +- Coordinate the overall build workflow + +**Key Functions**: +- `parse_args()`: Parse command-line arguments +- Main execution: Calls `All.apply()` and `process_local_examples()` + +**Inputs**: +- `--stack`: Path to stack definition (default: `./data/components/index.json`) +- `--skip-clone`: Skip git clone operations +- `--loglevel`: Python logging level +- `--tempdir`: Temporary directory for cloning repositories + +**Outputs**: +- Processed examples in `examples/` directory +- Updated `data/examples.json` metadata file + +#### `build/local_examples.py` + +**Purpose**: Process local example files from the `local_examples/` directory + +**Responsibilities**: +- Walk the `local_examples/` directory tree +- Identify example files by extension (see [Appendix: Language Mappings](#language-mappings)) +- Extract example IDs from file headers +- Process examples using the `Example` class +- Generate metadata and update `examples.json` +- Handle language-specific client name mapping (e.g., Java-Sync vs Java-Async) + +**Key Functions**: +- `process_local_examples()`: Main processing function +- `get_language_from_extension()`: Map file extensions to languages +- `get_client_name_from_language_and_path()`: Determine client name with path-based overrides +- `get_example_id_from_file()`: Extract example ID from first line + +**Path-Based Client Name Overrides**: + +Some languages have multiple client implementations (sync/async, different libraries). The system uses directory path to determine which variant: + +- Java files in `lettuce-async/` → `Java-Async` (Lettuce async client) +- Java files in `lettuce-reactive/` → `Java-Reactive` (Lettuce reactive client) +- Java files elsewhere → `Java-Sync` (Jedis synchronous client) +- Rust files in `rust-async/` → `Rust-Async` +- Rust files in `rust-sync/` → `Rust-Sync` +- C# files in `async/` → `C#-Async` +- C# files in `sync/` → `C#-Sync` + +This allows the same language to appear multiple times in the tab interface with different implementations. + +**Outputs**: +- Copies files to `examples/{example_id}/local_{filename}` +- Updates `data/examples.json` with metadata + +### 2. Component Processing + +#### `build/components/component.py` + +**Purpose**: Handle remote example processing from GitHub repositories + +**Key Classes**: + +**`Component`**: Base class for all components +- Handles URI parsing +- Manages git operations +- Provides utility methods for repository access + +**`All`**: Main component orchestrator +- Loads component definitions from `data/components/index.json` +- Processes clients, core, docs, modules, and assets +- Persists examples metadata to `data/examples.json` + +**`Client`**: Client library component handler +- Clones client library repositories +- Extracts examples based on component configuration +- Processes examples using the `Example` class +- Generates source URLs for GitHub links +- Creates metadata for each example + +**Key Methods**: +- `_git_clone()`: Clone repositories from GitHub +- `_copy_examples()`: Extract and process examples from repositories +- `_get_example_id_from_file()`: Extract example ID from file header +- `_get_default_branch()`: Query GitHub API for default branch name + +**GitHub Integration**: +- Uses GitHub API to fetch latest release tags +- Clones repositories at specific tags or branches +- Generates source URLs pointing to GitHub + +#### `build/components/example.py` + +**Purpose**: Parse and process individual example files + +**Special Comment Markers**: +- `EXAMPLE: {id}`: Defines the example identifier (required, must be first line) +- `BINDER_ID {hash}`: Defines the BinderHub commit hash for interactive notebook link (optional) +- `HIDE_START` / `HIDE_END`: Code blocks hidden by default (revealed with eye button) +- `REMOVE_START` / `REMOVE_END`: Code blocks completely removed from display +- `STEP_START {name}` / `STEP_END`: Named code blocks for step-by-step examples + +**BINDER_ID Marker**: + +The `BINDER_ID` marker provides a commit hash for [BinderHub](https://binderhub.readthedocs.io/en/latest/) integration, allowing users to run examples in an interactive Jupyter notebook environment. + +**Syntax**: +```python +# EXAMPLE: example_id +# BINDER_ID 6bbed3da294e8de5a8c2ad99abf883731a50d4dd +``` + +**Requirements**: +- Must appear after the `EXAMPLE:` marker (typically on line 2) +- Must use the language's comment prefix (e.g., `#` for Python, `//` for JavaScript) +- The hash value is a Git commit SHA from the binder-launchers repository +- Only one `BINDER_ID` per example file +- Optional - not all examples need BinderHub integration + +**Usage**: +The hash is used to construct a BinderHub URL like: +``` +https://redis.io/binder/v2/gh/redis/binder-launchers/{hash}?urlpath=%2Fdoc%2Ftree%2Fdemo.ipynb +``` + +This allows documentation to include "Try this in Jupyter" links that launch interactive notebook environments with the example pre-loaded. + +**Processing Algorithm**: +1. Read file line by line +2. Detect special comment markers (using language-specific comment prefix) +3. Extract example ID from `EXAMPLE:` marker (line 1) +4. Extract BinderHub hash from `BINDER_ID` marker if present (typically line 2) +5. Track hidden/highlighted/removed ranges +6. Extract named steps with `STEP_START`/`STEP_END` +7. Filter out test markers and removed blocks +8. Generate metadata (highlight ranges, hidden ranges, named steps, binder ID) +9. Write processed content back to file (in-place modification) + +**BINDER_ID Extraction Details**: + +The `BINDER_ID` marker allows example authors to specify a Git reference (branch name or commit SHA) from the `redis/binder-launchers` repository. This enables the Hugo templates to generate "Run this example in the browser" links that open the example in an interactive Jupyter notebook environment via BinderHub. + +**Quick Implementation Checklist**: +- [ ] Add constant: `BINDER_ID = 'BINDER_ID'` (around line 11 in `example.py`) +- [ ] Add class attribute: `binder_id = None` (around line 49 in `Example` class) +- [ ] Add regex pattern: `binder = re.compile(...)` (around line 94 in `make_ranges()`) +- [ ] Add detection logic in `elif` chain (around line 157 in main loop) +- [ ] Add conditional metadata field in `build/local_examples.py` (around line 183) +- [ ] Add conditional metadata field in `build/components/component.py` (around line 278) +- [ ] Test with both branch name and commit SHA +- [ ] Verify `BINDER_ID` line removed from processed output +- [ ] Verify `binderId` appears in `data/examples.json` + +The parser should implement the following logic in `build/components/example.py`: + +**1. Add Constant and Class Attribute**: + +First, add the constant at the top of the file with other marker constants: +```python +BINDER_ID = 'BINDER_ID' +``` + +Add the attribute to the `Example` class: +```python +class Example(object): + language = None + path = None + content = None + hidden = None + highlight = None + named_steps = None + binder_id = None # Add this +``` + +Initialize in `__init__`: +```python +self.binder_id = None +``` + +**2. Compile Regex Pattern**: + +In the `make_ranges()` method (around line 94), add the regex pattern compilation alongside other patterns (after `exid` pattern): +```python +exid = re.compile(f'{PREFIXES[self.language]}\\s?{EXAMPLE}') +binder = re.compile(f'{PREFIXES[self.language]}\\s?{BINDER_ID}\\s+([a-zA-Z0-9_-]+)') +go_output = re.compile(f'{PREFIXES[self.language]}\\s?{GO_OUTPUT}') +``` + +**Exact location**: In `build/components/example.py`, class `Example`, method `make_ranges()`, in the section where regex patterns are compiled (after line 93). + +**Pattern explanation**: +- `{PREFIXES[self.language]}` - Language-specific comment prefix (e.g., `#` or `//`) +- `\\s?` - Optional whitespace after comment prefix +- `{BINDER_ID}` - The literal string "BINDER_ID" +- `\\s+` - Required whitespace before identifier +- `([a-zA-Z0-9_-]+)` - Capture group for Git reference (commit SHA or branch name) + - Matches: lowercase letters (a-z), uppercase letters (A-Z), digits (0-9), hyphens (-), underscores (_) + - Length: 1 or more characters (no maximum) + - Examples: `6bbed3da294e8de5a8c2ad99abf883731a50d4dd` (commit SHA), `python-landing` (branch name), `main`, `feature-123` + +**Why this pattern works**: +- **Backward compatible**: The old pattern `([a-f0-9]{40})` only matched commit SHAs. The new pattern `([a-zA-Z0-9_-]+)` matches commit SHAs (which are valid under the new pattern) AND branch names. +- **No breaking changes**: Existing examples with commit SHAs continue to work without modification. +- **Flexible**: Supports common Git branch naming conventions (kebab-case, snake_case, alphanumeric). + +**3. Detection and Extraction**: + +Add detection logic in the main processing loop (around line 157), **after** the `EXAMPLE:` check and **before** the `GO_OUTPUT` check: + +```python +elif re.search(exid, l): + output = False + pass +elif re.search(binder, l): + # Extract BINDER_ID value (commit SHA or branch name) + match = re.search(binder, l) + if match: + self.binder_id = match.group(1) + logging.debug(f'Found BINDER_ID: {self.binder_id} in {self.path}:L{curr+1}') + output = False # CRITICAL: Skip this line from output +elif self.language == "go" and re.search(go_output, l): + # ... rest of processing +``` + +**Exact location**: In `build/components/example.py`, class `Example`, method `make_ranges()`, in the main `while curr < len(self.content):` loop, in the `elif` chain that handles special markers. + +**Critical implementation details**: +- **Must set `output = False`**: This prevents the line from being added to the `content` array +- **Placement matters**: Must be in the `elif` chain, not a separate `if` statement +- **No `content.append(l)`**: The line is skipped entirely, just like `EXAMPLE:` lines +- **Extract before setting output**: Get the value before marking the line to skip +- **Order in elif chain**: Must come after `exid` (EXAMPLE:) but before `go_output` to maintain proper precedence + +**4. Storage in Metadata**: + +In `build/local_examples.py`, add the `binderId` field conditionally after creating the metadata dictionary: + +```python +example_metadata = { + 'source': source_file, + 'language': language, + 'target': target_file, + 'highlight': example.highlight, + 'hidden': example.hidden, + 'named_steps': example.named_steps, + 'sourceUrl': None +} + +# Add binderId only if it exists +if example.binder_id: + example_metadata['binderId'] = example.binder_id + +examples_data[example_id][client_name] = example_metadata +``` + +In `build/components/component.py`, add similarly after setting other metadata fields: + +```python +example_metadata['highlight'] = e.highlight +example_metadata['hidden'] = e.hidden +example_metadata['named_steps'] = e.named_steps +example_metadata['sourceUrl'] = ( + f'{ex["git_uri"]}/tree/{default_branch}/{ex["path"]}/{os.path.basename(f)}' +) + +# Add binderId only if it exists +if e.binder_id: + example_metadata['binderId'] = e.binder_id + +examples = self._root._examples +``` + +**Why conditional addition**: +- Only add the field if `binder_id` is not `None` +- This keeps the JSON clean - examples without BinderHub links don't have the field +- Avoids `null` or empty string values in the metadata + +**5. Line Processing Behavior**: + +The `BINDER_ID` line is removed from output through the same mechanism as other marker lines: + +- **How it works**: Setting `output = False` prevents the line from reaching the `else` block that calls `content.append(l)` +- **Line number impact**: Because the line is never added to `content`, it doesn't affect line number calculations for steps, highlights, or hidden ranges +- **Result**: The processed file is clean, containing only the actual code without any marker comments + +**Common Pitfalls**: +1. **Forgetting `output = False`**: The line will appear in processed output +2. **Wrong placement in elif chain**: May not be detected or may interfere with other markers +3. **Using `if` instead of `elif`**: Could cause multiple conditions to match +4. **Not checking `if match`**: Could cause AttributeError if regex doesn't match +5. **Adding field unconditionally**: Results in `"binderId": null` in JSON for examples without the marker +6. **Regex pattern too restrictive**: Using `[a-f0-9]{40}` only matches commit SHAs, not branch names +7. **Regex pattern too permissive**: Using `.*` or `.+` could match invalid characters or whitespace +8. **Wrong capture group**: Using `match.group(0)` returns the entire match including comment prefix, not just the value + +**6. Complete Example Flow**: + +Here's a complete example showing how a file is processed: + +**Input file** (`local_examples/client-specific/redis-py/landing.py`): +```python +# EXAMPLE: landing +# BINDER_ID python-landing +import redis + +# STEP_START connect +r = redis.Redis(host='localhost', port=6379, decode_responses=True) +# STEP_END +``` + +**Processing steps**: +1. Line 1: `EXAMPLE:` detected → `output = False` → line skipped +2. Line 2: `BINDER_ID` detected → extract value `python-landing` → `output = False` → line skipped +3. Line 3: `import redis` → no marker → added to `content` array at index 0 +4. Line 4: Empty line → added to `content` array at index 1 +5. Line 5: `STEP_START` detected → record step start at line 3 (len(content) + 1) → line skipped +6. Line 6: Code → added to `content` array at index 2 +7. Line 7: `STEP_END` detected → record step range "3-3" → line skipped + +**Output file** (`examples/landing/local_client-specific_redis-py_landing.py`): +```python +import redis + +r = redis.Redis(host='localhost', port=6379, decode_responses=True) +``` + +**Metadata** (`data/examples.json`): +```json +{ + "landing": { + "Python": { + "source": "local_examples/client-specific/redis-py/landing.py", + "language": "python", + "target": "examples/landing/local_client-specific_redis-py_landing.py", + "highlight": ["1-3"], + "hidden": [], + "named_steps": { + "connect": "3-3" + }, + "sourceUrl": null, + "binderId": "python-landing" + } + } +} +``` + +**Key observations**: +- Both `EXAMPLE:` and `BINDER_ID` lines are removed from output +- Line numbers in metadata refer to the processed file (after marker removal) +- `binderId` is stored at the language level, not the example set level +- The value is extracted cleanly without comment prefix or keyword +- Value can be either a Git commit SHA (40 hex chars) or a branch name (letters, numbers, hyphens, underscores) + +**Output Metadata** (stored in `examples.json`): +- `highlight`: Line ranges to highlight (e.g., `["1-10", "15-20"]`) +- `hidden`: Line ranges initially hidden (e.g., `["5-8"]`) +- `named_steps`: Map of step names to line ranges (e.g., `{"connect": "1-5"}`) +- `binderId`: BinderHub commit hash (optional, e.g., `"6bbed3da294e8de5a8c2ad99abf883731a50d4dd"`) + +> **Note**: For language-specific configuration (comment prefixes, test markers), see [Appendix: Language Mappings](#language-mappings). + +### 3. Hugo Templates + +#### `layouts/shortcodes/clients-example.html` + +**Purpose**: Hugo shortcode for embedding code examples in Markdown + +**Parameters** (Named): +- `set`: Example set name (required) - matches the `EXAMPLE:` ID +- `step`: Example step name (optional) - references a `STEP_START` block +- `lang_filter`: Language filter (optional) - show only specific languages +- `max_lines`: Maximum lines shown by default (optional, default: 100) +- `dft_tab_name`: Custom first tab name (optional, default: ">_ Redis CLI") +- `dft_tab_link_title`: Custom first tab footer link title (optional) +- `dft_tab_url`: Custom first tab footer link URL (optional) +- `show_footer`: Show footer (optional, default: true) + +**Parameters** (Positional - for backward compatibility): +- Position 0: example set name +- Position 1: step name +- Position 2: language filter +- Position 3: max lines +- Position 4: custom first tab name +- Position 5: custom first tab footer link title +- Position 6: custom first tab footer link URL + +**Functionality**: +- Detects named vs positional parameters +- Normalizes parameters into Hugo scratch variables +- Captures inner content (for redis-cli examples) +- Delegates rendering to `tabbed-clients-example.html` partial + +#### `layouts/partials/tabbed-clients-example.html` + +**Purpose**: Generate the tabbed interface HTML + +**Responsibilities**: +- Load example metadata from `data/examples.json` +- Iterate through configured languages (from `config.toml`) +- Generate tabs for each available language +- Apply syntax highlighting using Hugo's `highlight` function +- Handle step-specific highlighting +- Render redis-cli tab if inner content provided +- Generate footer with quickstart links and source URLs + +**Data Sources**: +- `$.Site.Data.examples`: Loaded from `data/examples.json` +- `$.Site.Params.clientsexamples`: Language order from `config.toml` +- `$.Site.Params.clientsconfig`: Client configuration from `config.toml` + +**Tab Generation Logic**: +1. Check if example exists in `examples.json` +2. For each configured language: + - Check if example exists for that language + - Apply language filter if specified + - Load example file from `target` path + - Apply syntax highlighting with line numbers + - Apply step-specific or default highlighting + - Generate tab metadata (title, language, quickstart slug, source URL) +3. Render tabs using `tabs/wrapper.html` partial + +#### `layouts/partials/tabs/wrapper.html` + +**Purpose**: Render the interactive tabbed interface HTML + +**Features**: +- Language selector dropdown +- Visibility toggle button (show/hide hidden code) +- Copy to clipboard button +- BinderHub "Run in browser" link (conditional) +- Tab panels with syntax-highlighted code +- Footer with quickstart links and GitHub source links +- Responsive design with Tailwind CSS + +**JavaScript Integration**: + +The interactive features are implemented in JavaScript (location varies by theme): +- `toggleVisibleLinesForCodetabs()`: Toggle hidden code visibility +- `copyCodeToClipboardForCodetabs()`: Copy code to clipboard +- Language selector change handler: Switch between tabs +- Language preference persistence (localStorage) + +> **Note**: JavaScript implementation details are theme-specific and not covered in this specification. + +#### BinderHub Integration ("Run in Browser" Link) + +**Purpose**: Provide interactive Jupyter notebook environment for running examples + +**Feature Description**: + +The code example boxes can display a "Run this example in the browser" link that launches the example in a BinderHub-powered Jupyter notebook environment. This link appears in the top bar of the example box, next to the three-dot menu icon. + +**Conditional Display**: +- Only shown if the example has a `binderId` value in its metadata +- If no `binderId` exists, the link is not rendered (no placeholder, no broken link) +- The `binderId` is language-specific, so different languages in the same example set may have different BinderHub links +- BinderHub uses Jupyter notebooks which can run code in multiple languages (Python, Node.js, Java, etc.) + +**Link URL Format**: +``` +https://redis.io/binder/v2/gh/redis/binder-launchers/?urlpath=%2Fdoc%2Ftree%2Fdemo.ipynb +``` + +**URL Components**: +- **Base URL**: `https://redis.io/binder/v2/gh/redis/binder-launchers/` +- **Binder ID**: The Git reference from `binderId` field (commit SHA or branch name) + - **Commit SHA**: 40 hexadecimal characters (e.g., `6bbed3da294e8de5a8c2ad99abf883731a50d4dd`) + - **Branch name**: Letters, numbers, hyphens, underscores (e.g., `python-landing`, `main`, `feature-123`) +- **URL Path**: `?urlpath=%2Fdoc%2Ftree%2Fdemo.ipynb` (constant, URL-encoded path to notebook) +- **Notebook filename**: Always `demo.ipynb` - do NOT change per example + +**Examples**: +``` +# Using branch name +https://redis.io/binder/v2/gh/redis/binder-launchers/python-landing?urlpath=%2Fdoc%2Ftree%2Fdemo.ipynb + +# Using commit SHA +https://redis.io/binder/v2/gh/redis/binder-launchers/6bbed3da294e8de5a8c2ad99abf883731a50d4dd?urlpath=%2Fdoc%2Ftree%2Fdemo.ipynb +``` + +**Implementation in Hugo Templates**: + +The implementation spans two template files: + +**1. Extract and pass binderId in `layouts/partials/tabbed-clients-example.html`**: + +In the loop that builds tabs for each language, extract the `binderId` and include it in the tab dictionary: + +```go-html-template +{{ $clientExamples := index $.Site.Data.examples $id }} +{{ range $client := $.Site.Params.clientsexamples }} + {{ $example := index $clientExamples $client }} + {{ $clientConfig := index $.Site.Params.clientsconfig $client }} + {{ $language := index $example "language" }} + {{ $quickstartSlug := index $clientConfig "quickstartSlug" }} + + {{ if and ($example) (or (eq $lang "") (strings.Contains $lang $client)) }} + {{ $examplePath := index $example "target" }} + {{ $options := printf "linenos=false" }} + + {{/* ... highlight options logic ... */}} + + {{ if hasPrefix $language "java" }}{{ $language = "java"}}{{ end }} + {{ $params := dict "language" $language "contentPath" $examplePath "options" $options }} + {{ $content := partial "tabs/source.html" $params }} + + {{/* Extract binderId if it exists */}} + {{ $binderId := index $example "binderId" }} + + {{ $tabs = $tabs | append (dict "title" $client "language" $client "quickstartSlug" $quickstartSlug "content" $content "sourceUrl" (index $example "sourceUrl") "binderId" $binderId) }} + {{ end }} +{{ end }} +``` + +**Key points**: +- Extract `binderId` using `index $example "binderId"` +- Add it to the tab dictionary alongside other tab data +- If `binderId` doesn't exist, it will be `nil` (which is fine - handled later) + +**2. Add link container in `layouts/partials/tabs/wrapper.html` top bar**: + +Insert the BinderHub link container between the language selector and the control buttons: + +```go-html-template + +
+
+ + +
+ + {{/* BinderHub "Run in browser" link - shown conditionally based on current tab's binderId */}} + + +
+ {{/* Visibility toggle button */}} + {{/* Copy to clipboard button */}} +
+
+``` + +**Placement notes**: +- Container is placed **after** the language selector (`flex-1` div) +- Container is placed **before** the control buttons (visibility/copy) +- `ml-4` adds left margin to separate from language selector +- `ml-2` on buttons div adds small gap between link and buttons +- Container starts empty - JavaScript will populate it + +**3. Add binderId data attribute to tab panels**: + +In the tab panels loop, add the `data-binder-id` attribute if `binderId` exists: + +```go-html-template + +{{ range $i, $tab := $tabs }} + {{ $tid := printf "%s_%s" (replace (replace (index $tab "title") "#" "sharp") "." "") $id }} + {{ $pid := printf "panel_%s" $tid }} + {{ $dataLang := replace (or (index $tab "language") "redis-cli") "C#" "dotnet" }} + {{ $dataLang := replace $dataLang "." "-" }} + {{ $binderId := index $tab "binderId" }} + +
+ {{/* ... panel content ... */}} +
+{{ end }} +``` + +**Key points**: +- Extract `binderId` from tab data +- Only add `data-binder-id` attribute if `binderId` exists (conditional) +- Add `data-codetabs-id` to match panels to their container +- Both attributes are used by JavaScript to find and update the link + +**4. Add JavaScript to handle link display and updates**: + +Add this script at the end of `layouts/partials/tabs/wrapper.html` (after the closing `` of the codetabs container): + +```html + +``` + +**JavaScript implementation details**: + +**Function: `updateBinderLink()`** +- **Purpose**: Show or hide the BinderHub link based on the currently selected language tab +- **Trigger**: Called on page load and when language selector changes + +**Step-by-step logic**: +1. **Get references**: Find the link container and language selector by ID +2. **Get selected tab index**: Read `data-index` attribute from selected option +3. **Find corresponding panel**: Query all panels with matching `data-codetabs-id` +4. **Read binderId**: Get `data-binder-id` attribute from current panel +5. **Clear container**: Remove any existing link (important for language switches) +6. **Conditional rendering**: + - If `binderId` exists: Create link element with proper URL and append to container + - If `binderId` is null/undefined: Container remains empty (no link shown) + +**URL construction**: +```javascript +const binderUrl = 'https://redis.io/binder/v2/gh/redis/binder-launchers/' + + binderId + + '?urlpath=%2Fdoc%2Ftree%2Fdemo.ipynb'; +``` +- Base URL + commit hash + URL-encoded path +- `%2F` is the URL-encoded form of `/` +- Path is constant: `/doc/tree/demo.ipynb` + +**Link element properties**: +- `target="_blank"`: Opens in new tab +- `rel="noopener noreferrer"`: Security best practice for external links +- `className`: Tailwind CSS classes for styling (small text, hover effects, flex layout) +- `title`: Tooltip text for accessibility +- `innerHTML`: SVG play icon + text label + +**Event handling**: +- **Page load**: IIFE executes immediately, calls `updateBinderLink()` +- **Language change**: Event listener on ` -
+ + {{/* BinderHub "Run in browser" link - shown conditionally based on current tab's binderId */}} + + +