figures: move to matplotlib, make output SVGs affected by CSS theme, …

…hook up new preprocessing step to mdbook (#530)

* Supersedes and closes #452

Co-authored-by: Antonio Vivace <avivace4@gmail.com>

* Loosen requirements.txt versions

* Hook up new graph generator to mdBook

And also use the newly-generated graphs in the MBC5 page

---------

Co-authored-by: Antonio Vivace <avivace4@gmail.com>
Co-authored-by: Eldred HABERT <me@eldred.fr>

.gitignore

@@ -1,3 +1,5 @@
 /docs/
 /target/
 /env/
+/generated/
+__pycache__/

book.toml

@@ -10,15 +10,19 @@ build-dir = "docs"
 create-missing = true # This is kept for convenience, but CI sets it to false
 use-default-preprocessors = false
 
-# Custom preprocessor for internal links processing
-[preprocessor.pandocs]
-command = "cargo run -p pandocs-preproc --locked --release --"
-after = [ "links" ]
+[preprocessor.graph_gen]
+command = "src/imgs/src/preproc.py"
+before = [ "links" ] # This generates some of the files that get `{{#include}}`d
 
 # `{{#include }}` etc. resolution
 [preprocessor.links]
 
-# Custom back-end for our custom markup
+# Custom preprocessor for internal links processing and other custom markup
+[preprocessor.pandocs]
+command = "cargo run -p pandocs-preproc --locked --release --"
+after = [ "links" ]
+
+# Custom back-end to generate the single-file version and scrub off some generated files
 [output.pandocs]
 command = "cargo run -p pandocs-renderer --locked --release --"
 

renderer/src/main.rs

@@ -67,38 +67,6 @@ impl Renderer for Pandocs {
         path.set_file_name("print.html");
         gen_single_page(&mut path, &base_url).context("Failed to render single-page version")?;
 
-        // Generate the graphs in `imgs/src/` by shelling out to Python
-        let working_dir = ctx.destination.join("imgs");
-        let src_dir = working_dir.join("src");
-        let python = if cfg!(windows) { "python" } else { "python3" };
-        let gen_graph = |file_name, title| {
-            let mut file_name = PathBuf::from_str(file_name).unwrap(); // Can't fail
-            let output = File::create(working_dir.join(&file_name))?;
-
-            file_name.set_extension("csv");
-            let status = Command::new(python)
-                .current_dir(&src_dir)
-                .arg("graph_render.py")
-                .arg(&file_name)
-                .arg(title)
-                .stdout(output)
-                .status()
-                .with_context(|| format!("Failed to generate \"{}\"", file_name.display()))?;
-
-            if status.success() {
-                Ok(())
-            } else {
-                Err(Error::msg(format!(
-                    "Generating \"{}\" failed with {}",
-                    file_name.display(),
-                    status,
-                )))
-            }
-        };
-        gen_graph("MBC5_Rumble_Mild.svg", "Mild Rumble")?;
-        gen_graph("MBC5_Rumble_Strong.svg", "Strong Rumble")?;
-        fs::remove_dir_all(&src_dir).context(format!("Failed to remove {}", src_dir.display()))?;
-
         // Scrub off files that need not be published
         for path in GlobWalkerBuilder::from_patterns(&ctx.destination, &[".gitignore", "*.graphml"])
             .file_type(FileType::FILE)

requirements.txt

@@ -1 +1,4 @@
-pygal==3.0.0
+beautifulsoup4>=4.12.2,<=5.0
+lxml>=5.0.0,<=6.0
+matplotlib>=3.8.2,<=4.0
+pandas>=2.1.4,<=3.0

src/MBC5.md

@@ -58,6 +58,6 @@ bit to 1 enables the rumble motor and keeps it enabled until the bit is reset ag
 To control the rumble's intensity, it should be turned on and off repeatedly,
 as seen with these two examples from Pokémon Pinball:
 
-<img src="imgs/MBC5_Rumble_Mild.svg" width="950px">
+{{#include ../generated/MBC5_Rumble_Mild.svg}}
 
-<img src="imgs/MBC5_Rumble_Strong.svg" width="950px">
+{{#include ../generated/MBC5_Rumble_Strong.svg}}

src/imgs/src/graph_render.py

@@ -1,82 +1,103 @@
-import pygal
-from pygal.style import Style
-import math
+#!/usr/bin/env python3
 from sys import argv, stderr
+import io
 
-# ------------------------------------------------------------------------------------------------
-# Configuration Constants
-# ------------------------------------------------------------------------------------------------
+import lxml
+import matplotlib.pyplot as plt
+import pandas as pd
+from bs4 import BeautifulSoup
 
-# Rotation of X-Axis labels in degrees
-x_label_rotation = 0.01
 
-# Number of labels to be displayed on the X-Axis
-x_label_count = 10
+def gen_graph(in_path, title, out_path):
 
-# ------------------------------------------------------------------------------------------------
-# The first line of the file must contain the X- and Y-Axis labels seperated by commas.
-# The following lines are expected to contain the graph data in a comma-separated format
-#  and in the same order as the Axis labels.
-# ------------------------------------------------------------------------------------------------
+    ## Let's draw the plot
 
-def gen_graph(in_path, title):
-    custom_style = Style(
-        font_family="Inter",
-        label_font_size=12,
-        major_label_font_size=12,
-        title_font_size=16
+    plt.rcParams["figure.figsize"] = [7.50, 3.50]
+    plt.rcParams["figure.autolayout"] = True
+    plt.rcParams["font.family"] = "Inter"
+    # Assume fonts are installed on the machine where the SVG will be viewed
+    #  (we load Inter with the webpage so it should be there)
+    plt.rcParams["svg.fonttype"] = "none"
+
+    # Those are just used to "fingerprint" the resulting elements in the SVG export,
+    #  they will be replaced by CSS variables
+    COLOR_BASE = "#FFCD01".lower()
+    COLOR_LINE = "#FFCD02".lower()
+
+    # Set everything to the base color
+    plt.rcParams["text.color"] = COLOR_BASE
+    plt.rcParams["axes.labelcolor"] = COLOR_BASE
+    plt.rcParams["xtick.color"] = COLOR_BASE
+    plt.rcParams["ytick.color"] = COLOR_BASE
+
+    # Read the values to plot from the input CSV
+    df = pd.read_csv(in_path)
+
+    # Set the color of the actual plot line to the secondary color
+    plot = df.set_index("Time (ms)").plot(
+        legend=None, gid="fitted_curve", color=COLOR_LINE
     )
 
-    # Create Line Chart Object and Open File
-    chart = pygal.Line(
-        height=450,
-        show_dots=False,
-        show_legend=False,
-        show_minor_x_labels=False,
-        x_label_rotation=x_label_rotation,
-        style=custom_style
+    # Add grid lines on the y values
+    plot.yaxis.grid(True)
+
+    # Set the color of the plot box to the base color too
+    plt.setp(plot.spines.values(), color=COLOR_BASE)
+
+    # Add title at the top
+    plt.title(title)
+    plt.ylabel(df.columns[1])
+
+    ## Manipulate the SVG render of the plot to replace colors with CSS variables
+    with io.StringIO() as f:
+        plt.savefig(f, format="svg", transparent=True)
+
+        # It's an SVG, so let's use the XML parser
+        soup = BeautifulSoup(f.getvalue(), "xml")
+
+    replace_style_property(soup, "path", "stroke", COLOR_BASE, "var(--fg, #000)")
+    replace_style_property(soup, "path", "stroke", COLOR_LINE, "var(--inline-code-color, #320)")
+    replace_style_property(soup, "text", "fill", COLOR_BASE, "var(--fg, #000)")
+    replace_style_property(soup, "use", "stroke", COLOR_BASE, "var(--fg, #000)")
+    replace_style_property(soup, "use", "fill", COLOR_BASE, "var(--fg, #000)")
+
+    # Write the altered SVG file
+    with open(out_path, "wt") as f:
+        print(soup, file=f)
+
+
+def replace_style_property(
+    soup, element_name, css_property, value_to_replace, new_value
+):
+    """
+    Given a `Soup`, a CSS `property` applied inline, replace the a `specific value`
+    this property can assume with `another` one in all the elements with the specified `name`
+
+    E.g. the style of all the "path" elements whith a CSS property "fill" of
+    "#ffcd01" will change to "var(--fg):
+
+    `fill: #ffcd01; font: 12px 'Inter'; text-anchor: middle`
+    to
+    `fill: var(--fg); font: 12px 'Inter'; text-anchor: middle`
+
+    Soup and soup ResultSet are modified in-place
+    """
+    found_elements = soup.find_all(
+        element_name,
+        style=lambda value: value and f"{css_property}: {value_to_replace}" in value,
     )
-    csv = open(in_path, "r").readlines()
-
-    # Set Chart and Axis Titles
-    chart.title = title
-    headers = csv.pop(0).split(",")
-    chart.x_title = headers[0]
-    chart.y_title = headers[1]
-
-    # Generate label spacing variables
-    min_x_val = float(csv[0].split(",")[0])
-    max_x_val = float(csv[len(csv) - 1].split(",")[0])
-    x_mod_val = (max_x_val - min_x_val) / x_label_count
-
-    # Generate graph data arrays
-    x_labels = []
-    x_labels_major = []
-    y_data = []
-    last_x = None
-    for line in csv:
-        # Add data to label arrays
-        data = line.split(",")
-        x_labels.append(data[0])
-        y_data.append(float(data[1]))
-
-        # Check if current X-Label should be Major Label
-        xval_float = float(data[0])
-        if last_x is not None and ((last_x % x_mod_val) > (xval_float % x_mod_val)):
-            x_labels_major.append(math.floor(xval_float))
-            x_labels.append(math.floor(xval_float))
-        last_x = xval_float
-
-    # Load graph data into chart object and save to file
-    chart.x_labels = x_labels
-    chart.x_labels_major = x_labels_major
-
-    chart.add("", y_data)
-    print(chart.render(is_unicode=True))
-
-
-if len(argv) != 3:
-    print("Usage: python3 graph_render.py <path/to.csv> <graph title>", file=stderr)
-    exit(1)
-
-gen_graph(argv[1], argv[2])
+    # Replace the color magic value with the CSS variable
+    for element in found_elements:
+        element["style"] = element["style"].replace(
+            f"{css_property}: {value_to_replace}", f"{css_property}: {new_value}"
+        )
+
+    return
+
+# CLI interface.
+if __name__ == "__main__":
+    if len(argv) != 3:
+        print("Usage: python3 graph_render.py <path/to.csv> <graph title>", file=stderr)
+        exit(1)
+
+    gen_graph(argv[1], argv[2])

src/imgs/src/preproc.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+import json
+import pathlib
+import sys
+
+from graph_render import gen_graph
+
+if len(sys.argv) == 3 and sys.argv[1] == "supports":
+	sys.exit(sys.argv[2] == "not-supported")
+
+# Copy the book object from standard input to standard output.
+context,book = json.JSONDecoder().decode(sys.stdin.read())
+sys.stdout.write(json.JSONEncoder().encode(book))
+sys.stdout.close() # Ensure that nothing else gets written.
+
+pathlib.Path("./generated").mkdir(exist_ok=True)
+gen_graph("src/imgs/src/MBC5_Rumble_Mild.csv", "Mild Rumble", "./generated/MBC5_Rumble_Mild.svg")
+gen_graph("src/imgs/src/MBC5_Rumble_Strong.csv", "Strong Rumble", "./generated/MBC5_Rumble_Strong.svg")