diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 85b7d307..25218f64 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -147,6 +147,8 @@ jobs: uv run python -m zipfile --list "$file" | grep \\.so echo "Checking if the correct python version is contained in the wheel" uv run python -m zipfile --list "$file" | grep ${PY_CACHE_TAG} + echo "Checking that Blend2D debug assertions are not present" + uv run python scripts/check_no_blend2d_assertions.py "$file" done mkdir -p ./dist cp wheelhouse/*.whl ./dist/ @@ -200,6 +202,8 @@ jobs: uv run python -m zipfile --list "$file" | grep \\.so echo "Checking if the correct python version is contained in the wheel" uv run python -m zipfile --list "$file" | grep ${PY_CACHE_TAG} + echo "Checking that Blend2D debug assertions are not present" + uv run python scripts/check_no_blend2d_assertions.py "$file" done mkdir -p ./dist cp wheelhouse/*.whl ./dist/ @@ -243,6 +247,8 @@ jobs: uv run python -m zipfile --list "$file" | grep \\.so echo "Checking if the correct python version is contained in the wheel" uv run python -m zipfile --list "$file" | grep ${PY_CACHE_TAG} + echo "Checking that Blend2D debug assertions are not present" + uv run python scripts/check_no_blend2d_assertions.py "$file" done mkdir -p ./dist cp wheelhouse/*.whl ./dist/ @@ -348,6 +354,8 @@ jobs: $file = $_.FullName Write-Output "Inspecting $file" uv run python -m zipfile --list "$file" + Write-Output "Checking that Blend2D debug assertions are not present" + uv run python scripts/check_no_blend2d_assertions.py "$file" } if (-not (Test-Path -Path .\dist)) { New-Item -Path .\dist -ItemType Directory diff --git a/cmake/extlib_blend2d.cmake b/cmake/extlib_blend2d.cmake index e337d3bb..42aba91f 100644 --- a/cmake/extlib_blend2d.cmake +++ b/cmake/extlib_blend2d.cmake @@ -54,5 +54,38 @@ else() GIT_TAG 6dbc2cefbc996379e07104e34519a440b49b15d7 ) FetchContent_MakeAvailable(blend2d) + # Release wheels must not contain Blend2D's debug assertion path. Blend2D + # normally infers BL_BUILD_RELEASE from NDEBUG, but make that mode explicit + # for non-Debug builds so embedded builds do not depend on generator-specific + # CMAKE_BUILD_TYPE propagation. + target_compile_definitions( + blend2d + PUBLIC $<$>:BL_BUILD_RELEASE> + ) + + # Blend2D defines bl_runtime_assertion_failure() unconditionally in + # runtime.cpp. BL_BUILD_RELEASE removes assertion call sites, but the helper + # can still be retained from the static archive unless the final extension + # link can discard unused sections. + set(BLEND2D_GNU_GC_SECTIONS "$>,$,$,$>>>") + target_compile_options( + blend2d + PRIVATE "$<${BLEND2D_GNU_GC_SECTIONS}:-ffunction-sections;-fdata-sections>" + ) + target_link_options( + blend2d + INTERFACE "$<${BLEND2D_GNU_GC_SECTIONS}:-Wl,--gc-sections>" + ) + + # MinGW auto-exports global symbols from DLL/PYD links. If symbols from the + # static Blend2D archive are exported, they become linker roots and section + # GC cannot remove bl_runtime_assertion_failure(). Hide static-archive + # symbols from the final Windows GNU extension export table while preserving + # explicitly exported symbols such as the pybind module initializer. + set(BLEND2D_MINGW_EXCLUDE_STATIC_EXPORTS "$>,$,$>") + target_link_options( + blend2d + INTERFACE "$<${BLEND2D_MINGW_EXCLUDE_STATIC_EXPORTS}:-Wl,--exclude-libs,ALL>" + ) # FetchContent creates the target "blend2d" (and alias blend2d::blend2d). endif() diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse.md b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse.md new file mode 100644 index 00000000..085ba13a --- /dev/null +++ b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse.md @@ -0,0 +1,114 @@ + + +# Benchmark — docling-project/performance-dataset-bo767 on Apple M3 Max + +Generated: 2026-08-10T14:11:21 + +## Command + +```sh +python ./perf/run_scaling.py --threads 1,2,4,8,12 --compare all --mode parse --output-dir ./scratch8 --materialize-bitmaps=false +``` + +## Benchmark + +| parameter | value | +|------------------------|----------------------------------------------------------------------------------| +| dataset | docling-project/performance-dataset-bo767 | +| dataset source | huggingface | +| dataset revision | c684df54de2081a6de1192cee12713b54759c6bb | +| documents | 753 | +| pages | 54584 | +| mode | parse | +| thread counts | [1, 2, 4, 8, 12] | +| max concurrent results | 64 | +| other backends | ['pypdfium2'] | +| comparison suite | ['docling-parse', 'pymupdf', 'pypdfium2', 'pdfplumber', 'pdfminer.six', 'pypdf'] | +| max pages | (all) | +| pdf selection | (all) | +| bytesio | False | + +## System + +| parameter | value | +|----------------|---------------| +| cpu | Apple M3 Max | +| physical cores | 16 | +| logical cores | 16 | +| memory (GB) | 64.0 | +| platform | Darwin 25.2.0 | +| arch | arm64 | +| python | 3.12.11 | +| docling-parse | 7.11.0 | +| pymupdf | 1.27.1 | +| pypdfium2 | 5.4.0 | +| pdfplumber | 0.11.9 | +| pdfminer.six | 20251230 | +| pypdf | 6.14.2 | +| pillow | 12.2.0 | + +## Decode config + +| parameter | value | +|----------------------------------------------|---------| +| do_sanitization | 1 | +| enforce_same_font | 1 | +| horizontal_cell_tolerance | 1 | +| word_space_width_factor_for_merge | 0.33 | +| line_space_width_factor_for_merge | 1 | +| line_space_width_factor_for_merge_with_space | 0.33 | +| max_num_lines | -1 | +| max_num_bitmaps | -1 | +| do_thread_safe | 1 | +| release_native_memory_every_n_pages | 0 | +| keep_glyphs | 0 | +| keep_qpdf_warnings | 0 | + +## Content config + +| parameter | value | +|--------------------------|---------| +| char\_cells\_content\_level | 1 | +| word_cells_content_level | 0 | +| line_cells_content_level | 2 | +| shapes_content_level | 0 | +| bitmaps_content_level | 0 | +| include_bitmap_bytes | 0 | + +## Render config + +| parameter | value | +|-------------|---------| +| enabled | False | + +## Parsing and rendering performance + +| System hardware | dataset | Python package | Task | threads | total time (s) | average time/page | median time/page | 95 quantile time/page | 99 quantile time/page | +|--------------------------------------------------|---------------------------------------------------------------------------|------------------|---------|-----------|------------------|---------------------|--------------------|-------------------------|-------------------------| +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse` | 1 | 601.31 | 11.0 ms | 8.0 ms | 29.8 ms | 52.8 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse` | 2 | 314.2 | 11.4 ms | 8.2 ms | 31.1 ms | 54.9 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse` | 4 | 167 | 12.0 ms | 8.6 ms | 32.6 ms | 58.8 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse` | 8 | 85.42 | 12.4 ms | 8.9 ms | 33.3 ms | 60.2 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse` | 12 | 71.61 | 14.2 ms | 10.3 ms | 38.7 ms | 70.3 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pymupdf | `parse` | 1 | 1370.85 | 25.1 ms | 6.0 ms | 109.8 ms | 212.8 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pypdfium2 | `parse` | 1 | 255.23 | 4.7 ms | 1.6 ms | 17.5 ms | 42.9 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pdfplumber | `parse` | 1 | 2028.03 | 37.2 ms | 20.1 ms | 102.9 ms | 220.4 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pdfminer.six | `parse` | 1 | 2230.88 | 40.9 ms | 13.6 ms | 79.4 ms | 315.9 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pypdf | `parse` | 1 | 747.7 | 13.7 ms | 4.1 ms | 37.6 ms | 181.0 ms | + +## Thread scaling and speedup + +Task: `parse` + +| python package | threads | total time (s) | pages/sec | vs docling-parse (1t) | vs pymupdf (1t) | vs pypdfium2 (1t) | vs pdfplumber (1t) | vs pdfminer.six (1t) | vs pypdf (1t) | +|------------------|-----------|------------------|-------------|-------------------------|-------------------|---------------------|----------------------|------------------------|-----------------| +| docling-parse | 1 | 601.31 | 90.8 | 1.00x | 2.28x | 0.42x | 3.37x | 3.71x | 1.24x | +| docling-parse | 2 | 314.2 | 173.7 | 1.91x | 4.36x | 0.81x | 6.45x | 7.10x | 2.38x | +| docling-parse | 4 | 167 | 326.9 | 3.60x | 8.21x | 1.53x | 12.14x | 13.36x | 4.48x | +| docling-parse | 8 | 85.42 | 639 | 7.04x | 16.05x | 2.99x | 23.74x | 26.12x | 8.75x | +| docling-parse | 12 | 71.61 | 762.2 | 8.40x | 19.14x | 3.56x | 28.32x | 31.15x | 10.44x | +| pymupdf | 1 | 1370.85 | 39.8 | 0.44x | 1.00x | 0.19x | 1.48x | 1.63x | 0.55x | +| pypdfium2 | 1 | 255.23 | 213.9 | 2.36x | 5.37x | 1.00x | 7.95x | 8.74x | 2.93x | +| pdfplumber | 1 | 2028.03 | 26.9 | 0.30x | 0.68x | 0.13x | 1.00x | 1.10x | 0.37x | +| pdfminer.six | 1 | 2230.88 | 24.5 | 0.27x | 0.61x | 0.11x | 0.91x | 1.00x | 0.34x | +| pypdf | 1 | 747.7 | 73 | 0.80x | 1.83x | 0.34x | 2.71x | 2.98x | 1.00x | diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pdfminer.six-parse-.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pdfminer.six-parse-.png new file mode 100644 index 00000000..7a98f3db Binary files /dev/null and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pdfminer.six-parse-.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pdfplumber-parse-.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pdfplumber-parse-.png new file mode 100644 index 00000000..b9aba521 Binary files /dev/null and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pdfplumber-parse-.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pymupdf-parse-.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pymupdf-parse-.png new file mode 100644 index 00000000..fc29f6e6 Binary files /dev/null and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pymupdf-parse-.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pypdf-parse-.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pypdf-parse-.png new file mode 100644 index 00000000..9ff9a932 Binary files /dev/null and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pypdf-parse-.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pypdfium2-parse-.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pypdfium2-parse-.png new file mode 100644 index 00000000..d1b1b070 Binary files /dev/null and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hex_loglog_docling-parse-parse-_vs_pypdfium2-parse-.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hist_pages_per_document.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hist_pages_per_document.png new file mode 100644 index 00000000..3f454095 Binary files /dev/null and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hist_pages_per_document.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hist_stacked.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hist_stacked.png new file mode 100644 index 00000000..204575ab Binary files /dev/null and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/hist_stacked.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/scaling_parse.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/scaling_parse.png new file mode 100644 index 00000000..7efe6cee Binary files /dev/null and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_parse/scaling_parse.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render.md b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render.md index 4df524f7..84ee6a0a 100644 --- a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render.md +++ b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render.md @@ -2,12 +2,12 @@ # Benchmark — docling-project/performance-dataset-bo767 on Apple M3 Max -Generated: 2026-08-04T08:04:42 +Generated: 2026-08-10T17:06:40 ## Command ```sh -python ./perf/run_scaling.py --threads 1,4,8,12 --compare all --mode render --output-dir ./docs/performance_benchmarks/ +python ./perf/run_scaling.py --threads 1,2,4,8,12 --compare all --mode render --output-dir ./scratch9 ``` ## Benchmark @@ -20,11 +20,12 @@ python ./perf/run_scaling.py --threads 1,4,8,12 --compare all --mode render --ou | documents | 753 | | pages | 54584 | | mode | render | -| thread counts | [1, 4, 8, 12] | +| thread counts | [1, 2, 4, 8, 12] | | max concurrent results | 64 | | other backends | ['pypdfium2'] | | comparison suite | ['docling-parse', 'pymupdf', 'pypdfium2', 'pdfplumber', 'pdfminer.six', 'pypdf'] | | max pages | (all) | +| pdf selection | (all) | | bytesio | False | | render scale | 1.0 | @@ -39,7 +40,7 @@ python ./perf/run_scaling.py --threads 1,4,8,12 --compare all --mode render --ou | platform | Darwin 25.2.0 | | arch | arm64 | | python | 3.12.11 | -| docling-parse | 7.8.1 | +| docling-parse | 7.11.0 | | pymupdf | 1.27.1 | | pypdfium2 | 5.4.0 | | pdfplumber | 0.11.9 | @@ -80,8 +81,10 @@ python ./perf/run_scaling.py --threads 1,4,8,12 --compare all --mode render --ou | parameter | value | |--------------------------|---------| | render_text | 1 | +| min_stroke_width | 1 | | draw_text_bbox | 0 | | draw_text_basepoint | 0 | +| display_widgets | 0 | | fit_glyph_bbox_to_target | 0 | | resolve_fonts | 1 | | font_similarity_cutoff | 0.75 | @@ -93,13 +96,14 @@ python ./perf/run_scaling.py --threads 1,4,8,12 --compare all --mode render --ou | System hardware | dataset | Python package | Task | threads | total time (s) | average time/page | median time/page | 95 quantile time/page | 99 quantile time/page | |--------------------------------------------------|---------------------------------------------------------------------------|------------------|----------------|-----------|------------------|---------------------|--------------------|-------------------------|-------------------------| -| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse+render` | 1 | 1566.96 | 28.6 ms | 21.5 ms | 70.6 ms | 143.5 ms | -| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse+render` | 4 | 428.41 | 31.0 ms | 23.6 ms | 75.7 ms | 151.9 ms | -| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse+render` | 8 | 239.03 | 34.3 ms | 27.1 ms | 80.8 ms | 157.2 ms | -| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse+render` | 12 | 187.54 | 40.1 ms | 31.0 ms | 94.4 ms | 168.3 ms | -| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pymupdf | `parse+render` | 1 | 1786.95 | 32.7 ms | 11.0 ms | 125.3 ms | 235.8 ms | -| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pypdfium2 | `parse+render` | 1 | 761.86 | 13.9 ms | 7.6 ms | 41.8 ms | 106.2 ms | -| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pdfplumber | `parse+render` | 1 | 2934.31 | 47.8 ms | 29.2 ms | 118.5 ms | 242.9 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse+render` | 1 | 1673.55 | 30.5 ms | 19.5 ms | 82.6 ms | 180.6 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse+render` | 2 | 868.83 | 55.7 ms | 20.3 ms | 85.1 ms | 186.7 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse+render` | 4 | 466.4 | 33.7 ms | 22.5 ms | 89.0 ms | 193.1 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse+render` | 8 | 267 | 38.3 ms | 27.4 ms | 92.7 ms | 200.2 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | docling-parse | `parse+render` | 12 | 215.91 | 46.4 ms | 34.0 ms | 110.7 ms | 215.2 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pymupdf | `parse+render` | 1 | 1838.91 | 33.6 ms | 11.5 ms | 128.5 ms | 241.1 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pypdfium2 | `parse+render` | 1 | 763.8 | 13.9 ms | 7.7 ms | 42.0 ms | 106.9 ms | +| Apple M3 Max, 16 cores, 64 GB RAM, Darwin 25.2.0 | docling-project/performance-dataset-bo767@c684df5 (753 docs, 54584 pages) | pdfplumber | `parse+render` | 1 | 3027.25 | 49.3 ms | 30.2 ms | 119.7 ms | 253.2 ms | ## Thread scaling and speedup @@ -107,20 +111,21 @@ Task: `parse+render` | python package | threads | total time (s) | pages/sec | vs docling-parse (1t) | vs pymupdf (1t) | vs pypdfium2 (1t) | vs pdfplumber (1t) | |------------------|-----------|------------------|-------------|-------------------------|-------------------|---------------------|----------------------| -| docling-parse | 1 | 1566.96 | 34.8 | 1.00x | 1.14x | 0.49x | 1.87x | -| docling-parse | 4 | 428.41 | 127.4 | 3.66x | 4.17x | 1.78x | 6.85x | -| docling-parse | 8 | 239.03 | 228.4 | 6.56x | 7.48x | 3.19x | 12.28x | -| docling-parse | 12 | 187.54 | 291 | 8.36x | 9.53x | 4.06x | 15.65x | -| pymupdf | 1 | 1786.95 | 30.5 | 0.88x | 1.00x | 0.43x | 1.64x | -| pypdfium2 | 1 | 761.86 | 71.6 | 2.06x | 2.35x | 1.00x | 3.85x | -| pdfplumber | 1 | 2934.31 | 18.6 | 0.53x | 0.61x | 0.26x | 1.00x | +| docling-parse | 1 | 1673.55 | 32.6 | 1.00x | 1.10x | 0.46x | 1.81x | +| docling-parse | 2 | 868.83 | 62.8 | 1.93x | 2.12x | 0.88x | 3.48x | +| docling-parse | 4 | 466.4 | 117 | 3.59x | 3.94x | 1.64x | 6.49x | +| docling-parse | 8 | 267 | 204.4 | 6.27x | 6.89x | 2.86x | 11.34x | +| docling-parse | 12 | 215.91 | 252.8 | 7.75x | 8.52x | 3.54x | 14.02x | +| pymupdf | 1 | 1838.91 | 29.7 | 0.91x | 1.00x | 0.42x | 1.65x | +| pypdfium2 | 1 | 763.8 | 71.5 | 2.19x | 2.41x | 1.00x | 3.96x | +| pdfplumber | 1 | 3027.25 | 18 | 0.55x | 0.61x | 0.25x | 1.00x | ## Render size check Rasterised page size versus `pypdfium2`, tolerance 2 px. -| python package | pages compared | within tolerance | max delta (px) | worst page | pages w/o reference | -|------------------|------------------|--------------------|------------------|------------------------------------------|-----------------------| -| docling-parse | 54584 | 95.6% | 864 | 2216123.pdf p56 (2160x3024 vs 3024x2160) | 0 | -| pymupdf | 54584 | 100.0% | 1 | 1048308.pdf p2 (612x792 vs 613x793) | 0 | -| pdfplumber | 54584 | 100.0% | 1 | 1057315.pdf p80 (612x791 vs 612x792) | 0 | +| python package | pages compared | within tolerance | max delta (px) | worst page | pages w/o reference | +|------------------|------------------|--------------------|------------------|--------------------------------------|-----------------------| +| docling-parse | 54584 | 100.0% | 1 | 2407280.pdf p2 (612x793 vs 612x792) | 0 | +| pymupdf | 54584 | 100.0% | 1 | 1048308.pdf p2 (612x792 vs 613x793) | 0 | +| pdfplumber | 54584 | 100.0% | 1 | 1057315.pdf p80 (612x791 vs 612x792) | 0 | diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pdfplumber-parse-render-.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pdfplumber-parse-render-.png index 5a01f6c8..6db63f52 100644 Binary files a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pdfplumber-parse-render-.png and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pdfplumber-parse-render-.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pymupdf-parse-render-.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pymupdf-parse-render-.png index a9a3bd36..c02ab32f 100644 Binary files a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pymupdf-parse-render-.png and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pymupdf-parse-render-.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pypdfium2-parse-render-.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pypdfium2-parse-render-.png index 126aab8b..647363ac 100644 Binary files a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pypdfium2-parse-render-.png and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hex_loglog_docling-parse-parse-render-_vs_pypdfium2-parse-render-.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hist_stacked.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hist_stacked.png index 78151ede..31351176 100644 Binary files a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hist_stacked.png and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/hist_stacked.png differ diff --git a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/scaling_parse-render.png b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/scaling_parse-render.png index 6b39b786..96dea87e 100644 Binary files a/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/scaling_parse-render.png and b/docs/performance_benchmarks/apple_m3_max_performance-dataset-bo767_render/scaling_parse-render.png differ diff --git a/local_build.py b/local_build.py index e8001d32..1517fcfc 100644 --- a/local_build.py +++ b/local_build.py @@ -92,12 +92,14 @@ def run(cmd: List[str], cwd: str="./"): def build_local(num_threads: int): USE_SYSTEM_DEPS = os.getenv("USE_SYSTEM_DEPS", "OFF") + CMAKE_BUILD_TYPE = os.getenv("CMAKE_BUILD_TYPE", "Release") print("python prefix: ", sys.exec_prefix) print("python executable: ", sys.executable) config_cmd = [ "cmake", "-B", f"{BUILD_DIR}", + f"-DCMAKE_BUILD_TYPE={CMAKE_BUILD_TYPE}", f"-DUSE_SYSTEM_DEPS={USE_SYSTEM_DEPS}", f"-DPYTHON_EXECUTABLE={sys.executable}", "-DCMAKE_BUILD_TYPE=Release", diff --git a/perf/run_analysis.py b/perf/run_analysis.py index f351dd41..fc5582e8 100644 --- a/perf/run_analysis.py +++ b/perf/run_analysis.py @@ -38,6 +38,9 @@ from tabulate import tabulate from docling_parse.pdf_parser import ( + ContentConfig, + ContentLevel, + DecodeConfig, DoclingPdfParser, Timings, get_decode_page_timing_keys, @@ -56,6 +59,74 @@ class PageTimings: timings: Timings = field(default_factory=lambda: Timings()) +DECODE_PAGE_CHILDREN = [ + "to_json_page", + "extract_annots_json", + "decode_dimensions", + "decode_resources", + "decode_contents", + "decode_annots", + "rotate_contents", + "sanitize_orientation", + "sanitize_cells", + "sanitise_contents", +] + +TIMING_CHILDREN = { + "pipeline": [ + "decode_page", + "create_word_cells", + "create_line_cells", + ], + "decode_page": DECODE_PAGE_CHILDREN, + "sanitize_cells": [ + "sanitize_cells.remove_duplicate_cells", + "sanitize_cells.sanitize_text", + ], + "sanitise_contents": [ + "sanitise_contents.copy_cells", + "sanitise_contents.sanitize_bbox", + ], + "create_line_cells": [ + "create_line_cells.copy_cells", + "create_line_cells.sanitize_bbox", + "create_line_cells.remove_duplicate_cells", + ], + "decode_contents": [ + "content_decode_total", + "interprete_ops_total", + "decode_xobjects_total", + "decode_grphs_total", + "decode_fonts_total", + "parse_stream_total", + "do_form_machinery_total", + "do_image_total", + ], + "decode_fonts_total": [ + "font: init-copy", + "font: init-metrics", + "font: font-cmap", + "font: font-cmap-stream-decode", + "font: font-cmap-resources", + "font: font-chars", + ], + "font: font-cmap": ["cmap-parse-total"], + "cmap-parse-total": [ + "cmap-parse-endbfchar", + "cmap-parse-endbfrange", + "cmap-parse-endcodespacerange", + ], +} + +TOP_LEVEL_TIMING_KEYS = set(TIMING_CHILDREN["pipeline"]) +NESTED_TIMING_KEYS = { + child + for parent, children in TIMING_CHILDREN.items() + if parent != "pipeline" + for child in children +} + + # -------------- IO helpers -------------- @@ -90,6 +161,115 @@ def timestamped_out_path(prefix: str = "analysis") -> Path: return Path("perf") / "results" / f"{prefix}_{ts}.csv" +# -------------- Config helpers -------------- + + +def _add_bool_value_arg( + parser: argparse.ArgumentParser, + name: str, + *, + default: bool, + help: str, +) -> None: + parser.add_argument( + f"--{name}", + choices=["true", "false"], + default="true" if default else "false", + help=f"{help} (default: {str(default).lower()})", + ) + + +def _arg_was_passed(argv: List[str], name: str) -> bool: + option = f"--{name}" + return any(arg == option or arg.startswith(f"{option}=") for arg in argv) + + +def _parse_bool_arg(value: str) -> bool: + return value.lower() == "true" + + +def _decode_options_from_args(args: argparse.Namespace) -> dict[str, bool]: + return { + "keep_char_cells": _parse_bool_arg(args.keep_char_cells), + "keep_shapes": _parse_bool_arg(args.keep_shapes), + "keep_bitmaps": _parse_bool_arg(args.keep_bitmaps), + "create_word_cells": _parse_bool_arg(args.create_word_cells), + "create_line_cells": _parse_bool_arg(args.create_line_cells), + } + + +def _materialization_options_from_args(args: argparse.Namespace) -> dict[str, bool]: + return { + "materialize_char_cells": _parse_bool_arg(args.materialize_char_cells), + "materialize_word_cells": _parse_bool_arg(args.materialize_word_cells), + "materialize_line_cells": _parse_bool_arg(args.materialize_line_cells), + "materialize_shapes": _parse_bool_arg(args.materialize_shapes), + "materialize_bitmaps": _parse_bool_arg(args.materialize_bitmaps), + "materialize_bitmap_bytes": _parse_bool_arg(args.materialize_bitmap_bytes), + } + + +def _content_config( + decode_options: dict[str, bool], materialization_options: dict[str, bool] +) -> ContentConfig: + def _level(keep: bool, materialize: bool) -> ContentLevel: + if materialize: + return ContentLevel.COMPUTE_AND_MATERIALIZE + if keep: + return ContentLevel.COMPUTE + return ContentLevel.SKIP + + return ContentConfig( + char_cells_content_level=_level( + decode_options["keep_char_cells"], + materialization_options["materialize_char_cells"], + ), + word_cells_content_level=_level( + decode_options["create_word_cells"], + materialization_options["materialize_word_cells"], + ), + line_cells_content_level=_level( + decode_options["create_line_cells"], + materialization_options["materialize_line_cells"], + ), + shapes_content_level=_level( + decode_options["keep_shapes"], + materialization_options["materialize_shapes"], + ), + bitmaps_content_level=_level( + decode_options["keep_bitmaps"], + materialization_options["materialize_bitmaps"], + ), + include_bitmap_bytes=materialization_options["materialize_bitmap_bytes"], + ) + + +def _content_level_name(value: object) -> object: + if isinstance(value, ContentLevel): + return value.name + if isinstance(value, bool): + return str(value).lower() + return value + + +def print_effective_config( + decode_config: DecodeConfig, + content_config: ContentConfig, +) -> None: + decode_rows = [ + [key, _content_level_name(value)] + for key, value in sorted(decode_config.model_dump().items()) + ] + content_rows = [ + [key, _content_level_name(value)] + for key, value in content_config.model_dump().items() + ] + print("\nEffective decode config:") + print(tabulate(decode_rows, headers=["field", "value"])) + print("\nEffective content config:") + print(tabulate(content_rows, headers=["field", "value"])) + + # -------------- Timing extraction -------------- @@ -108,6 +288,8 @@ def extract_timings_for_page( def analyze_pages( csv_path: Path, top_n: int | None, + decode_config: DecodeConfig, + content_config: ContentConfig, min_sec: float | None = None, *, nth: int | None = None, @@ -150,7 +332,11 @@ def analyze_pages( pages.sort(key=lambda r: r.elapsed_s, reverse=True) try: doc = parser.load( - filename, lazy=True, boundary_type=PdfPageBoundaryType.CROP_BOX + filename, + lazy=True, + boundary_type=PdfPageBoundaryType.CROP_BOX, + decode_config=decode_config, + content_config=content_config, ) except Exception: # Unable to load document; record empty timings for its pages @@ -190,21 +376,35 @@ def analyze_pages( # -------------- Output: --top mode (CSV with static timings) -------------- +def ordered_static_timing_keys() -> List[str]: + ordered: List[str] = [] + + def add_children(parent: str) -> None: + for key in TIMING_CHILDREN.get(parent, []): + if key not in ordered: + ordered.append(key) + add_children(key) + + add_children("pipeline") + for key in sorted(set(get_static_timing_keys()) - set(ordered)): + ordered.append(key) + return ordered + + def write_static_timings_csv(out_path: Path, pages: List[PageTimings]) -> None: - """Write CSV with decode_page timing keys only, one row per page.""" + """Write CSV with static timing keys, one row per page.""" ensure_parent_dir(out_path) - # Get decode_page keys in order (excludes the global decode_page timer) - decode_page_keys = get_decode_page_timing_keys() + timing_keys = ordered_static_timing_keys() - header = ["filename", "page_number", "elapsed_original_sec", *decode_page_keys] + header = ["filename", "page_number", "elapsed_original_sec", *timing_keys] with out_path.open("w", newline="") as f: w = csv.writer(f) w.writerow(header) for p in pages: row = [p.filename, p.page_number, f"{p.elapsed_original:.9f}"] - for k in decode_page_keys: + for k in timing_keys: v = p.timings.get(k, 0.0) row.append(f"{v:.9f}" if v else "") w.writerow(row) @@ -221,36 +421,93 @@ def print_top_summary(pages: List[PageTimings]) -> None: def print_aggregate_breakdown(pages: List[PageTimings]) -> None: - """Average cost and share of each static timing key across the selection. + """Average cost and share of each timing bucket across the selection. - This is the whole-selection view; `--nth` gives the same breakdown for one - page. + Percentages are branch-local to avoid double-counting nested timers. The + `% original` column keeps the link to the per-page cost from run_scaling.py. """ analysed = [p for p in pages if p.timings.data] if not analysed: return total_elapsed = sum(p.elapsed_original for p in analysed) + totals = { + key: sum(p.timings.get(key, 0.0) for p in analysed) + for key in get_static_timing_keys() + } + + root_total = sum(totals.get(key, 0.0) for key in TIMING_CHILDREN["pipeline"]) rows = [] - for key in sorted(get_static_timing_keys()): - key_total = sum(p.timings.get(key, 0.0) for p in analysed) + seen: set[str] = set() + + def add_scope(parent: str) -> None: + children = TIMING_CHILDREN.get(parent, []) + sibling_total = sum(totals.get(key, 0.0) for key in children) + for key in children: + key_total = totals.get(key, 0.0) + if key_total <= 0.0: + continue + seen.add(key) + parent_pct = ( + key_total / sibling_total * 100.0 if sibling_total > 0.0 else 0.0 + ) + original_pct = ( + key_total / total_elapsed * 100.0 if total_elapsed > 0.0 else 0.0 + ) + pipeline_pct = key_total / root_total * 100.0 if root_total > 0.0 else 0.0 + rows.append( + [ + parent, + key, + f"{key_total:.6f}", + f"{key_total / len(analysed):.6f}", + f"{parent_pct:.2f}%", + f"{pipeline_pct:.2f}%", + f"{original_pct:.2f}%", + ] + ) + + for scope in TIMING_CHILDREN: + add_scope(scope) + + for key in sorted(set(get_static_timing_keys()) - seen - NESTED_TIMING_KEYS): + key_total = totals.get(key, 0.0) if key_total <= 0.0: continue - share = (key_total / total_elapsed * 100.0) if total_elapsed > 0 else 0.0 + original_pct = (key_total / total_elapsed * 100.0) if total_elapsed > 0 else 0.0 rows.append( [ + "(unmapped)", key, f"{key_total:.6f}", f"{key_total / len(analysed):.6f}", - f"{share:.2f}%", + "", + "", + f"{original_pct:.2f}%", ] ) if not rows: return - rows.sort(key=lambda r: float(r[1]), reverse=True) print(f"\nTiming breakdown across {len(analysed)} analysed pages:") - print(tabulate(rows, headers=["timing_key", "total_sec", "avg_sec", "avg_%"])) + print( + tabulate( + rows, + headers=[ + "scope", + "timing_key", + "total_sec", + "avg_sec", + "% parent", + "% pipeline", + "% original", + ], + ) + ) + print( + "\nPercentages are not meant to sum down the whole table: nested rows are " + "shown as a share of their parent branch." + ) # -------------- Output: --nth mode (table with all timings) -------------- @@ -315,9 +572,15 @@ def add_timing_row(key: str, is_static: bool): print(tabulate(table_data, headers=headers, tablefmt="grid")) # Print totals - print(f"\nTotal static time: {sum(timings.get_static_timings().values()):.6f} sec") - print(f"Total dynamic time: {sum(timings.get_dynamic_timings().values()):.6f} sec") - print(f"Total all timings: {timings.total():.6f} sec") + print( + "\nSum of reported static timings " + f"(nested; not wall time): {sum(timings.get_static_timings().values()):.6f} sec" + ) + print( + "Sum of reported dynamic timings " + f"(nested; not wall time): {sum(timings.get_dynamic_timings().values()):.6f} sec" + ) + print(f"decode_page timer: {timings.get('decode_page', 0.0):.6f} sec") # -------------- Main -------------- @@ -363,8 +626,82 @@ def main(argv: List[str]) -> int: ap.add_argument( "--threads", type=int, default=None, help="Keep only this thread count" ) + _add_bool_value_arg( + ap, + "keep-char-cells", + default=True, + help="Populate character cells and emit text render instructions", + ) + _add_bool_value_arg( + ap, + "create-word-cells", + default=False, + help="Create word cells during decoding", + ) + _add_bool_value_arg( + ap, + "create-line-cells", + default=False, + help="Create line cells during decoding", + ) + _add_bool_value_arg( + ap, + "keep-shapes", + default=False, + help="Keep vector shape cells", + ) + _add_bool_value_arg( + ap, + "keep-bitmaps", + default=False, + help="Keep bitmap resources/cells", + ) + _add_bool_value_arg( + ap, + "materialize-char-cells", + default=False, + help="Materialize character cells into SegmentedPdfPage", + ) + _add_bool_value_arg( + ap, + "materialize-word-cells", + default=False, + help="Materialize word cells into SegmentedPdfPage", + ) + _add_bool_value_arg( + ap, + "materialize-line-cells", + default=True, + help="Materialize line cells into SegmentedPdfPage", + ) + _add_bool_value_arg( + ap, + "materialize-shapes", + default=False, + help="Materialize vector shapes into SegmentedPdfPage", + ) + _add_bool_value_arg( + ap, + "materialize-bitmaps", + default=True, + help="Materialize bitmap locations into SegmentedPdfPage", + ) + _add_bool_value_arg( + ap, + "materialize-bitmap-bytes", + default=False, + help="Materialize bitmap image bytes when bitmap locations are materialized", + ) args = ap.parse_args(argv) + if args.task == "parse" and not _arg_was_passed(argv, "materialize-bitmaps"): + args.materialize_bitmaps = "false" + + decode_config = DecodeConfig() + content_config = _content_config( + _decode_options_from_args(args), + _materialization_options_from_args(args), + ) # Validate arguments if args.top is None and args.nth is None: @@ -384,6 +721,8 @@ def main(argv: List[str]) -> int: pages = analyze_pages( csv_path, top_n=args.top, + decode_config=decode_config, + content_config=content_config, min_sec=args.min_sec, nth=args.nth, loglevel=args.loglevel, @@ -402,6 +741,7 @@ def main(argv: List[str]) -> int: # Output based on mode if args.nth is not None: # --nth mode: print detailed table + print_effective_config(decode_config, content_config) print_nth_table(pages[0]) else: # --top mode: write CSV with static timings @@ -410,6 +750,7 @@ def main(argv: List[str]) -> int: ) write_static_timings_csv(out_path, pages) print_top_summary(pages) + print_effective_config(decode_config, content_config) print_aggregate_breakdown(pages) print(f"\nWrote static timings CSV: {out_path}") diff --git a/perf/run_eval.py b/perf/run_eval.py index 332701bc..ea31700d 100644 --- a/perf/run_eval.py +++ b/perf/run_eval.py @@ -324,12 +324,12 @@ def plot_histograms_stacked( def _hex_pairs_to_plot( per_series_rows: Dict[str, List[PageRow]], ) -> List[Tuple[str, str]]: - """docling-parse at one thread against every other package, per task. + """docling-parse against every other package, per task. Pairs never cross tasks --- a `parse` time against a `parse+render` time is - not a like-for-like page. Other docling-parse thread counts are excluded - too: per-page cost is the same quantity at any thread count, so those plots - would just be the diagonal. + not a like-for-like page. Prefer docling-parse at one thread when present; + otherwise fall back to the lowest available docling-parse thread count for + that task. """ by_task: Dict[str, List[str]] = defaultdict(list) for series, rows in per_series_rows.items(): @@ -338,18 +338,27 @@ def _hex_pairs_to_plot( pairs: List[Tuple[str, str]] = [] for task, names in sorted(by_task.items()): - reference = next( + references = sorted( ( s for s in names if per_series_rows[s][0].backend == REFERENCE_BACKEND - and per_series_rows[s][0].threads == 1 ), - None, + key=lambda s: ( + per_series_rows[s][0].threads != 1, + per_series_rows[s][0].threads, + s, + ), ) + reference = references[0] if references else None if reference is None: - print(f" no {REFERENCE_BACKEND} (1t) series for task {task}; no hexbins") + print(f" no {REFERENCE_BACKEND} series for task {task}; no hexbins") continue + if per_series_rows[reference][0].threads != 1: + print( + f" no {REFERENCE_BACKEND} (1t) series for task {task}; " + f"using {reference} for hexbins" + ) pairs.extend( (reference, other) for other in names diff --git a/perf/run_scaling.py b/perf/run_scaling.py index e3bedc9f..14a53a3e 100644 --- a/perf/run_scaling.py +++ b/perf/run_scaling.py @@ -64,6 +64,7 @@ PageRow, find_pdfs, percentile, + read_page_rows, series_label, write_page_rows, ) @@ -125,6 +126,43 @@ def resolve_pdf_inputs( return find_pdfs(pdf_dir, recursive=True), info +def filter_pdf_selection(pdf_paths: List[Path], selection: str) -> List[Path]: + """Filter PDFs by comma/semicolon-separated filename, stem, or path tokens.""" + tokens = [ + token.strip() + for token in re.split(r"[;,]", selection) + if token.strip() + ] + if not tokens: + return pdf_paths + + selected: List[Path] = [] + selected_set = set() + unmatched = set(tokens) + for pdf_path in pdf_paths: + path = str(pdf_path) + name = pdf_path.name + stem = pdf_path.stem + matched = [ + token + for token in tokens + if token == name or token == stem or token in path + ] + if matched and pdf_path not in selected_set: + selected.append(pdf_path) + selected_set.add(pdf_path) + unmatched.difference_update(matched) + + if unmatched: + print( + "Warning: --pdf-selection did not match: " + + ", ".join(sorted(unmatched)), + file=sys.stderr, + ) + + return selected + + def page_counts(pdf_paths: List[Path]) -> List[Tuple[Path, int]]: """Count pages per PDF using DoclingPdfParser.""" from docling_parse.pdf_parser import DoclingPdfParser @@ -384,6 +422,11 @@ def _add_bool_value_arg( ) +def _arg_was_passed(argv: List[str], name: str) -> bool: + option = f"--{name}" + return any(arg == option or arg.startswith(f"{option}=") for arg in argv) + + def _decode_options_from_args(args: argparse.Namespace) -> dict[str, bool]: return { "keep_char_cells": args.keep_char_cells, @@ -1369,6 +1412,131 @@ def parse_compare_arg(arg: str) -> List[str]: return names +ComparisonSeriesKey = Tuple[str, str, int] + + +def comparison_series_key( + backend: str, task: str, threads: int +) -> ComparisonSeriesKey: + return (backend, task, threads) + + +def comparison_series_sort_key(key: ComparisonSeriesKey) -> Tuple[int, int, int]: + backend, task, threads = key + return (_TASK_ORDER[task], _BACKEND_ORDER[backend], threads) + + +def expected_comparison_series( + backends: List[str], + tasks: List[str], + thread_counts: List[int], +) -> List[ComparisonSeriesKey]: + series: List[ComparisonSeriesKey] = [] + for task in tasks: + for backend in backends: + _, supported = COMPARISON_BACKENDS[backend] + if task not in supported: + continue + threads_for_backend = ( + thread_counts if backend in THREADED_COMPARISON_BACKENDS else [1] + ) + for threads in threads_for_backend: + series.append(comparison_series_key(backend, task, threads)) + series.sort(key=comparison_series_sort_key) + return series + + +def expected_page_keys( + schedule: List[Tuple[Path, List[int] | None]], +) -> set[Tuple[str, int]]: + keys: set[Tuple[str, int]] = set() + for pdf_path, page_numbers in schedule: + if page_numbers is None: + continue + doc_key = str(pdf_path) + for page_number in page_numbers: + keys.add((doc_key, page_number)) + return keys + + +def series_is_complete( + rows: List[PageRow], + expected_pages: set[Tuple[str, int]], + total_pages: int, +) -> bool: + if len(rows) < total_pages: + return False + if not expected_pages: + return True + row_pages = {(row.doc_key, row.page_number) for row in rows} + return expected_pages.issubset(row_pages) + + +def complete_existing_comparison_rows( + csv_path: Path, + desired_series: set[ComparisonSeriesKey], + expected_pages: set[Tuple[str, int]], + total_pages: int, +) -> Tuple[List[PageRow], set[ComparisonSeriesKey]]: + if not csv_path.exists(): + return [], set() + + rows = read_page_rows(csv_path) + grouped: Dict[ComparisonSeriesKey, List[PageRow]] = {} + for row in rows: + key = comparison_series_key(row.backend, row.task, row.threads) + if key in desired_series: + grouped.setdefault(key, []).append(row) + + reusable_rows: List[PageRow] = [] + complete: set[ComparisonSeriesKey] = set() + for key, series_rows in grouped.items(): + if series_is_complete(series_rows, expected_pages, total_pages): + complete.add(key) + reusable_rows.extend(series_rows) + + return reusable_rows, complete + + +def backend_runs_from_page_rows(rows: List[PageRow]) -> List[BackendRun]: + grouped: Dict[ComparisonSeriesKey, List[PageRow]] = {} + for row in rows: + grouped.setdefault( + comparison_series_key(row.backend, row.task, row.threads), + [], + ).append(row) + + runs: List[BackendRun] = [] + for key, samples in grouped.items(): + backend, task, threads = key + wall_values = [ + sample.wall_gap_s for sample in samples if sample.wall_gap_s > 0 + ] + wall_s = sum(wall_values) if wall_values else sum( + sample.elapsed_s for sample in samples + ) + per_page_source = ( + PER_PAGE_INTERNAL + if backend in THREADED_COMPARISON_BACKENDS + else PER_PAGE_WALL + ) + runs.append( + BackendRun( + backend=backend, + task=task, + threads=threads, + wall_s=wall_s, + per_page_source=per_page_source, + samples=samples, + ) + ) + + runs.sort( + key=lambda r: (_TASK_ORDER[r.task], _BACKEND_ORDER[r.backend], r.threads) + ) + return runs + + def run_comparison( schedule: List[Tuple[Path, List[int] | None]], total_pages: int, @@ -1381,6 +1549,7 @@ def run_comparison( decode_options: dict[str, bool], materialization_options: dict[str, bool], bytesio: bool = False, + skip_series: set[ComparisonSeriesKey] | None = None, ) -> List[BackendRun]: """Run every (backend, task, threads) combination once. @@ -1390,6 +1559,7 @@ def run_comparison( """ runs: List[BackendRun] = [] skipped: List[str] = [] + skip_series = skip_series or set() ordered = sorted( backends, key=lambda name: ( @@ -1411,6 +1581,10 @@ def run_comparison( thread_counts if name in THREADED_COMPARISON_BACKENDS else [1] ) for threads in threads_for_backend: + key = comparison_series_key(name, task, threads) + if key in skip_series: + print(f"Reusing {name} [{task}] with {threads} thread(s) from CSV") + continue label = f"{name} [{task}]" if name in THREADED_COMPARISON_BACKENDS: label += f" with {threads} thread(s)" @@ -2005,6 +2179,15 @@ def main(argv: List[str]) -> int: default=None, help="Maximum number of pages to process across all input PDFs", ) + ap.add_argument( + "--pdf-selection", + type=str, + default="", + help=( + "Comma/semicolon-separated PDF filename, stem, or path tokens to " + "benchmark after input resolution, e.g. '1235350.pdf;1061589.pdf'" + ), + ) ap.add_argument( "--max-concurrent-results", type=int, @@ -2131,6 +2314,8 @@ def main(argv: List[str]) -> int: ) args = ap.parse_args(argv) + if args.mode == "parse" and not _arg_was_passed(argv, "materialize-bitmaps"): + args.materialize_bitmaps = False # Recorded verbatim in the report so a published number can be traced back # to the invocation that produced it. @@ -2146,6 +2331,8 @@ def main(argv: List[str]) -> int: system_info = collect_system_info() pdfs, dataset_info = resolve_pdf_inputs(args.input, recursive=args.recursive) + if args.pdf_selection: + pdfs = filter_pdf_selection(pdfs, args.pdf_selection) if not pdfs: print(f"No PDFs found for input: {args.input}", file=sys.stderr) return 2 @@ -2165,6 +2352,7 @@ def main(argv: List[str]) -> int: "other backends": other_backends or "(none)", "comparison suite": compare_backends or "(off)", "max pages": args.max_pages if args.max_pages else "(all)", + "pdf selection": args.pdf_selection or "(all)", "bytesio": args.bytesio, } if args.mode in ("render", "both"): @@ -2206,8 +2394,28 @@ def main(argv: List[str]) -> int: if args.mode in ("render", "both"): tasks.append(TASK_RENDER) + desired_series = set( + expected_comparison_series(compare_backends, tasks, thread_counts) + ) + reusable_rows, reusable_series = complete_existing_comparison_rows( + pages_csv_path, + desired_series, + expected_page_keys(pdf_schedule), + total_pages, + ) + if reusable_series: + print( + f"Found {len(reusable_series)} complete comparison series in " + f"{pages_csv_path}; only missing series will be measured." + ) + for backend, task, threads in sorted( + reusable_series, + key=comparison_series_sort_key, + ): + print(f" - {backend} [{task}] with {threads} thread(s)") + print("\n##### COMPARISON SUITE #####") - runs = run_comparison( + new_runs = run_comparison( pdf_schedule, total_pages, compare_backends, @@ -2218,6 +2426,11 @@ def main(argv: List[str]) -> int: decode_options=decode_options, materialization_options=materialization_options, bytesio=args.bytesio, + skip_series=reusable_series, + ) + runs = backend_runs_from_page_rows(reusable_rows) + new_runs + runs.sort( + key=lambda r: (_TASK_ORDER[r.task], _BACKEND_ORDER[r.backend], r.threads) ) print_comparison_table(runs) print_speedup_table(runs) diff --git a/scripts/check_no_blend2d_assertions.py b/scripts/check_no_blend2d_assertions.py new file mode 100644 index 00000000..340be5b8 --- /dev/null +++ b/scripts/check_no_blend2d_assertions.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python + +import sys +import zipfile +from pathlib import Path + + +# Guards that release wheels do not ship Blend2D's debug assertion path, i.e. +# that Blend2D was compiled with BL_BUILD_RELEASE (see cmake/extlib_blend2d.cmake). +# +# Do NOT scan for "[Blend2D] ASSERTION FAILURE" or "bl_runtime_assertion_failure". +# Blend2D compiles bl_runtime_assertion_failure() unconditionally in +# blend2d/core/runtime.cpp -- there is no #ifdef around it -- and its object file +# is always pulled out of libblend2d.a because other symbols in it are needed. +# BL_BUILD_RELEASE only removes the *call sites* (BL_ASSERT in blend2d/core/api.h, +# BL_NOT_REACHED in blend2d/core/api-internal_p.h). So the function and its message +# string enter the link on every platform, and whether they survive into the final +# artifact depends purely on how well the linker dead-strips: +# +# macOS ld64 drops unreferenced cstring literal atoms, plus pybind11's +# POST_BUILD `strip -x`. +# manylinux -ffunction-sections + -Wl,--gc-sections; the literal lives in a +# SHF_MERGE|SHF_STRINGS .rodata.str1.1 and is dropped along with it. +# win_arm64 MSVC: /O2 implies /Gy, the linker's /OPT:REF drops the COMDAT, and +# symbol names go to the PDB rather than into the image. +# win_amd64 MinGW GCC targeting PE/COFF: nothing removes it. --gc-sections for +# COFF/PE is experimental, and COFF has no SHF_MERGE/SHF_STRINGS +# equivalent -- mingw GCC emits all of a translation unit's string +# literals into one shared .rdata, and -fdata-sections does not split +# anonymous string constants. The rest of runtime.cpp.obj keeps that +# .rdata alive, so section-granularity GC can never reach the string. +# +# Those markers therefore produced a false positive on win_amd64 (the only GNU-on-PE +# job in the matrix) for a correctly configured Release build. +# +# BL_NOT_REACHED()'s expression string is a marker that is impossible in a release +# build: only the BL_BUILD_DEBUG expansion emits it as a string literal, while the +# GCC/Clang release expansion is __builtin_unreachable(). With ~100 call sites across +# the library, at least one lands in any linked debug build. +BLEND2D_ASSERTION_MARKERS = (b"BL_NOT_REACHED()",) +NATIVE_SUFFIXES = {".so", ".pyd", ".dll", ".dylib"} + + +def native_files(path: Path) -> list[Path]: + if path.is_file(): + if path.suffix.lower() == ".whl" or path.suffix.lower() in NATIVE_SUFFIXES: + return [path] + return [] + return [ + item + for item in path.rglob("*") + if item.is_file() and item.suffix.lower() in NATIVE_SUFFIXES + ] + + +def scan_file(path: Path) -> list[str]: + if path.suffix.lower() == ".whl": + findings: list[str] = [] + with zipfile.ZipFile(path) as zf: + for name in zf.namelist(): + if Path(name).suffix.lower() not in NATIVE_SUFFIXES: + continue + data = zf.read(name) + if has_blend2d_assertion_marker(data): + findings.append(f"{path}!{name}") + return findings + + try: + data = path.read_bytes() + except OSError as exc: + print(f"warning: could not read {path}: {exc}", file=sys.stderr) + return [] + + return [str(path)] if has_blend2d_assertion_marker(data) else [] + + +def has_blend2d_assertion_marker(data: bytes) -> bool: + return any(marker in data for marker in BLEND2D_ASSERTION_MARKERS) + + +def main(argv: list[str]) -> int: + if not argv: + print( + "usage: check_no_blend2d_assertions.py ...", + file=sys.stderr, + ) + return 2 + + findings: list[str] = [] + for arg in argv: + path = Path(arg) + if not path.exists(): + print(f"warning: path does not exist: {path}", file=sys.stderr) + continue + for item in native_files(path): + findings.extend(scan_file(item)) + + if findings: + print( + "Blend2D debug assertion code found in release artifact " + "(Blend2D was not compiled with BL_BUILD_RELEASE):", + file=sys.stderr, + ) + for finding in findings: + print(f" {finding}", file=sys.stderr) + return 1 + + print("No Blend2D debug assertion code found.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/src/parse.h b/src/parse.h index 7d210a92..52ad5cc4 100644 --- a/src/parse.h +++ b/src/parse.h @@ -53,6 +53,7 @@ #include #include +#include #include #include #include diff --git a/src/parse/config.h b/src/parse/config.h index aed5628b..ad0111a4 100644 --- a/src/parse/config.h +++ b/src/parse/config.h @@ -8,6 +8,10 @@ namespace pdflib struct decode_config { + static constexpr double DEFAULT_HORIZONTAL_CELL_TOLERANCE = 1.0; + static constexpr double DEFAULT_WORD_SPACE_WIDTH_FACTOR_FOR_MERGE = 0.33; + static constexpr double DEFAULT_LINE_SPACE_WIDTH_FACTOR_FOR_MERGE = 1.0; + static constexpr double DEFAULT_LINE_SPACE_WIDTH_FACTOR_FOR_MERGE_WITH_SPACE = 0.33; static constexpr double DEFAULT_MIN_VISIBLE_CLIP_EXTENT = 1e-3; std::string page_boundary = "crop_box"; @@ -27,14 +31,15 @@ namespace pdflib bool enforce_same_font = true; // word & line cell creation // word & line cell creation parameters - double horizontal_cell_tolerance = 1.0; + double horizontal_cell_tolerance = DEFAULT_HORIZONTAL_CELL_TOLERANCE; // word cell creation - double word_space_width_factor_for_merge = 0.33; + double word_space_width_factor_for_merge = DEFAULT_WORD_SPACE_WIDTH_FACTOR_FOR_MERGE; // line cell creation - double line_space_width_factor_for_merge = 1.0; - double line_space_width_factor_for_merge_with_space = 0.33; + double line_space_width_factor_for_merge = DEFAULT_LINE_SPACE_WIDTH_FACTOR_FOR_MERGE; + double line_space_width_factor_for_merge_with_space = + DEFAULT_LINE_SPACE_WIDTH_FACTOR_FOR_MERGE_WITH_SPACE; bool populate_json_objects = false; diff --git a/src/parse/page_item_sanitators/cells.h b/src/parse/page_item_sanitators/cells.h index 556e98c9..9fc7e777 100644 --- a/src/parse/page_item_sanitators/cells.h +++ b/src/parse/page_item_sanitators/cells.h @@ -352,6 +352,11 @@ namespace pdflib { std::string& text = cells.at(i).text; + if(text.empty()) + { + continue; + } + for(const std::pair& pair:text_constants::replacements) { utils::string::replace(text, pair.first, pair.second); @@ -359,11 +364,16 @@ namespace pdflib } { - std::regex pattern(R"(^\/([A-Za-z])_([A-Za-z])(_([A-Za-z]))?$)"); + static const std::regex pattern(R"(^\/([A-Za-z])_([A-Za-z])(_([A-Za-z]))?$)"); for(int i=0; i word_cells; page_item line_cells; + bool sanitised_cells_created = false; bool word_cells_created = false; bool line_cells_created = false; @@ -809,6 +812,7 @@ namespace pdflib void pdf_decoder::decode_page(const decode_config& config) { page_config = config; + sanitised_cells_created = false; if(owned_qpdf_document != nullptr) { @@ -882,11 +886,17 @@ namespace pdflib page_item_sanitator sanitator; { + utils::timer step_timer; sanitator.remove_duplicate_cells(page_cells, 0.5, true); + timings.add_timing(pdf_timings::KEY_SANITIZE_CELLS_REMOVE_DUPLICATE_CELLS, + step_timer.get_time()); } { + utils::timer step_timer; sanitator.sanitize_text(page_cells); + timings.add_timing(pdf_timings::KEY_SANITIZE_CELLS_SANITIZE_TEXT, + step_timer.get_time()); } timings.add_timing(pdf_timings::KEY_SANITIZE_CELLS, local.get_time()); } @@ -1828,6 +1838,19 @@ namespace pdflib page_hyperlinks.rotate(angle, delta); } + bool pdf_decoder::can_reuse_sanitised_cells_for_line_cells(const decode_config& config) const + { + return (sanitised_cells_created and + config.do_sanitization and + config.enforce_same_font and + (std::abs(config.horizontal_cell_tolerance - + config.DEFAULT_HORIZONTAL_CELL_TOLERANCE) < 1.e-6) and + (std::abs(config.line_space_width_factor_for_merge - + config.DEFAULT_LINE_SPACE_WIDTH_FACTOR_FOR_MERGE) < 1.e-6) and + (std::abs(config.line_space_width_factor_for_merge_with_space - + config.DEFAULT_LINE_SPACE_WIDTH_FACTOR_FOR_MERGE_WITH_SPACE) < 1.e-6)); + } + void pdf_decoder::sanitise_contents(std::string page_boundary) { LOG_S(INFO) << __FUNCTION__; @@ -1847,20 +1870,35 @@ namespace pdflib //sanitator.remove_duplicate_chars(page_cells, 0.5); //sanitator.sanitize_text(page_cells); - cells = page_cells; + { + utils::timer step_timer; + cells = page_cells; + timings.add_timing(pdf_timings::KEY_SANITISE_CONTENTS_COPY_CELLS, + step_timer.get_time()); + } - double horizontal_cell_tolerance=1.0; + double horizontal_cell_tolerance = + decode_config::DEFAULT_HORIZONTAL_CELL_TOLERANCE; bool enforce_same_font=true; //double space_width_factor_for_merge=1.5; - double space_width_factor_for_merge=1.0; - double space_width_factor_for_merge_with_space=0.33; + double space_width_factor_for_merge = + decode_config::DEFAULT_LINE_SPACE_WIDTH_FACTOR_FOR_MERGE; + double space_width_factor_for_merge_with_space = + decode_config::DEFAULT_LINE_SPACE_WIDTH_FACTOR_FOR_MERGE_WITH_SPACE; - sanitator.sanitize_bbox(cells, - horizontal_cell_tolerance, - enforce_same_font, - space_width_factor_for_merge, - space_width_factor_for_merge_with_space, - false); + { + utils::timer step_timer; + sanitator.sanitize_bbox(cells, + horizontal_cell_tolerance, + enforce_same_font, + space_width_factor_for_merge, + space_width_factor_for_merge_with_space, + false); + timings.add_timing(pdf_timings::KEY_SANITISE_CONTENTS_SANITIZE_BBOX, + step_timer.get_time()); + } + + sanitised_cells_created = true; //sanitator.sanitize_text(cells); @@ -1876,10 +1914,55 @@ namespace pdflib page_item_sanitator sanitizer; - word_cells = sanitizer.create_word_cells(page_cells, config); + { + utils::timer step_timer; + word_cells = page_cells; + timings.add_timing(pdf_timings::KEY_CREATE_WORD_CELLS_COPY_CELLS, + step_timer.get_time()); + } + + LOG_S(INFO) << "#-char cells: " << word_cells.size(); + + { + utils::timer step_timer; + double space_width_factor_for_merge_with_space = + 2.0*config.word_space_width_factor_for_merge; + + sanitizer.sanitize_bbox(word_cells, + config.horizontal_cell_tolerance, + config.enforce_same_font, + config.word_space_width_factor_for_merge, + space_width_factor_for_merge_with_space, + true); + timings.add_timing(pdf_timings::KEY_CREATE_WORD_CELLS_SANITIZE_BBOX, + step_timer.get_time()); + } + + { + utils::timer step_timer; + auto itr = word_cells.begin(); + while(itr != word_cells.end()) + { + if(utils::string::is_space(itr->text)) + { + itr = word_cells.erase(itr); + } + else + { + itr++; + } + } + timings.add_timing(pdf_timings::KEY_CREATE_WORD_CELLS_ERASE_SPACES, + step_timer.get_time()); + } // Remove duplicates (quadratic but necessary) - sanitizer.remove_duplicate_cells(word_cells, 0.5, true); + { + utils::timer step_timer; + sanitizer.remove_duplicate_cells(word_cells, 0.5, true); + timings.add_timing(pdf_timings::KEY_CREATE_WORD_CELLS_REMOVE_DUPLICATE_CELLS, + step_timer.get_time()); + } word_cells_created = true; @@ -1893,11 +1976,40 @@ namespace pdflib utils::timer timer; page_item_sanitator sanitizer; + const bool reuse_sanitised_cells = + can_reuse_sanitised_cells_for_line_cells(config); - line_cells = sanitizer.create_line_cells(page_cells, config); + { + utils::timer step_timer; + line_cells = reuse_sanitised_cells ? cells : page_cells; + timings.add_timing(pdf_timings::KEY_CREATE_LINE_CELLS_COPY_CELLS, + step_timer.get_time()); + } + + LOG_S(INFO) << "# char-cells: " << line_cells.size(); + + if(!reuse_sanitised_cells) + { + utils::timer step_timer; + sanitizer.sanitize_bbox(line_cells, + config.horizontal_cell_tolerance, + config.enforce_same_font, + config.line_space_width_factor_for_merge, + config.line_space_width_factor_for_merge_with_space, + false); + timings.add_timing(pdf_timings::KEY_CREATE_LINE_CELLS_SANITIZE_BBOX, + step_timer.get_time()); + } + + LOG_S(INFO) << "# line-cells: " << line_cells.size(); // Remove duplicates (quadratic but necessary) - sanitizer.remove_duplicate_cells(line_cells, 0.5, true); + { + utils::timer step_timer; + sanitizer.remove_duplicate_cells(line_cells, 0.5, true); + timings.add_timing(pdf_timings::KEY_CREATE_LINE_CELLS_REMOVE_DUPLICATE_CELLS, + step_timer.get_time()); + } line_cells_created = true; diff --git a/src/parse/pdf_decoders/stream.h b/src/parse/pdf_decoders/stream.h index c7e04677..ed8da969 100644 --- a/src/parse/pdf_decoders/stream.h +++ b/src/parse/pdf_decoders/stream.h @@ -500,8 +500,8 @@ namespace pdflib for(size_t i = 0; i + 1 < parameters.size(); i += 2) { - const qpdf_stream_instruction& key_instruction = parameters[i]; - const qpdf_stream_instruction& value_instruction = parameters[i + 1]; + qpdf_stream_instruction key_instruction = parameters[i]; + qpdf_stream_instruction value_instruction = parameters[i + 1]; if(not key_instruction.obj.isName()) { @@ -511,7 +511,7 @@ namespace pdflib } const std::string key = canonical_name(key_instruction.obj.getName()); - const QPDFObjectHandle& value = value_instruction.obj; + QPDFObjectHandle value = value_instruction.obj; if(key == "/Width" and value.isInteger()) { @@ -580,7 +580,8 @@ namespace pdflib return; } - inline_image.data = instruction.obj.getInlineImageValue(); + QPDFObjectHandle obj = instruction.obj; + inline_image.data = obj.getInlineImageValue(); inline_image.has_data = true; } diff --git a/src/parse/pdf_resources/page_font.h b/src/parse/pdf_resources/page_font.h index a7ce6561..3faa58de 100644 --- a/src/parse/pdf_resources/page_font.h +++ b/src/parse/pdf_resources/page_font.h @@ -78,8 +78,7 @@ namespace pdflib // only needed for the cmap-resource files bool numb_is_in_cmap(uint32_t c); - void set(std::string font_key_, - nlohmann::json& json_font_, + void set(std::string font_key_, QPDFObjectHandle qpdf_font_); private: @@ -96,14 +95,11 @@ namespace pdflib void init_font_matrix(); void init_font_program(); bool try_init_font_program_from_descriptor(QPDFObjectHandle font_obj, - nlohmann::json const& font_json, bool from_descendant_font); bool try_init_font_program_direct(QPDFObjectHandle font_obj, - nlohmann::json const& font_json, bool from_descendant_font); void populate_font_program(QPDFObjectHandle descriptor_obj, QPDFObjectHandle stream_obj, - nlohmann::json const& descriptor_json, std::string const& source_path, embedded_font_file_kind kind, bool from_descendant_font); @@ -147,10 +143,9 @@ namespace pdflib pdf_timings& timings; nlohmann::json json_font; - nlohmann::json desc_font; // derived from json_font, only for '/Type-0' QPDFObjectHandle qpdf_font; - //QPDFObjectHandle qpdf_desc_font; // derived from json_font, only for '/Type-0' + QPDFObjectHandle qpdf_desc_font; // derived from qpdf_font, only for '/Type-0' std::string encoding_name; font_encoding_name encoding; @@ -296,6 +291,11 @@ namespace pdflib nlohmann::json pdf_resource::get() { + if(json_font.is_null() and not qpdf_font.isNull()) + { + json_font = to_json(qpdf_font); + } + return json_font; } @@ -680,8 +680,7 @@ namespace pdflib } } - void pdf_resource::set(std::string font_key_, - nlohmann::json& json_font_, + void pdf_resource::set(std::string font_key_, QPDFObjectHandle qpdf_font_) { LOG_S(INFO) << __FUNCTION__ << " font: " << font_key_; @@ -693,7 +692,7 @@ namespace pdflib try { - LOG_S(INFO) << "font [key='" << font_key_ << "']:\n" << json_font_.dump(2); + LOG_S(INFO) << "font [key='" << font_key_ << "']:\n" << qpdf_object::debug(qpdf_font_); } catch(std::exception e) { @@ -707,8 +706,9 @@ namespace pdflib utils::timer font_timer; font_key = font_key_; - json_font = json_font_; + json_font = nullptr; qpdf_font = qpdf_font_; + qpdf_desc_font = QPDFObjectHandle::newNull(); double font_time = font_timer.get_time(); timings.add_timing(pdf_timings::KEY_FONT_INIT_COPY, font_time); @@ -787,26 +787,20 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__; - std::vector keys_0 = {"/Encoding", "/BaseEncoding"}; - std::vector keys_1 = {"/Encoding"}; - std::string name; - if(utils::json::has(keys_0, json_font)) + if(qpdf_object::get_name_or_string(qpdf_font, {"/Encoding", "/BaseEncoding"}, name)) { - name = utils::json::get(keys_0, json_font); encoding = to_encoding_name(name); has_explicit_encoding = true; LOG_S(INFO) << "font-encoding [" << name << "]: " << to_string(encoding); } - else if(utils::json::has(keys_1, json_font)) + else if(qpdf_font.isDictionary() and qpdf_font.hasKey("/Encoding")) { - auto result = utils::json::get(keys_1, json_font); + auto result = qpdf_font.getKey("/Encoding"); - if(result.is_string()) + if(qpdf_object::get_name_or_string(result, encoding_name)) { - encoding_name = result.get(); - if(cids.has(encoding_name)) { encoding = CMAP_RESOURCES; @@ -849,17 +843,10 @@ namespace pdflib LOG_S(INFO) << "font-encoding [" << name << "]: " << to_string(encoding); } - else if(result.is_object() && result.count("/BaseEncoding") == 1 && result["/BaseEncoding"].is_string()) - { - // Extract /BaseEncoding from encoding dictionary - std::string base_enc = result["/BaseEncoding"].get(); - encoding = to_encoding_name(base_enc); - has_explicit_encoding = true; - LOG_S(INFO) << "font-encoding from object /BaseEncoding [" << base_enc << "]: " << to_string(encoding); - } else { - LOG_S(WARNING) << " --> font-encoding falling back to STANDARD with font-encoding [object]: " << result.dump(); + LOG_S(WARNING) << " --> font-encoding falling back to STANDARD with font-encoding [object]: " + << qpdf_object::debug(result); encoding = STANDARD; has_explicit_encoding = false; @@ -877,26 +864,20 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__; - std::vector keys = {"/Subtype"}; - - if(utils::json::has(keys, json_font)) + std::string name; + if(qpdf_object::get_name_or_string(qpdf_font, {"/Subtype"}, name)) { - std::string name = utils::json::get(keys, json_font); subtype = to_subtype_name(name); LOG_S(INFO) << "subtype [" << name << "]: " << to_string(subtype); - std::vector keys_0 = {"/DescendantFonts"}; - if(subtype==TYPE_0 and utils::json::has(keys_0, json_font)) + QPDFObjectHandle desc_fonts = qpdf_object::get_path(qpdf_font, {"/DescendantFonts"}); + if(subtype==TYPE_0 and desc_fonts.isArray()) { - auto desc_fonts = utils::json::get(keys_0, json_font); - - if(desc_fonts.size()==1) + if(desc_fonts.getArrayNItems()==1) { LOG_S(INFO) << "found the descendant font";// << desc_font.dump(2); - desc_font = desc_fonts[0]; - - //qpdf_desc_font = qpdf_font.getKey(keys_0.at(0)).getArrayItem(0); + qpdf_desc_font = desc_fonts.getArrayItem(0); } else { @@ -918,7 +899,7 @@ namespace pdflib else { subtype=NULL_TYPE; - LOG_S(ERROR) << "could not find subtype in font: " << json_font.dump(2); + LOG_S(ERROR) << "could not find subtype in font: " << qpdf_object::debug(qpdf_font); } } @@ -926,17 +907,13 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__; - std::vector keys = {"/BaseFont"}; - base_font = "null"; - if(utils::json::has(keys, json_font)) + if(qpdf_object::get_name_or_string(qpdf_font, {"/BaseFont"}, base_font)) { - base_font = utils::json::get(keys, json_font); LOG_S(INFO) << "base-font: " << base_font; } - else if(utils::json::has(keys, desc_font)) + else if(qpdf_object::get_name_or_string(qpdf_desc_font, {"/BaseFont"}, base_font)) { - base_font = utils::json::get(keys, desc_font); LOG_S(INFO) << "base-font: " << base_font; } else @@ -949,23 +926,17 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__; - std::vector keys_0 = {"/FontDescriptor", "/FontName"}; - std::vector keys_1 = {"/Name"}; - font_name = "null"; - if(utils::json::has(keys_0, json_font)) + if(qpdf_object::get_name_or_string(qpdf_font, {"/FontDescriptor", "/FontName"}, font_name)) { - font_name = utils::json::get(keys_0, json_font); LOG_S(INFO) << "font-name: " << font_name; } - else if(utils::json::has(keys_0, desc_font)) + else if(qpdf_object::get_name_or_string(qpdf_desc_font, {"/FontDescriptor", "/FontName"}, font_name)) { - font_name = utils::json::get(keys_0, desc_font); LOG_S(INFO) << "font-name: " << font_name; } - else if(utils::json::has(keys_1, json_font)) + else if(qpdf_object::get_name_or_string(qpdf_font, {"/Name"}, font_name)) { - font_name = utils::json::get(keys_1, json_font); LOG_S(INFO) << "font-name: " << font_name; } else if(base_font!="null") @@ -983,29 +954,25 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__;// << "\t" << json_font.dump(2); - std::vector keys_0 = {"/FontDescriptor", "/FontBBox"}; - std::vector keys_1 = {"/FontBBox"}; - nlohmann::json json_bbox; + bool found_bbox = false; - if(utils::json::has(keys_0, json_font)) + if(qpdf_object::get_number_array(qpdf_font, {"/FontDescriptor", "/FontBBox"}, font_bbox)) { - json_bbox = utils::json::get(keys_0, json_font); + found_bbox = true; } - else if(utils::json::has(keys_0, desc_font)) + else if(qpdf_object::get_number_array(qpdf_desc_font, {"/FontDescriptor", "/FontBBox"}, font_bbox)) { - json_bbox = utils::json::get(keys_0, desc_font); + found_bbox = true; } - else if(utils::json::has(keys_1, json_font)) + else if(qpdf_object::get_number_array(qpdf_font, {"/FontBBox"}, font_bbox)) { //assert(subtype==TYPE_3); - - json_bbox = utils::json::get(keys_1, json_font); + found_bbox = true; } - else if(utils::json::has(keys_1, desc_font)) + else if(qpdf_object::get_number_array(qpdf_desc_font, {"/FontBBox"}, font_bbox)) { //assert(subtype==TYPE_3); - - json_bbox = utils::json::get(keys_1, desc_font); + found_bbox = true; } else if(bfonts.has(base_font)==1) { @@ -1017,18 +984,16 @@ namespace pdflib LOG_S(WARNING) << "could not find font-bbox"; } - if (json_bbox != nullptr) + if (not found_bbox) { - if (json_bbox.is_array() and json_bbox.size() == 4) - { - for(int d=0; d<4; d++) - { - font_bbox[d] = json_bbox[d].get(); - } - } - else + QPDFObjectHandle bbox = qpdf_object::get_path(qpdf_font, {"/FontDescriptor", "/FontBBox"}); + if(bbox.isNull()) { bbox = qpdf_object::get_path(qpdf_desc_font, {"/FontDescriptor", "/FontBBox"}); } + if(bbox.isNull()) { bbox = qpdf_object::get_path(qpdf_font, {"/FontBBox"}); } + if(bbox.isNull()) { bbox = qpdf_object::get_path(qpdf_desc_font, {"/FontBBox"}); } + if(not bbox.isNull()) { - LOG_S(ERROR) << "expected 4 elements in font-bbox, got: " << json_bbox; + LOG_S(ERROR) << "expected 4 numeric elements in font-bbox, got: " + << qpdf_object::debug(bbox); } } @@ -1044,26 +1009,11 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__;// << "\t" << json_font.dump(2); - std::vector keys_0 = {"/FontMatrix"}; - - if(utils::json::has(keys_0, json_font)) + if(qpdf_object::get_number_array(qpdf_font, {"/FontMatrix"}, font_matrix)) { //assert(subtype==TYPE_3); - auto json_matrix = utils::json::get(keys_0, json_font); - - if (json_matrix.is_array() and json_matrix.size() == 6) - { - for(int d=0; d<6; d++) - { - font_matrix[d] = json_matrix[d].get(); - } - type3_xscale = font_matrix[0] * 1000.0; - type3_yscale = font_matrix[3] * 1000.0; - } - else - { - LOG_S(ERROR) << "expected 6 elements in font-matrix, got: " << json_matrix; - } + type3_xscale = font_matrix[0] * 1000.0; + type3_yscale = font_matrix[3] * 1000.0; } else { @@ -1094,43 +1044,29 @@ namespace pdflib bool found = false; LOG_S(INFO) << __FUNCTION__ << ": probing primary font descriptor"; - found = try_init_font_program_from_descriptor(qpdf_font, json_font, false); + found = try_init_font_program_from_descriptor(qpdf_font, false); - if(not found and subtype==TYPE_0 and qpdf_font.hasKey("/DescendantFonts")) + if(not found and subtype==TYPE_0 and not qpdf_desc_font.isNull()) { LOG_S(INFO) << __FUNCTION__ << ": probing descendant font descriptor"; - auto desc_fonts = qpdf_font.getKey("/DescendantFonts"); - if(desc_fonts.isArray() and desc_fonts.getArrayNItems() > 0) + found = try_init_font_program_from_descriptor(qpdf_desc_font, true); + if(not found) { - auto qpdf_desc_font = desc_fonts.getArrayItem(0); - found = try_init_font_program_from_descriptor(qpdf_desc_font, desc_font, true); - if(not found) - { - LOG_S(INFO) << __FUNCTION__ << ": probing descendant font directly"; - found = try_init_font_program_direct(qpdf_desc_font, desc_font, true); - } - } - else - { - LOG_S(INFO) << __FUNCTION__ << ": /DescendantFonts missing or empty at qpdf level"; + LOG_S(INFO) << __FUNCTION__ << ": probing descendant font directly"; + found = try_init_font_program_direct(qpdf_desc_font, true); } } if(not found) { LOG_S(INFO) << __FUNCTION__ << ": probing primary font object directly"; - found = try_init_font_program_direct(qpdf_font, json_font, false); + found = try_init_font_program_direct(qpdf_font, false); } - if(not found and subtype==TYPE_0 and qpdf_font.hasKey("/DescendantFonts")) + if(not found and subtype==TYPE_0 and not qpdf_desc_font.isNull()) { - auto desc_fonts = qpdf_font.getKey("/DescendantFonts"); - if(desc_fonts.isArray() and desc_fonts.getArrayNItems() > 0) - { - LOG_S(INFO) << __FUNCTION__ << ": probing descendant font directly as final fallback"; - auto qpdf_desc_font = desc_fonts.getArrayItem(0); - found = try_init_font_program_direct(qpdf_desc_font, desc_font, true); - } + LOG_S(INFO) << __FUNCTION__ << ": probing descendant font directly as final fallback"; + found = try_init_font_program_direct(qpdf_desc_font, true); } if(found) @@ -1155,7 +1091,6 @@ namespace pdflib bool pdf_resource::try_init_font_program_from_descriptor( QPDFObjectHandle font_obj, - nlohmann::json const& font_json, bool from_descendant_font) { LOG_S(INFO) << __FUNCTION__ @@ -1180,12 +1115,6 @@ namespace pdflib return false; } - nlohmann::json descriptor_json = nullptr; - if(font_json.is_object() and font_json.count("/FontDescriptor") == 1) - { - descriptor_json = font_json["/FontDescriptor"]; - } - struct candidate_spec { const char* key; @@ -1215,7 +1144,6 @@ namespace pdflib populate_font_program(descriptor_obj, stream_obj, - descriptor_json, std::string("/FontDescriptor") + spec.key, spec.kind, from_descendant_font); @@ -1228,7 +1156,6 @@ namespace pdflib bool pdf_resource::try_init_font_program_direct( QPDFObjectHandle font_obj, - nlohmann::json const& font_json, bool from_descendant_font) { LOG_S(INFO) << __FUNCTION__ @@ -1269,7 +1196,6 @@ namespace pdflib populate_font_program(font_obj, stream_obj, - font_json, spec.key, spec.kind, from_descendant_font); @@ -1283,7 +1209,6 @@ namespace pdflib void pdf_resource::populate_font_program( QPDFObjectHandle descriptor_obj, QPDFObjectHandle stream_obj, - nlohmann::json const& descriptor_json, std::string const& source_path, embedded_font_file_kind kind, bool from_descendant_font) @@ -1300,7 +1225,7 @@ namespace pdflib font_program.base_font = base_font; font_program.font_name = font_name; font_program.from_descendant_font = from_descendant_font; - font_program.descriptor_json = descriptor_json; + font_program.descriptor_json = to_json(descriptor_obj, {}, 0, 2); font_program.stream_dict_json = to_json(stream_obj, {}, 0, 2); // Disabled: dumping the full stream dictionary per font floods the logs. // LOG_S(INFO) << __FUNCTION__ @@ -1453,14 +1378,14 @@ namespace pdflib // PDF spec: /CIDToGIDMap defaults to /Identity when absent. Any stream // value means an explicit map that we do not resolve here. - std::vector keys = {"/CIDToGIDMap"}; - if(not utils::json::has(keys, desc_font)) + QPDFObjectHandle value = qpdf_object::get_path(qpdf_desc_font, {"/CIDToGIDMap"}); + if(value.isNull()) { return true; } - nlohmann::json value = utils::json::get(keys, desc_font); - return value.is_string() and value.get() == "/Identity"; + std::string name; + return qpdf_object::get_name_or_string(value, name) and name == "/Identity"; } void pdf_resource::build_embedded_font_blob() @@ -1491,17 +1416,12 @@ namespace pdflib const int FLAG_SYMBOLIC = 1 << 2; // /Flags bit 3 int flags = 0; - std::vector keys = {"/FontDescriptor", "/Flags"}; - if(utils::json::has(keys, json_font)) - { - flags = utils::json::get(keys, json_font); - } - else if(utils::json::has(keys, desc_font)) + if(not qpdf_object::get_int(qpdf_font, {"/FontDescriptor", "/Flags"}, flags)) { - flags = utils::json::get(keys, desc_font); + qpdf_object::get_int(qpdf_desc_font, {"/FontDescriptor", "/Flags"}, flags); } - const bool has_encoding = utils::json::has({"/Encoding"}, json_font); + const bool has_encoding = qpdf_object::has_path(qpdf_font, {"/Encoding"}); uses_builtin_encoding = (subtype != TYPE_0) and ((flags & FLAG_SYMBOLIC) != 0) and (not has_encoding); @@ -1536,19 +1456,15 @@ namespace pdflib ascent=0; { - std::vector keys = {"/FontDescriptor", "/Ascent"}; - bool ascent_defined=false; - if(utils::json::has(keys, json_font)) + if(qpdf_object::get_number(qpdf_font, {"/FontDescriptor", "/Ascent"}, ascent)) { - ascent = utils::json::get(keys, json_font); ascent_defined=true; LOG_S(INFO) << "ascent: " << ascent; } - else if(utils::json::has(keys, desc_font)) + else if(qpdf_object::get_number(qpdf_desc_font, {"/FontDescriptor", "/Ascent"}, ascent)) { - ascent = utils::json::get(keys, desc_font); ascent_defined=true; LOG_S(INFO) << "ascent: " << ascent; @@ -1581,19 +1497,15 @@ namespace pdflib descent=0; { - std::vector keys = {"/FontDescriptor", "/Descent"}; - bool descent_defined=false; - if(utils::json::has(keys, json_font)) + if(qpdf_object::get_number(qpdf_font, {"/FontDescriptor", "/Descent"}, descent)) { - descent = utils::json::get(keys, json_font); descent_defined=true; LOG_S(INFO) << "descent: " << descent; } - else if(utils::json::has(keys, desc_font)) + else if(qpdf_object::get_number(qpdf_desc_font, {"/FontDescriptor", "/Descent"}, descent)) { - descent = utils::json::get(keys, desc_font); descent_defined=true; LOG_S(INFO) << "descent: " << descent; @@ -1645,19 +1557,15 @@ namespace pdflib capheight=0; { - std::vector keys = {"/FontDescriptor", "/CapHeight"}; - //bool capheight_defined=false; - if(utils::json::has(keys, json_font)) + if(qpdf_object::get_number(qpdf_font, {"/FontDescriptor", "/CapHeight"}, capheight)) { - capheight = utils::json::get(keys, json_font); //capheight_defined=true; LOG_S(INFO) << "capheight: " << capheight; } - else if(utils::json::has(keys, desc_font)) + else if(qpdf_object::get_number(qpdf_desc_font, {"/FontDescriptor", "/CapHeight"}, capheight)) { - capheight = utils::json::get(keys, desc_font); //capheight_defined=true; LOG_S(INFO) << "capheight: " << capheight; @@ -1680,19 +1588,15 @@ namespace pdflib xheight=0; { - std::vector keys = {"/FontDescriptor", "/XHeight"}; - //bool xheight_defined=false; - if(utils::json::has(keys, json_font)) + if(qpdf_object::get_number(qpdf_font, {"/FontDescriptor", "/XHeight"}, xheight)) { - xheight = utils::json::get(keys, json_font); //xheight_defined=true; LOG_S(INFO) << "xheight: " << xheight; } - else if(utils::json::has(keys, desc_font)) + else if(qpdf_object::get_number(qpdf_desc_font, {"/FontDescriptor", "/XHeight"}, xheight)) { - xheight = utils::json::get(keys, desc_font); //xheight_defined=true; LOG_S(INFO) << "xheight: " << xheight; @@ -1723,19 +1627,15 @@ namespace pdflib has_default_width=false; - std::vector f_keys = {"/DW"}; - - if(utils::json::has(f_keys, json_font)) + if(qpdf_object::get_number(qpdf_font, {"/DW"}, default_width)) { has_default_width = true; - default_width = utils::json::get(f_keys, json_font).get(); LOG_S(INFO) << "default-width: " << default_width; } - else if(utils::json::has(f_keys, desc_font)) + else if(qpdf_object::get_number(qpdf_desc_font, {"/DW"}, default_width)) { has_default_width = true; - default_width = utils::json::get(f_keys, desc_font).get(); LOG_S(INFO) << "default-width: " << default_width; } @@ -1768,17 +1668,21 @@ namespace pdflib LOG_S(INFO) << "vertical writing mode from encoding " << encoding_name; { - std::vector keys = {"/DW2"}; - - nlohmann::json dw2; - if(utils::json::has(keys, json_font)) { dw2 = utils::json::get(keys, json_font); } - else if(utils::json::has(keys, desc_font)) { dw2 = utils::json::get(keys, desc_font); } + QPDFObjectHandle dw2 = qpdf_object::get_path(qpdf_font, {"/DW2"}); + if(dw2.isNull()) { dw2 = qpdf_object::get_path(qpdf_desc_font, {"/DW2"}); } - if(dw2.is_array() and dw2.size() >= 2 and - dw2[0].is_number() and dw2[1].is_number()) + if(dw2.isArray() and dw2.getArrayNItems() >= 2) { - vertical_origin_y = dw2[0].get() / 1000.0; - default_vertical_displacement = dw2[1].get() / 1000.0; + double origin_y = 0.0; + double displacement = 0.0; + QPDFObjectHandle origin_obj = dw2.getArrayItem(0); + QPDFObjectHandle displacement_obj = dw2.getArrayItem(1); + if(qpdf_object::get_number(origin_obj, origin_y) and + qpdf_object::get_number(displacement_obj, displacement)) + { + vertical_origin_y = origin_y / 1000.0; + default_vertical_displacement = displacement / 1000.0; + } } LOG_S(INFO) << "vertical metrics: origin-y " << vertical_origin_y @@ -1788,31 +1692,30 @@ namespace pdflib // /W2 entries come as `c [w1 v_x v_y ...]` or `c_first c_last w1 v_x v_y`; // only w1 is read, since the horizontal half of v is taken as w0/2 either // way and its vertical half rarely differs from /DW2. - std::vector keys = {"/W2"}; - - nlohmann::json w2; - if(utils::json::has(keys, json_font)) { w2 = utils::json::get(keys, json_font); } - else if(utils::json::has(keys, desc_font)) { w2 = utils::json::get(keys, desc_font); } + QPDFObjectHandle w2 = qpdf_object::get_path(qpdf_font, {"/W2"}); + if(w2.isNull()) { w2 = qpdf_object::get_path(qpdf_desc_font, {"/W2"}); } - if(not w2.is_array()) + if(not w2.isArray()) { return; } - for(std::size_t l = 0; l + 1 < w2.size(); ) + for(int l = 0; l + 1 < w2.getArrayNItems(); ) { - if(not w2[l].is_number()) + QPDFObjectHandle beg_obj = w2.getArrayItem(l); + if(not beg_obj.isNumber()) { LOG_S(WARNING) << "/W2 entry " << l << " is not a CID"; break; } - const int beg = w2[l].get(); + const int beg = static_cast(utils::numeric::locale_safe_numeric_value(beg_obj)); l += 1; - if(w2[l].is_array()) + QPDFObjectHandle value_obj = w2.getArrayItem(l); + if(value_obj.isArray()) { - std::vector triples = w2[l].get>(); + std::vector triples = qpdf_object::get_number_array(value_obj); l += 1; for(std::size_t k = 0; k + 2 < triples.size(); k += 3) @@ -1821,10 +1724,18 @@ namespace pdflib triples[k] / 1000.0; } } - else if(l + 3 < w2.size()) + else if(l + 3 < w2.getArrayNItems()) { - const int end = w2[l].get(); - const double w1 = w2[l + 1].get() / 1000.0; + QPDFObjectHandle end_obj = w2.getArrayItem(l); + QPDFObjectHandle w1_obj = w2.getArrayItem(l + 1); + if(not end_obj.isNumber() or not w1_obj.isNumber()) + { + LOG_S(WARNING) << "/W2 range contains non-numeric values"; + break; + } + + const int end = static_cast(utils::numeric::locale_safe_numeric_value(end_obj)); + const double w1 = utils::numeric::locale_safe_numeric_value(w1_obj) / 1000.0; l += 4; // c_last, w1, v_x, v_y for(int id = beg; id <= end; id++) @@ -1868,15 +1779,12 @@ namespace pdflib fchar=-1; - std::vector f_keys = {"/FirstChar"}; - if(utils::json::has(f_keys, json_font)) + if(qpdf_object::get_int(qpdf_font, {"/FirstChar"}, fchar)) { - fchar = utils::json::get(f_keys, json_font).get(); LOG_S(INFO) << "fchar: " << fchar; } - else if(utils::json::has(f_keys, desc_font)) + else if(qpdf_object::get_int(qpdf_desc_font, {"/FirstChar"}, fchar)) { - fchar = utils::json::get(f_keys, desc_font).get(); LOG_S(INFO) << "fchar: " << fchar; } else @@ -1891,15 +1799,12 @@ namespace pdflib lchar=-1; - std::vector l_keys = {"/LastChar"}; - if(utils::json::has(l_keys, json_font)) + if(qpdf_object::get_int(qpdf_font, {"/LastChar"}, lchar)) { - lchar = utils::json::get(l_keys, json_font).get(); LOG_S(INFO) << "lchar: " << lchar; } - else if(utils::json::has(l_keys, desc_font)) + else if(qpdf_object::get_int(qpdf_desc_font, {"/LastChar"}, lchar)) { - lchar = utils::json::get(l_keys, desc_font).get(); LOG_S(INFO) << "lchar: " << lchar; } else @@ -1914,32 +1819,17 @@ namespace pdflib std::vector values={}; { - std::vector keys = {"/Widths"}; - bool found_widths = false; - if(utils::json::has(keys, json_font) and (not found_widths)) - { - auto result = utils::json::get(keys, json_font); - LOG_S(INFO) << "widths: " << result.dump(); + QPDFObjectHandle widths = qpdf_object::get_path(qpdf_font, {"/Widths"}); + if(widths.isNull()) { widths = qpdf_object::get_path(qpdf_desc_font, {"/Widths"}); } - if(result.is_array()) - { - values = result.get >(); - found_widths = true; - } - } - else if(utils::json::has(keys, desc_font) and (not found_widths)) + if(widths.isArray() and (not found_widths)) { - auto result = utils::json::get(keys, desc_font); - LOG_S(INFO) << "widths: " << result.dump(); - - if(result.is_array()) - { - values = result.get >(); - found_widths = true; - } + LOG_S(INFO) << "widths: " << qpdf_object::debug(widths); + values = qpdf_object::get_number_array(widths); + found_widths = true; } - else if(not found_widths) + if(not found_widths) { LOG_S(WARNING) << "could not find widths"; } @@ -1975,20 +1865,13 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__; - nlohmann::json ws; + QPDFObjectHandle ws; { - std::vector keys = {"/W"}; + ws = qpdf_object::get_path(qpdf_font, {"/W"}); + if(ws.isNull()) { ws = qpdf_object::get_path(qpdf_desc_font, {"/W"}); } - if(utils::json::has(keys, json_font)) - { - ws = utils::json::get(keys, json_font); - } - else if(utils::json::has(keys, desc_font)) - { - ws = utils::json::get(keys, desc_font); - } - else + if(not ws.isArray()) { LOG_S(WARNING) << "could not find '/W'"; return; @@ -2000,13 +1883,20 @@ namespace pdflib int beg=-1; int end=-1; - for(int l=0; l(); + if(not beg_obj.isNumber()) + { + LOG_S(WARNING) << "/W entry " << l << " is not a CID"; + break; + } + + beg = static_cast(utils::numeric::locale_safe_numeric_value(beg_obj)); l += 1; if(l==0) @@ -2014,31 +1904,32 @@ namespace pdflib fchar=beg; } - if(ws[l].is_number()) + if(l>=ws.getArrayNItems()) { - //LOG_S(INFO) << l << "\t" << ws[l].is_number() << "\t end: " << ws[l].dump(); + LOG_S(WARNING) << "index " << l << " is out of bounds " << ws.getArrayNItems(); + continue; + } - //assert(l(utils::numeric::locale_safe_numeric_value(value_obj)); + l += 1; - if(l>=ws.size()) + if(l>=ws.getArrayNItems()) { - LOG_S(WARNING) << "index " << l << " is out of bounds " << ws.size(); + LOG_S(WARNING) << "index " << l << " is out of bounds " << ws.getArrayNItems(); continue; } - - end = ws[l].get(); - l += 1; - //LOG_S(INFO) << l << "\t" << ws[l].is_number() << "\t w: " << ws[l].dump(); + QPDFObjectHandle width_obj = ws.getArrayItem(l); + if(not width_obj.isNumber()) + { + LOG_S(WARNING) << "/W range width is not numeric: " << qpdf_object::debug(width_obj); + break; + } - //assert(l=ws.size()) - { - LOG_S(WARNING) << "index " << l << " is out of bounds " << ws.size(); - continue; - } - - double w = ws[l].get(); + double w = utils::numeric::locale_safe_numeric_value(width_obj); l += 1; for(int id=beg; id<=end; id++) @@ -2047,18 +1938,9 @@ namespace pdflib numb_to_widths[id] = w * type3_xscale; } } - else if(ws[l].is_array()) + else if(value_obj.isArray()) { - //LOG_S(INFO) << l << "\t" << ws[l].is_number() << "\t widths: " << ws[l].dump(); - - //assert(l=ws.size()) - { - LOG_S(WARNING) << "index " << l << " is out of bounds " << ws.size(); - continue; - } - - std::vector w = ws[l].get >(); + std::vector w = qpdf_object::get_number_array(value_obj); l += 1; for(int k=0; k keys = { "/ToUnicode" }; + QPDFObjectHandle qpdf_obj = qpdf_object::get_path(qpdf_font, {"/ToUnicode"}); - if(utils::json::has(keys, json_font)) + if(not qpdf_obj.isNull()) { LOG_S(INFO) << "found a /ToUnicode cmap: starting to decode ..."; - if(not qpdf_font.hasKey("/ToUnicode")) - { - auto tmp = to_json(qpdf_font); - - std::stringstream ss; - ss << "qpdf-font: " << tmp.dump(); - - LOG_S(ERROR) << ss.str(); - throw std::logic_error(ss.str()); - } - - auto qpdf_obj = qpdf_font.getKey("/ToUnicode"); - if(qpdf_obj.isStream()) { std::vector stream; @@ -2140,24 +2009,21 @@ namespace pdflib } else if(qpdf_obj.isString()) { - auto _ = to_json(qpdf_obj); - std::string message = "qpdf_obj.isString(): " + _.dump(2); + std::string message = "qpdf_obj.isString(): " + qpdf_object::debug(qpdf_obj); LOG_S(ERROR) << message; throw std::logic_error(message); } else if(qpdf_obj.isName()) { - auto _ = to_json(qpdf_obj); - std::string message = "qpdf_obj.isName(): " + _.dump(2); + std::string message = "qpdf_obj.isName(): " + qpdf_object::debug(qpdf_obj); LOG_S(ERROR) << message; //throw std::logic_error(message); } else { - auto _ = to_json(qpdf_obj); - std::string message = "qpdf_obj is unknown: " + _.dump(2); + std::string message = "qpdf_obj is unknown: " + qpdf_object::debug(qpdf_obj); LOG_S(ERROR) << message; throw std::logic_error(message); @@ -2191,12 +2057,12 @@ namespace pdflib } //else - if(subtype==TYPE_0 and desc_font!=NULL and + if(subtype==TYPE_0 and not qpdf_desc_font.isNull() and cids.has(encoding_name) ) { try { - LOG_S(INFO) << "descendant-font: " << desc_font.dump(2); + LOG_S(INFO) << "descendant-font: " << qpdf_object::debug(qpdf_desc_font); } catch(const std::exception& exc) { @@ -2221,11 +2087,11 @@ namespace pdflib cmap_initialized = false; } } - else if(subtype==TYPE_0 and desc_font!=NULL) + else if(subtype==TYPE_0 and not qpdf_desc_font.isNull()) { try { - LOG_S(INFO) << "descendant-font: " << desc_font.dump(2); + LOG_S(INFO) << "descendant-font: " << qpdf_object::debug(qpdf_desc_font); } catch(const std::exception& exc) { @@ -2236,13 +2102,17 @@ namespace pdflib LOG_S(INFO) << "encoding-type: " << to_string(encoding); LOG_S(INFO) << "encoding-name: " << encoding_name; - std::vector key_registry = {"/CIDSystemInfo", "/Registry"}; - std::vector key_ordering = {"/CIDSystemInfo", "/Ordering"}; - std::vector key_supplement = {"/CIDSystemInfo", "/Supplement"}; - - std::string registry_ = utils::json::get(key_registry, desc_font).get(); - std::string ordering_ = utils::json::get(key_ordering, desc_font).get(); - int supplement_ = utils::json::get(key_supplement, desc_font).get(); + std::string registry_; + std::string ordering_; + int supplement_ = 0; + if(not qpdf_object::get_name_or_string(qpdf_desc_font, {"/CIDSystemInfo", "/Registry"}, registry_) or + not qpdf_object::get_name_or_string(qpdf_desc_font, {"/CIDSystemInfo", "/Ordering"}, ordering_) or + not qpdf_object::get_int(qpdf_desc_font, {"/CIDSystemInfo", "/Supplement"}, supplement_)) + { + LOG_S(ERROR) << "incomplete /CIDSystemInfo in descendant font"; + cmap_initialized = false; + return; + } LOG_S(INFO) << "found descendant-font without /ToUnicode"; LOG_S(INFO) << " --> registry: " << registry_; @@ -2320,7 +2190,7 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__; - std::vector keys = { "/Encoding", "/Differences" }; + QPDFObjectHandle diffs = qpdf_object::get_path(qpdf_font, {"/Encoding", "/Differences"}); // Create a regex object std::regex re_01(R"(\/(.+)\.(.+))"); @@ -2362,26 +2232,24 @@ namespace pdflib return result; }; - if(utils::json::has(keys, json_font)) + if(not diffs.isNull()) { - auto diffs = utils::json::get(keys, json_font); //LOG_S(INFO) << "diffs: " << diffs.dump(2); - if(diffs.is_array()) + if(diffs.isArray()) { int numb=-1; std::string name="null"; - for(int l=0; l(); + numb = static_cast(utils::numeric::locale_safe_numeric_value(diff)); } - else if(diffs[l].is_string()) + else if(qpdf_object::get_name_or_string(diff, name)) { - name = diffs[l].get(); - // Object to hold the match results std::smatch match; @@ -2545,14 +2413,15 @@ namespace pdflib } else { - LOG_S(WARNING) << "item [" << diffs[l].dump(2) - << "] is not a string nor a number in the difference-vector: " << diffs.dump(2); + LOG_S(WARNING) << "item [" << qpdf_object::debug(diff) + << "] is not a string nor a number in the difference-vector: " + << qpdf_object::debug(diffs); } } } else { - LOG_S(WARNING) << "/Differences is not a vector: " << diffs.dump(2); + LOG_S(WARNING) << "/Differences is not a vector: " << qpdf_object::debug(diffs); } diff_initialized = true; @@ -2568,21 +2437,22 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__; - std::vector keys = { "/CharProcs" }; + QPDFObjectHandle qpdf_char_procs = qpdf_object::get_path(qpdf_font, {"/CharProcs"}); - if(utils::json::has(keys, json_font)) + if(not qpdf_char_procs.isNull()) { //assert(subtype==TYPE_3); - QPDFObjectHandle qpdf_char_procs = qpdf_font.getKey(keys.front()); LOG_S(WARNING) << "found CharProcs: " << qpdf_char_procs.getTypeName(); - - auto json_char_procs = utils::json::get(keys, json_font); - - for(auto& pair : json_char_procs.items()) + + if(not qpdf_char_procs.isDictionary()) { - std::string key = pair.key(); + LOG_S(WARNING) << "/CharProcs is not a dictionary"; + return; + } + for(auto& key : qpdf_char_procs.getKeys()) + { if(qpdf_char_procs.hasKey(key)) { QPDFObjectHandle qpdf_char_proc = qpdf_char_procs.getKey(key); diff --git a/src/parse/pdf_resources/page_fonts.h b/src/parse/pdf_resources/page_fonts.h index d65f6170..3f2ae143 100644 --- a/src/parse/pdf_resources/page_fonts.h +++ b/src/parse/pdf_resources/page_fonts.h @@ -140,12 +140,9 @@ namespace pdflib utils::timer font_timer; QPDFObjectHandle qpdf_font = qpdf_fonts.getKey(key); - nlohmann::json json_font = to_json(qpdf_font); - - LOG_S(INFO) << json_font.dump(2); pdf_resource page_font(timings); - page_font.set(key, json_font, qpdf_font); + page_font.set(key, qpdf_font); if(page_fonts.count(key)==1) { diff --git a/src/parse/qpdf/object.h b/src/parse/qpdf/object.h new file mode 100644 index 00000000..4b0907ea --- /dev/null +++ b/src/parse/qpdf/object.h @@ -0,0 +1,203 @@ +//-*-C++-*- + +#ifndef QPDF_OBJECT_ACCESS_H +#define QPDF_OBJECT_ACCESS_H + +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace pdflib +{ + namespace qpdf_object + { + + inline QPDFObjectHandle get_path(QPDFObjectHandle obj, + const std::vector& keys) + { + for(const auto& key : keys) + { + if(obj.isStream()) + { + obj = obj.getDict(); + } + + if(not obj.isDictionary() or not obj.hasKey(key)) + { + return QPDFObjectHandle::newNull(); + } + + obj = obj.getKey(key); + } + + return obj; + } + + inline bool has_path(QPDFObjectHandle obj, + const std::vector& keys) + { + return not get_path(obj, keys).isNull(); + } + + inline std::string debug(QPDFObjectHandle obj) + { + if(obj.isNull()) + { + return "null"; + } + + try + { + std::string result = obj.unparse(); + constexpr std::size_t max_size = 512; + if(result.size() > max_size) + { + result.resize(max_size); + result += "..."; + } + return result; + } + catch(const std::exception& exc) + { + return std::string("<") + obj.getTypeName() + ": " + exc.what() + ">"; + } + } + + inline std::string sanitize_utf8(const std::string& val) + { + if(utf8::is_valid(val.begin(), val.end())) + { + return val; + } + + std::string result; + utf8::replace_invalid(val.begin(), val.end(), std::back_inserter(result)); + return result; + } + + inline bool get_name_or_string(QPDFObjectHandle obj, std::string& result) + { + if(obj.isName()) + { + result = sanitize_utf8(obj.getName()); + return true; + } + if(obj.isString()) + { + result = sanitize_utf8(obj.getUTF8Value()); + return true; + } + + return false; + } + + inline bool get_name_or_string(QPDFObjectHandle obj, + const std::vector& keys, + std::string& result) + { + return get_name_or_string(get_path(obj, keys), result); + } + + inline bool get_int(QPDFObjectHandle obj, int& result) + { + if(obj.isInteger()) + { + result = obj.getIntValue(); + return true; + } + + return false; + } + + inline bool get_int(QPDFObjectHandle obj, + const std::vector& keys, + int& result) + { + return get_int(get_path(obj, keys), result); + } + + inline bool get_number(QPDFObjectHandle obj, double& result) + { + if(obj.isNumber()) + { + result = utils::numeric::locale_safe_numeric_value(obj); + return true; + } + + return false; + } + + inline bool get_number(QPDFObjectHandle obj, + const std::vector& keys, + double& result) + { + return get_number(get_path(obj, keys), result); + } + + inline std::vector get_number_array(QPDFObjectHandle obj) + { + std::vector result; + if(not obj.isArray()) + { + return result; + } + + for(int i=0; i get_number_array(QPDFObjectHandle obj, + const std::vector& keys) + { + return get_number_array(get_path(obj, keys)); + } + + template + inline bool get_number_array(QPDFObjectHandle obj, + std::array& result) + { + if(not obj.isArray() or obj.getArrayNItems() != static_cast(N)) + { + return false; + } + + for(std::size_t i=0; i(i)); + if(not get_number(item, result[i])) + { + return false; + } + } + + return true; + } + + template + inline bool get_number_array(QPDFObjectHandle obj, + const std::vector& keys, + std::array& result) + { + return get_number_array(get_path(obj, keys), result); + } + + } +} + +#endif diff --git a/src/parse/utils/pdf_timings.h b/src/parse/utils/pdf_timings.h index 065e528b..afc6995c 100644 --- a/src/parse/utils/pdf_timings.h +++ b/src/parse/utils/pdf_timings.h @@ -40,6 +40,15 @@ namespace pdflib static const std::string KEY_SANITISE_CONTENTS; static const std::string KEY_CREATE_WORD_CELLS; static const std::string KEY_CREATE_LINE_CELLS; + static const std::string KEY_SANITISE_CONTENTS_COPY_CELLS; + static const std::string KEY_SANITISE_CONTENTS_SANITIZE_BBOX; + static const std::string KEY_CREATE_WORD_CELLS_COPY_CELLS; + static const std::string KEY_CREATE_WORD_CELLS_SANITIZE_BBOX; + static const std::string KEY_CREATE_WORD_CELLS_ERASE_SPACES; + static const std::string KEY_CREATE_WORD_CELLS_REMOVE_DUPLICATE_CELLS; + static const std::string KEY_CREATE_LINE_CELLS_COPY_CELLS; + static const std::string KEY_CREATE_LINE_CELLS_SANITIZE_BBOX; + static const std::string KEY_CREATE_LINE_CELLS_REMOVE_DUPLICATE_CELLS; // Additional decode_page step keys static const std::string KEY_TO_JSON_PAGE; @@ -47,6 +56,8 @@ namespace pdflib static const std::string KEY_ROTATE_CONTENTS; static const std::string KEY_SANITIZE_ORIENTATION; static const std::string KEY_SANITIZE_CELLS; + static const std::string KEY_SANITIZE_CELLS_REMOVE_DUPLICATE_CELLS; + static const std::string KEY_SANITIZE_CELLS_SANITIZE_TEXT; // Font timing keys static const std::string KEY_DECODE_FONTS_TOTAL; @@ -419,6 +430,15 @@ namespace pdflib const std::string pdf_timings::KEY_SANITISE_CONTENTS = "sanitise_contents"; const std::string pdf_timings::KEY_CREATE_WORD_CELLS = "create_word_cells"; const std::string pdf_timings::KEY_CREATE_LINE_CELLS = "create_line_cells"; + const std::string pdf_timings::KEY_SANITISE_CONTENTS_COPY_CELLS = "sanitise_contents.copy_cells"; + const std::string pdf_timings::KEY_SANITISE_CONTENTS_SANITIZE_BBOX = "sanitise_contents.sanitize_bbox"; + const std::string pdf_timings::KEY_CREATE_WORD_CELLS_COPY_CELLS = "create_word_cells.copy_cells"; + const std::string pdf_timings::KEY_CREATE_WORD_CELLS_SANITIZE_BBOX = "create_word_cells.sanitize_bbox"; + const std::string pdf_timings::KEY_CREATE_WORD_CELLS_ERASE_SPACES = "create_word_cells.erase_spaces"; + const std::string pdf_timings::KEY_CREATE_WORD_CELLS_REMOVE_DUPLICATE_CELLS = "create_word_cells.remove_duplicate_cells"; + const std::string pdf_timings::KEY_CREATE_LINE_CELLS_COPY_CELLS = "create_line_cells.copy_cells"; + const std::string pdf_timings::KEY_CREATE_LINE_CELLS_SANITIZE_BBOX = "create_line_cells.sanitize_bbox"; + const std::string pdf_timings::KEY_CREATE_LINE_CELLS_REMOVE_DUPLICATE_CELLS = "create_line_cells.remove_duplicate_cells"; const std::string pdf_timings::KEY_DECODE_FONTS_TOTAL = "decode_fonts_total"; const std::string pdf_timings::KEY_FONT_INIT_COPY = "font: init-copy"; @@ -441,6 +461,8 @@ namespace pdflib const std::string pdf_timings::KEY_ROTATE_CONTENTS = "rotate_contents"; const std::string pdf_timings::KEY_SANITIZE_ORIENTATION = "sanitize_orientation"; const std::string pdf_timings::KEY_SANITIZE_CELLS = "sanitize_cells"; + const std::string pdf_timings::KEY_SANITIZE_CELLS_REMOVE_DUPLICATE_CELLS = "sanitize_cells.remove_duplicate_cells"; + const std::string pdf_timings::KEY_SANITIZE_CELLS_SANITIZE_TEXT = "sanitize_cells.sanitize_text"; const std::string pdf_timings::KEY_PROCESS_DOCUMENT_FROM_FILE = "process_document_from_file"; const std::string pdf_timings::KEY_PROCESS_DOCUMENT_FROM_BYTESIO = "process_document_from_bytesio"; @@ -476,6 +498,15 @@ namespace pdflib KEY_SANITISE_CONTENTS, KEY_CREATE_WORD_CELLS, KEY_CREATE_LINE_CELLS, + KEY_SANITISE_CONTENTS_COPY_CELLS, + KEY_SANITISE_CONTENTS_SANITIZE_BBOX, + KEY_CREATE_WORD_CELLS_COPY_CELLS, + KEY_CREATE_WORD_CELLS_SANITIZE_BBOX, + KEY_CREATE_WORD_CELLS_ERASE_SPACES, + KEY_CREATE_WORD_CELLS_REMOVE_DUPLICATE_CELLS, + KEY_CREATE_LINE_CELLS_COPY_CELLS, + KEY_CREATE_LINE_CELLS_SANITIZE_BBOX, + KEY_CREATE_LINE_CELLS_REMOVE_DUPLICATE_CELLS, KEY_DECODE_FONTS_TOTAL, KEY_FONT_INIT_COPY, KEY_FONT_INIT_METRICS, @@ -499,6 +530,8 @@ namespace pdflib KEY_ROTATE_CONTENTS, KEY_SANITIZE_ORIENTATION, KEY_SANITIZE_CELLS, + KEY_SANITIZE_CELLS_REMOVE_DUPLICATE_CELLS, + KEY_SANITIZE_CELLS_SANITIZE_TEXT, KEY_PROCESS_DOCUMENT_FROM_FILE, KEY_PROCESS_DOCUMENT_FROM_BYTESIO, KEY_QPDF_PROCESS, @@ -564,6 +597,8 @@ namespace pdflib {KEY_SANITIZE_ORIENTATION, KEY_DECODE_PAGE}, {KEY_SANITIZE_CELLS, KEY_DECODE_PAGE}, {KEY_SANITISE_CONTENTS, KEY_DECODE_PAGE}, + {KEY_SANITIZE_CELLS_REMOVE_DUPLICATE_CELLS, KEY_SANITIZE_CELLS}, + {KEY_SANITIZE_CELLS_SANITIZE_TEXT, KEY_SANITIZE_CELLS}, // --- decode_contents sub-timings --- {KEY_CONTENT_DECODE_TOTAL, KEY_DECODE_CONTENTS}, @@ -576,8 +611,17 @@ namespace pdflib {KEY_DO_IMAGE_TOTAL, KEY_DECODE_CONTENTS}, // --- sanitise_contents sub-timings --- + {KEY_SANITISE_CONTENTS_COPY_CELLS, KEY_SANITISE_CONTENTS}, + {KEY_SANITISE_CONTENTS_SANITIZE_BBOX, KEY_SANITISE_CONTENTS}, {KEY_CREATE_WORD_CELLS, KEY_SANITISE_CONTENTS}, {KEY_CREATE_LINE_CELLS, KEY_SANITISE_CONTENTS}, + {KEY_CREATE_WORD_CELLS_COPY_CELLS, KEY_CREATE_WORD_CELLS}, + {KEY_CREATE_WORD_CELLS_SANITIZE_BBOX, KEY_CREATE_WORD_CELLS}, + {KEY_CREATE_WORD_CELLS_ERASE_SPACES, KEY_CREATE_WORD_CELLS}, + {KEY_CREATE_WORD_CELLS_REMOVE_DUPLICATE_CELLS, KEY_CREATE_WORD_CELLS}, + {KEY_CREATE_LINE_CELLS_COPY_CELLS, KEY_CREATE_LINE_CELLS}, + {KEY_CREATE_LINE_CELLS_SANITIZE_BBOX, KEY_CREATE_LINE_CELLS}, + {KEY_CREATE_LINE_CELLS_REMOVE_DUPLICATE_CELLS, KEY_CREATE_LINE_CELLS}, // --- font sub-timings (aggregated across fonts) --- {KEY_FONT_INIT_COPY, KEY_DECODE_FONTS_TOTAL}, diff --git a/tests/constants.py b/tests/constants.py index 4ed50cda..767dbae3 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,5 +1,5 @@ HF_DATASET_REPO_ID = "docling-project/regression-dataset-for-docling-parse" -HF_DATASET_REVISION = "aeba6102e75612efd676aad80a648bd2e845ecab" +HF_DATASET_REVISION = "0e0ddfa0c87f84ca513c536b5fb5bc1cdfddaa2f" REGRESSION_DIR = "tests/data/regression"