From 9d6f351452338b25455e8fcfabce392c38c2123f Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 13 Nov 2024 16:12:09 +0100 Subject: [PATCH 1/5] listing - Correctly handle non ASCII category This is additional fix to #11177 so that category with special characters, and UTF-8 characters are correctly handled. Using base64 encoding works, in the limite of ASCII character range. For content with UTF-8 characters, we need to take extra measure. --- src/core/base64.ts | 7 +++++++ .../listing/website-listing-categories.ts | 3 ++- .../listing/website-listing-template.ts | 9 ++++++++- .../website/listing/item-default.ejs.md | 2 +- .../projects/website/listing/item-grid.ejs.md | 2 +- .../website/listing/listing-default.ejs.md | 2 +- .../website/listing/listing-grid.ejs.md | 2 +- .../website/listing/quarto-listing.js | 13 +++++++++--- .../blog/simple-blog/posts/welcome/index.qmd | 2 +- .../playwright/tests/blog-simple-blog.spec.ts | 20 +++++++++++++++++++ 10 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 src/core/base64.ts diff --git a/src/core/base64.ts b/src/core/base64.ts new file mode 100644 index 00000000000..b6f03869f32 --- /dev/null +++ b/src/core/base64.ts @@ -0,0 +1,7 @@ +export function b64EncodeUnicode(str: string) { + return btoa(encodeURIComponent(str)); +} + +export function UnicodeDecodeB64(str: string) { + return decodeURIComponent(atob(str)); +} diff --git a/src/project/types/website/listing/website-listing-categories.ts b/src/project/types/website/listing/website-listing-categories.ts index 911aeeaca07..e86d6bfa110 100644 --- a/src/project/types/website/listing/website-listing-categories.ts +++ b/src/project/types/website/listing/website-listing-categories.ts @@ -17,6 +17,7 @@ import { ListingDescriptor, ListingSharedOptions, } from "./website-listing-shared.ts"; +import { b64EncodeUnicode } from "../../../../core/base64.ts"; export function categorySidebar( doc: Document, @@ -117,7 +118,7 @@ function categoryElement( categoryEl.classList.add("category"); categoryEl.setAttribute( "data-category", - value !== undefined ? btoa(value) : btoa(category), + value !== undefined ? b64EncodeUnicode(value) : b64EncodeUnicode(category), ); categoryEl.innerHTML = contents; return categoryEl; diff --git a/src/project/types/website/listing/website-listing-template.ts b/src/project/types/website/listing/website-listing-template.ts index c3e270db035..3527eeccba2 100644 --- a/src/project/types/website/listing/website-listing-template.ts +++ b/src/project/types/website/listing/website-listing-template.ts @@ -44,6 +44,7 @@ import { formatDate, parsePandocDate } from "../../../../core/date.ts"; import { truncateText } from "../../../../core/text.ts"; import { encodeAttributeValue } from "../../../../core/html.ts"; import { imagePlaceholder, isPlaceHolder } from "./website-listing-read.ts"; +import { b64EncodeUnicode, UnicodeDecodeB64 } from "../../../../core/base64.ts"; export const kDateFormat = "date-format"; @@ -160,6 +161,12 @@ export function templateMarkdownHandler( ejsParams["metadataAttrs"] = reshapedListing.utilities.metadataAttrs; ejsParams["templateParams"] = reshapedListing["template-params"]; } + // some custom utils function + ejsParams["utils"] = { + b64encode: b64EncodeUnicode, + b64decode: UnicodeDecodeB64, + }; + return ejsParams; }; @@ -455,7 +462,7 @@ export function reshapeListing( attr["index"] = (index++).toString(); if (item.categories) { const str = (item.categories as string[]).join(","); - attr["categories"] = btoa(str); + attr["categories"] = b64EncodeUnicode(str); } // Add magic attributes for the sortable values diff --git a/src/resources/projects/website/listing/item-default.ejs.md b/src/resources/projects/website/listing/item-default.ejs.md index d39324c6716..278b153b32e 100644 --- a/src/resources/projects/website/listing/item-default.ejs.md +++ b/src/resources/projects/website/listing/item-default.ejs.md @@ -56,7 +56,7 @@ print(`
${listing.utilities.outputLi <% if (fields.includes('categories') && item.categories) { %>
<% for (const category of item.categories) { %> -
<%= category %>
+
<%= category %>
<% } %>
<% } %> diff --git a/src/resources/projects/website/listing/item-grid.ejs.md b/src/resources/projects/website/listing/item-grid.ejs.md index 8c2423bcd07..860e77ad6a2 100644 --- a/src/resources/projects/website/listing/item-grid.ejs.md +++ b/src/resources/projects/website/listing/item-grid.ejs.md @@ -64,7 +64,7 @@ return !["title", "image", "image-alt", "date", "author", "subtitle", "descripti
<% for (const category of item.categories) { %> -
<%= category %>
+
<%= category %>
<% } %>
diff --git a/src/resources/projects/website/listing/listing-default.ejs.md b/src/resources/projects/website/listing/listing-default.ejs.md index 6146d61917c..a191a600f2b 100644 --- a/src/resources/projects/website/listing/listing-default.ejs.md +++ b/src/resources/projects/website/listing/listing-default.ejs.md @@ -1,5 +1,5 @@ :::{.list .quarto-listing-default} <% for (const item of items) { %> -<% partial('item-default.ejs.md', {listing, item }) %> +<% partial('item-default.ejs.md', {listing, item, utils }) %> <% } %> ::: diff --git a/src/resources/projects/website/listing/listing-grid.ejs.md b/src/resources/projects/website/listing/listing-grid.ejs.md index 441eb8e36e1..855865c50c3 100644 --- a/src/resources/projects/website/listing/listing-grid.ejs.md +++ b/src/resources/projects/website/listing/listing-grid.ejs.md @@ -4,6 +4,6 @@ const cols = listing['grid-columns']; :::{.list .grid .quarto-listing-cols-<%=cols%>} <% for (const item of items) { %> -<% partial('item-grid.ejs.md', {listing, item }) %> +<% partial('item-grid.ejs.md', {listing, item, utils }) %> <% } %> ::: diff --git a/src/resources/projects/website/listing/quarto-listing.js b/src/resources/projects/website/listing/quarto-listing.js index ac3817ac0b2..ee86dda3a2e 100644 --- a/src/resources/projects/website/listing/quarto-listing.js +++ b/src/resources/projects/website/listing/quarto-listing.js @@ -59,7 +59,10 @@ window.document.addEventListener("DOMContentLoaded", function (_event) { ); for (const categoryEl of categoryEls) { - const category = atob(categoryEl.getAttribute("data-category")); + // category needs to support non ASCII characters + const category = decodeURIComponent( + atob(categoryEl.getAttribute("data-category")) + ); categoryEl.onclick = () => { activateCategory(category); setCategoryHash(category); @@ -209,7 +212,9 @@ function activateCategory(category) { // Activate this category const categoryEl = window.document.querySelector( - `.quarto-listing-category .category[data-category='${btoa(category)}']` + `.quarto-listing-category .category[data-category='${btoa( + encodeURIComponent(category) + )}']` ); if (categoryEl) { categoryEl.classList.add("active"); @@ -232,7 +237,9 @@ function filterListingCategory(category) { list.filter(function (item) { const itemValues = item.values(); if (itemValues.categories !== null) { - const categories = atob(itemValues.categories).split(","); + const categories = decodeURIComponent( + atob(itemValues.categories) + ).split(","); return categories.includes(category); } else { return false; diff --git a/tests/docs/playwright/blog/simple-blog/posts/welcome/index.qmd b/tests/docs/playwright/blog/simple-blog/posts/welcome/index.qmd index b8cb583f96f..d39a92a0269 100644 --- a/tests/docs/playwright/blog/simple-blog/posts/welcome/index.qmd +++ b/tests/docs/playwright/blog/simple-blog/posts/welcome/index.qmd @@ -2,7 +2,7 @@ title: "Welcome To My Blog" author: "Tristan O'Malley" date: "2024-09-03" -categories: [news] +categories: [news, 'euros (€)', 免疫] --- This is the first post in a Quarto blog. Welcome! diff --git a/tests/integration/playwright/tests/blog-simple-blog.spec.ts b/tests/integration/playwright/tests/blog-simple-blog.spec.ts index b247fe71fc4..1b952e567bd 100644 --- a/tests/integration/playwright/tests/blog-simple-blog.spec.ts +++ b/tests/integration/playwright/tests/blog-simple-blog.spec.ts @@ -1,4 +1,5 @@ import { expect, test } from "@playwright/test"; +import { getUrl } from "../src/utils"; test('List.js is correctly patch to allow searching with lowercase and uppercase', async ({ page }) => { @@ -29,4 +30,23 @@ test('Categories link are clickable', async ({ page }) => { await page.locator('div').filter({ hasText: /^news$/ }).click(); await expect(page).toHaveURL(/_site\/index\.html#category=news$/); await expect(page.locator(`div.category[data-category="${btoa('news')}"]`)).toHaveClass(/active/); +}); + +test('Categories link with special chars are clickable', async ({ page }) => { + await page.goto('./blog/simple-blog/_site/posts/welcome/'); + await page.getByRole('link', { name: 'news' }).click(); + await expect(page).toHaveURL(/_site\/index\.html#category=news$/); + await expect(page.locator(`div.category[data-category="${btoa('news')}"]`)).toHaveClass(/active/); + await page.getByRole('link', { name: 'Welcome To My Blog' }).click(); + await page.getByRole('link', { name: 'euros (€)' }).click(); + await expect(page).toHaveURL(getUrl(`blog/simple-blog/_site/index.html#category=${encodeURIComponent('euros (€)')}`)); + await expect(page.locator(`div.category[data-category="${btoa(encodeURIComponent('euros (€)'))}"]`)).toHaveClass(/active/); + await page.getByRole('link', { name: 'Welcome To My Blog' }).click(); + await page.getByRole('link', { name: '免疫' }).click(); + await expect(page).toHaveURL(getUrl(`blog/simple-blog/_site/index.html#category=${encodeURIComponent('免疫')}`)); + await expect(page.locator(`div.category[data-category="${btoa(encodeURIComponent('免疫'))}"]`)).toHaveClass(/active/); + await page.goto('./blog/simple-blog/_site/posts/welcome/#img-lst'); + await page.locator('div').filter({ hasText: /^news$/ }).click(); + await expect(page).toHaveURL(/_site\/index\.html#category=news$/); + await expect(page.locator(`div.category[data-category="${btoa('news')}"]`)).toHaveClass(/active/); }); \ No newline at end of file From 8668ceb497d82e466049514dfa8400dd966c66b3 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 13 Nov 2024 23:50:55 +0100 Subject: [PATCH 2/5] listing - avoir categories to be processed as Markdown before being inserted in templates. This also prevent +smart extension from pandoc to apply and modify some character like a single quote --- src/command/render/pandoc.ts | 5 +++++ src/project/types/website/website.ts | 16 ++++++++++++++++ .../projects/website/listing/quarto-listing.js | 4 +++- .../simple-blog/posts/post-with-code/index.qmd | 2 +- .../playwright/tests/blog-simple-blog.spec.ts | 4 ++++ 5 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/command/render/pandoc.ts b/src/command/render/pandoc.ts index d4f03d4f19c..40d3651cb78 100644 --- a/src/command/render/pandoc.ts +++ b/src/command/render/pandoc.ts @@ -207,6 +207,7 @@ import { BrandFontFile, BrandFontGoogle, } from "../../resources/types/schema-types.ts"; +import { kFieldCategories } from "../../project/types/website/listing/website-listing-shared.ts"; // in case we are running multiple pandoc processes // we need to make sure we capture all of the trace files @@ -1020,6 +1021,10 @@ export async function runPandoc( if (key === kTheme && isRevealjsOutput(options.format.pandoc)) { continue; } + // - categories are handled specifically already for website projects with a metadata override and should not be overridden by user input + if (key === kFieldCategories && projectIsWebsite(options.project)) { + continue; + } // perform the override pandocMetadata[key] = engineMetadata[key]; } diff --git a/src/project/types/website/website.ts b/src/project/types/website/website.ts index 70805522f73..2f8bc5063ee 100644 --- a/src/project/types/website/website.ts +++ b/src/project/types/website/website.ts @@ -86,6 +86,9 @@ import { formatDate } from "../../../core/date.ts"; import { projectExtensionPathResolver } from "../../../extension/extension.ts"; import { websiteDraftPostProcessor } from "./website-draft.ts"; import { projectDraftMode } from "./website-utils.ts"; +import { kFieldCategories } from "./listing/website-listing-shared.ts"; +import { pandocNativeStr } from "../../../core/pandoc/codegen.ts"; +import { asArray } from "../../../core/array.ts"; export const kSiteTemplateDefault = "default"; export const kSiteTemplateBlog = "blog"; @@ -157,6 +160,7 @@ export const websiteProjectType: ProjectType = { // add some title related variables extras.pandoc = extras.pandoc || {}; extras.metadata = extras.metadata || {}; + extras.metadataOverride = extras.metadataOverride || {}; // Resolve any giscus information resolveFormatForGiscus(project, format); @@ -196,6 +200,18 @@ export const websiteProjectType: ProjectType = { extras.metadata[kPageTitle] = title; } + // categories metadata needs to be escaped from Markdown processing to + // avoid +smart applying to it. Categories are expected to be non markdown. + // So we provide an override to ensure they are not processed. + if (format.metadata[kFieldCategories]) { + extras.metadataOverride[kFieldCategories] = asArray( + format.metadata[kFieldCategories], + ).map( + (category) => + pandocNativeStr(category as string).mappedString().value, + ); + } + // html metadata extras.html = extras.html || {}; extras.html[kHtmlPostprocessors] = extras.html[kHtmlPostprocessors] || []; diff --git a/src/resources/projects/website/listing/quarto-listing.js b/src/resources/projects/website/listing/quarto-listing.js index ee86dda3a2e..54d0e1e7f2f 100644 --- a/src/resources/projects/website/listing/quarto-listing.js +++ b/src/resources/projects/website/listing/quarto-listing.js @@ -16,7 +16,9 @@ window["quarto-listing-loaded"] = () => { if (hash) { // If there is a category, switch to that if (hash.category) { - activateCategory(hash.category); + // category hash are URI encoded so we need to decode it before processing + // so that we can match it with the category element processed in JS + activateCategory(decodeURIComponent(hash.category)); } // Paginate a specific listing const listingIds = Object.keys(window["quarto-listings"]); diff --git a/tests/docs/playwright/blog/simple-blog/posts/post-with-code/index.qmd b/tests/docs/playwright/blog/simple-blog/posts/post-with-code/index.qmd index bb7ef9d8f87..5a9b290c4ca 100644 --- a/tests/docs/playwright/blog/simple-blog/posts/post-with-code/index.qmd +++ b/tests/docs/playwright/blog/simple-blog/posts/post-with-code/index.qmd @@ -2,7 +2,7 @@ title: "Post With Code" author: "Harlow Malloc" date: "2024-09-06" -categories: [news, code, analysis] +categories: [news, code, analysis, apos'trophe] image: "image.jpg" --- diff --git a/tests/integration/playwright/tests/blog-simple-blog.spec.ts b/tests/integration/playwright/tests/blog-simple-blog.spec.ts index 1b952e567bd..7d822be2ee3 100644 --- a/tests/integration/playwright/tests/blog-simple-blog.spec.ts +++ b/tests/integration/playwright/tests/blog-simple-blog.spec.ts @@ -45,6 +45,10 @@ test('Categories link with special chars are clickable', async ({ page }) => { await page.getByRole('link', { name: '免疫' }).click(); await expect(page).toHaveURL(getUrl(`blog/simple-blog/_site/index.html#category=${encodeURIComponent('免疫')}`)); await expect(page.locator(`div.category[data-category="${btoa(encodeURIComponent('免疫'))}"]`)).toHaveClass(/active/); + await page.goto('./blog/simple-blog/_site/posts/post-with-code/'); + await page.getByRole('link', { name: "apos'trophe" }).click(); + await expect(page).toHaveURL(getUrl(`blog/simple-blog/_site/index.html#category=${encodeURIComponent("apos'trophe")}`)); + await expect(page.locator(`div.category[data-category="${btoa(encodeURIComponent("apos'trophe"))}"]`)).toHaveClass(/active/); await page.goto('./blog/simple-blog/_site/posts/welcome/#img-lst'); await page.locator('div').filter({ hasText: /^news$/ }).click(); await expect(page).toHaveURL(/_site\/index\.html#category=news$/); From 2c4bcda9aff76806bb819c6a58e11f67c5b1e086 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 13 Nov 2024 23:54:37 +0100 Subject: [PATCH 3/5] listing - Refactor test for easier reading --- .../playwright/tests/blog-simple-blog.spec.ts | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/tests/integration/playwright/tests/blog-simple-blog.spec.ts b/tests/integration/playwright/tests/blog-simple-blog.spec.ts index 7d822be2ee3..3494ecd44c2 100644 --- a/tests/integration/playwright/tests/blog-simple-blog.spec.ts +++ b/tests/integration/playwright/tests/blog-simple-blog.spec.ts @@ -32,25 +32,23 @@ test('Categories link are clickable', async ({ page }) => { await expect(page.locator(`div.category[data-category="${btoa('news')}"]`)).toHaveClass(/active/); }); -test('Categories link with special chars are clickable', async ({ page }) => { +test('Categories links are clickable', async ({ page }) => { + const checkCategoryLink = async (category: string) => { + await page.getByRole('link', { name: category }).click(); + await expect(page).toHaveURL(getUrl(`blog/simple-blog/_site/index.html#category=${encodeURIComponent(category)}`)); + await expect(page.locator(`div.category[data-category="${btoa(encodeURIComponent(category))}"]`)).toHaveClass(/active/); + }; + // Checking link is working await page.goto('./blog/simple-blog/_site/posts/welcome/'); - await page.getByRole('link', { name: 'news' }).click(); - await expect(page).toHaveURL(/_site\/index\.html#category=news$/); - await expect(page.locator(`div.category[data-category="${btoa('news')}"]`)).toHaveClass(/active/); - await page.getByRole('link', { name: 'Welcome To My Blog' }).click(); - await page.getByRole('link', { name: 'euros (€)' }).click(); - await expect(page).toHaveURL(getUrl(`blog/simple-blog/_site/index.html#category=${encodeURIComponent('euros (€)')}`)); - await expect(page.locator(`div.category[data-category="${btoa(encodeURIComponent('euros (€)'))}"]`)).toHaveClass(/active/); - await page.getByRole('link', { name: 'Welcome To My Blog' }).click(); - await page.getByRole('link', { name: '免疫' }).click(); - await expect(page).toHaveURL(getUrl(`blog/simple-blog/_site/index.html#category=${encodeURIComponent('免疫')}`)); - await expect(page.locator(`div.category[data-category="${btoa(encodeURIComponent('免疫'))}"]`)).toHaveClass(/active/); - await page.goto('./blog/simple-blog/_site/posts/post-with-code/'); - await page.getByRole('link', { name: "apos'trophe" }).click(); - await expect(page).toHaveURL(getUrl(`blog/simple-blog/_site/index.html#category=${encodeURIComponent("apos'trophe")}`)); - await expect(page.locator(`div.category[data-category="${btoa(encodeURIComponent("apos'trophe"))}"]`)).toHaveClass(/active/); + await checkCategoryLink('news'); + // Including for special characters + await page.getByRole('link', { name: 'Welcome To My Blog' }).click(); + await checkCategoryLink('euros (€)'); + await page.getByRole('link', { name: 'Welcome To My Blog' }).click(); + await checkCategoryLink('免疫'); + await page.goto('./blog/simple-blog/_site/posts/post-with-code/'); + await checkCategoryLink("apos'trophe"); + // special check for when a page is not loaded from non root path await page.goto('./blog/simple-blog/_site/posts/welcome/#img-lst'); - await page.locator('div').filter({ hasText: /^news$/ }).click(); - await expect(page).toHaveURL(/_site\/index\.html#category=news$/); - await expect(page.locator(`div.category[data-category="${btoa('news')}"]`)).toHaveClass(/active/); -}); \ No newline at end of file + await checkCategoryLink('news'); +}); From c77c44c03dbb8dcdfb4e6e76a60fad99d85dd1df Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 13 Nov 2024 23:57:15 +0100 Subject: [PATCH 4/5] test - add gitignore to avoid having _site when running test --- tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore diff --git a/tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore b/tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore new file mode 100644 index 00000000000..92d902f2733 --- /dev/null +++ b/tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore @@ -0,0 +1,2 @@ +/.quarto/ +_site/ \ No newline at end of file From c4f393b605709200a77905e5c02c590235fb19f2 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Thu, 14 Nov 2024 17:30:34 +0100 Subject: [PATCH 5/5] lowercase for start of function --- src/core/base64.ts | 2 +- src/project/types/website/listing/website-listing-template.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/base64.ts b/src/core/base64.ts index b6f03869f32..8eed9dd7783 100644 --- a/src/core/base64.ts +++ b/src/core/base64.ts @@ -2,6 +2,6 @@ export function b64EncodeUnicode(str: string) { return btoa(encodeURIComponent(str)); } -export function UnicodeDecodeB64(str: string) { +export function unicodeDecodeB64(str: string) { return decodeURIComponent(atob(str)); } diff --git a/src/project/types/website/listing/website-listing-template.ts b/src/project/types/website/listing/website-listing-template.ts index 3527eeccba2..335deeaaa06 100644 --- a/src/project/types/website/listing/website-listing-template.ts +++ b/src/project/types/website/listing/website-listing-template.ts @@ -44,7 +44,7 @@ import { formatDate, parsePandocDate } from "../../../../core/date.ts"; import { truncateText } from "../../../../core/text.ts"; import { encodeAttributeValue } from "../../../../core/html.ts"; import { imagePlaceholder, isPlaceHolder } from "./website-listing-read.ts"; -import { b64EncodeUnicode, UnicodeDecodeB64 } from "../../../../core/base64.ts"; +import { b64EncodeUnicode, unicodeDecodeB64 } from "../../../../core/base64.ts"; export const kDateFormat = "date-format"; @@ -164,7 +164,7 @@ export function templateMarkdownHandler( // some custom utils function ejsParams["utils"] = { b64encode: b64EncodeUnicode, - b64decode: UnicodeDecodeB64, + b64decode: unicodeDecodeB64, }; return ejsParams;