feat(packages): Keep track of cited bibliography entries

This should even be the default when generating a bibliography.

Author: Omikhleia

csl/core/engine.lua

@@ -13,6 +13,7 @@
 --
 -- THINGS NOT DONE
 --  - disambiguation logic (not done at all)
+--  - collapse logic in citations (not done at all)
 --  - other FIXME/TODOs in the code on specific features
 --
 -- luacheck: no unused args
@@ -875,7 +876,7 @@ function CslEngine:_names (options, content, entry)
    local name_delimiter = name_node.options.delimiter or inherited_opts["names-delimiter"]
    -- local delimiter_precedes_et_al = name_node.options["delimiter-precedes-et-al"] -- TODO NOT IMPLEMENTED
 
-   if not self.cache[name_delimiter] then
+   if name_delimiter and not self.cache[name_delimiter] then
       name_delimiter = self:_xmlEscape(name_delimiter)
       self.cache[name_delimiter] = name_delimiter
    end
@@ -885,7 +886,7 @@ function CslEngine:_names (options, content, entry)
       et_al_min = et_al_min,
       et_al_use_first = et_al_use_first,
       and_word = and_word,
-      name_delimiter = self.cache[name_delimiter],
+      name_delimiter = name_delimiter and self.cache[name_delimiter],
       is_label_first = is_label_first,
       label_opts = label_opts,
       et_al_opts = et_al_opts,
@@ -1083,9 +1084,10 @@ function CslEngine:_key (options, content, entry)
 end
 
 -- FIXME: A bit ugly: When implementing SU.collatedSort, I didn't consider
--- sorting structured tables, so I need to go low level here.
+-- sorting structured tables, so we need to go low level here.
 -- Moreover, I made icu.compare return a boolean, so we have to pay twice
 -- the comparison cost to check equality...
+-- See PR #2105
 local icu = require("justenoughicu")
 
 function CslEngine:_sort (options, content, entries)
@@ -1116,14 +1118,32 @@ function CslEngine:_sort (options, content, entries)
    local lang = self.locale.lang
    local collator = icu.collation_create(lang, {})
    table.sort(entries, function (a, b)
+      if a["citation-key"] == b["citation-key"] then
+         -- Lua can invoke the comparison function with the same entry.
+         -- Really! Due to the way it handles it pivot on partitioning.
+         -- Shortcut the inner keys comparison in that case.
+         return false
+      end
+      -- NOT IMPLEMENTED (not bothering for now):
+      -- "Items with an empty sort key value are placed at the end of the sort,
+      -- both for ascending and descending sorts."
       local ak = a._keys
       local bk = b._keys
       for i = 1, math.min(#ak, #bk) do
-         if ak[i] ~= bk[i] then -- See comment, ugly inequality check)
-            return icu.compare(collator, ak[i], bk[i])
+         if ak[i] ~= bk[i] then -- HACK: See comment above, ugly inequality check
+            local cmp = icu.compare(collator, ak[i], bk[i])
+            if type(cmp) == "number" then
+               return cmp < 0 -- To keep on working whenever PR #2105 lands
+            end
+            return cmp
          end
       end
-      return false
+      -- If we reach this point, the keys are equal.
+      -- Probably unlikely in real life, and not mentioned in the CSL spec
+      -- unless I missed it. Let's fallback to the citation order, so at
+      -- least cited entries are ordered predictably.
+      SU.warn("CSL sort keys are equal for " .. a["citation-key"] .. " and " .. b["citation-key"])
+      return a["citation-number"] < b["citation-number"]
    end)
    icu.collation_destroy(collator)
 end
@@ -1212,6 +1232,28 @@ function CslEngine:_process (entries, mode)
       self.sorting = true
       self:_sort(sort.options, sort, entries)
       self.sorting = false
+   else
+      -- The CSL specification says:
+      -- "In the absence of cs:sort, cites and bibliographic entries appear in
+      -- the order in which they are cited."
+      -- The wording is ambiguous!
+      -- We tracked the first citation number in 'citation-number', so for
+      -- the bibliography, using it makes sense.
+      -- For citations, we use the exact order of the input. Consider a cite
+      -- (work1, work2) and a subsequent cite (work2, work1). The order of
+      -- the bibliography should be (work1, work2), but the order of the cites
+      -- should be (work1, work2) and (work2, work1) respectively.
+      -- It seeems to be the case: Some styles (ex. American Chemical Society)
+      -- have an explicit sort by 'citation-number' in the citations section,
+      -- which would be useless if that order was impplied.
+      if mode == "bibliography" then
+         table.sort(entries, function (e1, e2)
+            if not e1["citation-number"] or not e2["citation-number"] then
+               return false -- Safeguard?
+            end
+            return e1["citation-number"] < e2["citation-number"]
+         end)
+      end
    end
 
    local res = self:_render_children(ast, entries)

packages/bibtex/init.lua

@@ -195,7 +195,7 @@ local function parseBibtex (fn, biblio)
    for i = 1, #t do
       if t[i].id == "entry" then
          local ent = t[i][1]
-         local entry = { type = ent.type, attributes = ent[1] }
+         local entry = { type = ent.type, label = ent.label, attributes = ent[1] }
          if biblio[ent.label] then
             SU.warn("Duplicate entry key '" .. ent.label .. "', picking the last one")
          end
@@ -301,7 +301,7 @@ end
 
 function package:_init ()
    base._init(self)
-   SILE.scratch.bibtex = { bib = {} }
+   SILE.scratch.bibtex = { bib = {}, cited = { keys = {}, citnums = {} } }
    Bibliography = require("packages.bibtex.bibliography")
    -- For DOI, PMID, PMCID and URL support.
    self:loadPackage("url")
@@ -457,8 +457,6 @@ function package:registerCommands ()
       -- - multiple citation keys
       if not SILE.scratch.bibtex.engine then
          SILE.call("bibliographystyle", { lang = "en-US", style = "chicago-author-date" })
-         -- SILE.call("bibliographystyle", { lang = "en-US", style = "chicago-fullnote-bibliography" })
-         -- SILE.call("bibliographystyle", { lang = "en-US", style = "apa" })
       end
       local engine = SILE.scratch.bibtex.engine
       if not options.key then
@@ -469,7 +467,12 @@ function package:registerCommands ()
          return
       end
 
-      local csljson = bib2csl(entry)
+      -- Keep track of cited entries
+      table.insert(SILE.scratch.bibtex.cited.keys, options.key)
+      local citnum = #SILE.scratch.bibtex.cited.keys
+      SILE.scratch.bibtex.cited.citnums[options.key] = citnum
+
+      local csljson = bib2csl(entry, citnum)
       -- csljson.locator = { -- EXPERIMENTAL
       --    label = "page",
       --    value = "123-125"
@@ -482,8 +485,6 @@ function package:registerCommands ()
    self:registerCommand("csl:reference", function (options, content)
       if not SILE.scratch.bibtex.engine then
          SILE.call("bibliographystyle", { lang = "en-US", style = "chicago-author-date" })
-         -- SILE.call("bibliographystyle", { lang = "en-US", style = "chicago-fullnote-bibliography" })
-         -- SILE.call("bibliographystyle", { lang = "en-US", style = "apa" })
       end
       local engine = SILE.scratch.bibtex.engine
       if not options.key then
@@ -494,34 +495,65 @@ function package:registerCommands ()
          return
       end
 
-      local cslentry = bib2csl(entry)
+      local citnum = SILE.scratch.bibtex.cited.citnums[options.key]
+      if not citnum then
+         SU.warn("Reference to a non-cited entry " .. options.key)
+         -- Make it cited
+         table.insert(SILE.scratch.bibtex.cited.keys, options.key)
+         citnum = #SILE.scratch.bibtex.cited.keys
+         SILE.scratch.bibtex.cited.citnums[options.key] = citnum
+      end
+      local cslentry = bib2csl(entry, citnum)
       local cite = engine:reference(cslentry)
 
       SILE.processString(("<sile>%s</sile>"):format(cite), "xml")
    end)
 
-   self:registerCommand("printbibliography", function (_, _)
+   self:registerCommand("printbibliography", function (options, _)
       if not SILE.scratch.bibtex.engine then
          SILE.call("bibliographystyle", { lang = "en-US", style = "chicago-author-date" })
-         -- SILE.call("bibliographystyle", { lang = "en-US", style = "chicago-fullnote-bibliography" })
-         -- SILE.call("bibliographystyle", { lang = "en-US", style = "apa" })
       end
       local engine = SILE.scratch.bibtex.engine
 
-      local bib = SILE.scratch.bibtex.bib
+      local bib
+      if SU.boolean(options.cited, true) then
+         bib = {}
+         for _, key in ipairs(SILE.scratch.bibtex.cited.keys) do
+            bib[key] = SILE.scratch.bibtex.bib[key]
+         end
+      else
+         bib = SILE.scratch.bibtex.bib
+      end
+
       local entries = {}
-      for _, entry in pairs(bib) do
+      local ncites = #SILE.scratch.bibtex.cited.keys
+      for key, entry in pairs(bib) do
          if entry.type ~= "xdata" then
             crossrefAndXDataResolve(bib, entry)
             if entry then
-               local cslentry = bib2csl(entry)
+               local citnum = SILE.scratch.bibtex.cited.citnums[key]
+               if not citnum then
+                  -- This is just to make happy CSL styles that require a citation number
+                  -- However, table order is not guaranteed in Lua so the output may be
+                  -- inconsistent across runs with styles that use this number for sorting.
+                  -- This may only happen for non-cited entries in the bibliography, and it
+                  -- would be a bad practive to use such a style to print the full bibliography,
+                  -- so I don't see a strong need to fix this at the expense of performance.
+                  -- (and we can't really, some styles might have several sorting criteria
+                  -- leading to impredictable order anyway).
+                  ncites = ncites + 1
+                  citnum = ncites
+               end
+               local cslentry = bib2csl(entry, citnum)
                table.insert(entries, cslentry)
             end
          end
       end
       print("<bibliography: " .. #entries .. " entries>")
       local cite = engine:reference(entries)
       SILE.processString(("<sile>%s</sile>"):format(cite), "xml")
+
+      SILE.scratch.bibtex.cited = { keys = {}, citnums = {} }
    end)
 end
 
@@ -531,7 +563,6 @@ BibTeX is a citation management system.
 It was originally designed for TeX but has since been integrated into a variety of situations.
 
 This experimental package allows SILE to read and process BibTeX \code{.bib} files and output citations and full text references.
-(It doesn’t currently produce full bibliography listings.)
 
 To load a BibTeX file, issue the command \autodoc:command{\loadbibliography[file=<whatever.bib>]}
 
@@ -544,9 +575,9 @@ To load a BibTeX file, issue the command \autodoc:command{\loadbibliography[file
 To produce an inline citation, call \autodoc:command{\cite{<key>}}, which will typeset something like “Jones 1982”.
 If you want to cite a particular page number, use \autodoc:command{\cite[page=22]{<key>}}.
 
-To produce a full reference, use \autodoc:command{\reference{<key>}}.
+To produce a bibliographic reference, use \autodoc:command{\reference{<key>}}.
 
-Currently, the only supported bibliography style is Chicago referencing.
+This implementation doesn’t currently produce full bibliography listings, and the only supported bibliography style is Chicago referencing.
 
 \smallskip
 \noindent
@@ -556,7 +587,7 @@ Currently, the only supported bibliography style is Chicago referencing.
 \indent
 While an experimental work-in-progress, the CSL (Citation Style Language) implementation is more powerful and flexible than the legacy commands.
 
-You must first invoke \autodoc:command{\bibliographystyle[style=<style>, lang=<lang>]}, where \autodoc:parameter{style} is the name of the CSL style file (without the \code{.csl} extension), and \autodoc:parameter{lang} is the language code of the CSL locale to use (e.g., \code{en-US}).
+You should first invoke \autodoc:command{\bibliographystyle[style=<style>, lang=<lang>]}, where \autodoc:parameter{style} is the name of the CSL style file (without the \code{.csl} extension), and \autodoc:parameter{lang} is the language code of the CSL locale to use (e.g., \code{en-US}).
 
 The command accepts a few additional options:
 
@@ -572,11 +603,13 @@ If you don’t specify a style or locale, the author-date style and the \code{en
 
 To produce an inline citation, call \autodoc:command{\csl:cite{<key>}}, which will typeset something like “(Jones 1982)”.
 
-To produce a full reference, use \autodoc:command{\csl:reference{<key>}}.
+To produce a bibliography of cited references, use \autodoc:command{\printbibliography}.
+After printing the bibliography, the list of cited entries will be cleared. This allows you to start fresh for subsequent uses (e.g., in a different chapter).
+If you want to include all entries in the bibliography, not just those that have been cited, set the option \autodoc:parameter{cited} to false.
 
-To produce a complete bibliography, use \autodoc:command{\printbibliography}.
-As of yet, this command is for testing purposes only.
-It does not handle filtering of the bibliography.
+To produce a bibliographic reference, use \autodoc:command{\csl:reference{<key>}}.
+Note that this command is not intended for actual use, but for testing purposes.
+It may be removed in the future.
 
 \smallskip
 \noindent

packages/bibtex/support/bib2csl.lua

@@ -78,8 +78,9 @@ end
 
 --- Convert a BibTeX entry to a CSL item.
 -- @tparam table entry The BibTeX entry
+-- @tparam number citnum The citation number (computed, not from BibTeX but from actual citation order)
 -- @treturn table The CSL item
-local function bib2csl (entry)
+local function bib2csl (entry, citnum)
    local csl = {}
    local bibtex = entry.attributes
    local bibtype = entry.type:lower()
@@ -88,6 +89,17 @@ local function bib2csl (entry)
    local t = BIBTEX2CSL_TYPES[bibtype] or "document"
    csl.type = t
 
+   -- Citation key may be wanted by some styles
+   csl["citation-key"] = entry.label
+   -- Citation number is used by some styles such as ACS
+   csl["citation-number"] = citnum
+
+   -- BibLaTeX label / shorthand
+   -- The label "provides a substitute for any missing data"
+   -- it relates to shorthand, which overrides the label.
+   -- Some CSL styles such as USITC use citation-label in priority over author, etc.
+   csl["citation-label"] = bibtex.shorthand or bibtex.label
+
    -- BibTeX address / BibLaTeX location
    if bibtex.location then
       csl["event-place"] = bibtex.location
@@ -105,7 +117,12 @@ local function bib2csl (entry)
 
    -- BibTex editor
    csl.editor = bibtex.editor
-   csl["collection-editor"] = bibtex.editor
+   -- N.B. BibLaTeX does not have a "collection-editor".
+   -- Using some editora and editoratype hint is sometimes mentioned on forums
+   -- but it's ad hoc and not part of any 'standard' recommendation.
+   -- Biber would allow to define extra fields (e.g. serieseditor) but the issue
+   -- is the same: lack of standardization.
+   -- csl["collection-editor"] = bibtex.editoratype == "serieseditor" and bibtex.editora
 
    -- BibLaTeX date / BibTeX year and month
    local date = bibtex.date and bibtex.date or toDate(bibtex.year, bibtex.month)