diff --git a/core/font.lua b/core/font.lua
index 1b673015c..21cb2caee 100644
--- a/core/font.lua
+++ b/core/font.lua
@@ -48,9 +48,8 @@ SILE.registerCommand("font", function (options, content)
          -- end
          options.language = newlang
       end
-      SILE.settings:set("document.language", options.language)
-      fluent:set_locale(options.language)
       SILE.languageSupport.loadLanguage(options.language)
+      SILE.settings:set("document.language", options.language)
    end
    if options.script then
       SILE.settings:set("font.script", options.script)
diff --git a/core/languages.lua b/core/languages.lua
index b5c0f59a3..c2b262176 100644
--- a/core/languages.lua
+++ b/core/languages.lua
@@ -10,17 +10,17 @@ loadkit.register("ftl", function (file)
    return assert(fluent:add_messages(contents))
 end)
 
-local loadonce = {}
+SILE.scratch.loaded_languages = {}
 
 SILE.languageSupport = {
    languages = {},
    loadLanguage = function (language)
       language = language or SILE.settings:get("document.language")
       language = cldr.locales[language] and language or "und"
-      if loadonce[language] then
+      if SILE.scratch.loaded_languages[language] then
          return
       end
-      loadonce[language] = true
+      SILE.scratch.loaded_languages[language] = true
       local langresource = string.format("languages.%s", language)
       local gotlang, lang = pcall(require, langresource)
       if not gotlang then
@@ -33,6 +33,10 @@ SILE.languageSupport = {
       end
       local ftlresource = string.format("i18n.%s", language)
       SU.debug("fluent", "Loading FTL resource", ftlresource, "into locale", language)
+      -- This needs to be set so that we load localizations into the right bundle,
+      -- but this breaks the sync enabled by the hook in the document.language
+      -- setting, so we want to set it back when we're done.
+      local original_language = fluent:get_locale()
       fluent:set_locale(language)
       local gotftl, ftl = pcall(require, ftlresource)
       if not gotftl then
@@ -46,6 +50,7 @@ SILE.languageSupport = {
       if type(lang) == "table" and lang.init then
          lang.init()
       end
+      fluent:set_locale(original_language)
    end,
 }
 
@@ -65,10 +70,11 @@ end, nil, nil, true)
 SILE.registerCommand("fluent", function (options, content)
    local key = content[1]
    local locale = options.locale or SILE.settings:get("document.language")
+   local original_locale = fluent:get_locale()
+   fluent:set_locale(locale)
    SU.debug("fluent", "Looking for", key, "in", locale)
    local entry
    if key then
-      fluent:set_locale(locale)
       entry = fluent:get_message(key)
    else
       SU.warn("Fluent localization function called without passing a valid message id")
@@ -83,12 +89,13 @@ SILE.registerCommand("fluent", function (options, content)
       if entry then
          message = entry:format(options)
       end
-      fluent:set_locale(locale)
    end
+   fluent:set_locale(original_locale)
    SILE.processString(("<sile>%s</sile>"):format(message), "xml")
 end, nil, nil, true)
 
 SILE.registerCommand("ftl", function (options, content)
+   local original_locale = fluent:get_locale()
    local locale = options.locale or SILE.settings:get("document.language")
    SU.debug("fluent", "Loading message(s) into locale", locale)
    fluent:set_locale(locale)
@@ -98,6 +105,7 @@ SILE.registerCommand("ftl", function (options, content)
       local input = content[1]
       fluent:add_messages(input, locale)
    end
+   fluent:set_locale(original_locale)
 end, nil, nil, true)
 
 require("languages.unicode")
diff --git a/core/settings.lua b/core/settings.lua
index 0d1acea0c..c5e8a45e0 100644
--- a/core/settings.lua
+++ b/core/settings.lua
@@ -19,6 +19,16 @@ function settings:_init ()
       parameter = "document.language",
       type = "string",
       default = "en",
+      hook = function (language)
+         if SILE.scratch.loaded_languages and not SILE.scratch.loaded_languages[language] then
+            SU.warn(([[Setting document.language to '%s', but support for '%s' has not been loaded!
+
+  Consider invoking \language[main=%s] which loads language support before
+  setting it or manually calling SILE.languageSupport.loadLanguage("%s").
+            ]]):format(language, language, language, language))
+         end
+         fluent:set_locale(language)
+      end,
       help = "Locale for localized language support",
    })
 
diff --git a/spec/hyphenator_spec.lua b/spec/hyphenator_spec.lua
index e41b37297..59bc34afb 100644
--- a/spec/hyphenator_spec.lua
+++ b/spec/hyphenator_spec.lua
@@ -6,9 +6,9 @@ SILE.init()
 describe("Hyphenation module", function ()
    local hyphenate = SILE.showHyphenationPoints
 
+   SILE.settings:set("document.language", "fr")
+
    describe("minWord with UTF8 in input text", function ()
-      SILE.languageSupport.loadLanguage("fr")
-      fluent:set_locale("fr")
       -- Trigger the initialization of the hyphenator
       -- so SILE._hyphenators["fr"] is created
       hyphenate("série", "fr")
@@ -33,10 +33,7 @@ describe("Hyphenation module", function ()
    end)
 
    describe("exceptions with UTF8 in input text", function ()
-      SILE.languageSupport.loadLanguage("fr")
-      fluent:set_locale("fr")
-
-      SILE.call("hyphenator:add-exceptions", { lang = "fr" }, { "légè-rement" })
+      SILE.call("hyphenator:add-exceptions", {}, { "légè-rement" })
 
       it("should hyphenate with exception rule", function ()
          assert.is.equal("légè-rement", hyphenate("légèrement", "fr"))
diff --git a/spec/languages_spec.lua b/spec/languages_spec.lua
index 1568e4f40..6d6e7016a 100644
--- a/spec/languages_spec.lua
+++ b/spec/languages_spec.lua
@@ -4,8 +4,7 @@ describe("Language module", function ()
    describe("Norwegian", function ()
       local hyphenate = SILE.showHyphenationPoints
 
-      SILE.languageSupport.loadLanguage("no")
-      fluent:set_locale("no")
+      SILE.settings:set("document.language", "no")
 
       it("should hyphenate", function ()
          assert.is.equal("Nor-we-gian", hyphenate("Norwegian", "no"))
@@ -13,14 +12,12 @@ describe("Language module", function ()
       end)
 
       it("should have localizations", function ()
-         fluent:set_locale("no")
          local hello = fluent:get_message("hello")({ name = "Busted" })
          assert.is.equal("Hei <em>Busted</em>!", hello)
       end)
 
       describe("Norwegian Bokmål", function ()
-         SILE.languageSupport.loadLanguage("nb")
-         fluent:set_locale("nb")
+         SILE.settings:set("document.language", "nb")
 
          it("should hyphenate", function ()
             assert.is.equal("Nor-we-gian", hyphenate("Norwegian", "nb"))
@@ -34,8 +31,7 @@ describe("Language module", function ()
       end)
 
       describe("Norwegian Nynorsk", function ()
-         SILE.languageSupport.loadLanguage("nn")
-         fluent:set_locale("nn")
+         SILE.settings:set("document.language", "nn")
 
          it("should hyphenate", function ()
             assert.is.equal("Nor-we-gian", hyphenate("Norwegian", "nn"))
diff --git a/tests/bug-2054.bib b/tests/bug-2054.bib
new file mode 100644
index 000000000..27b494cac
--- /dev/null
+++ b/tests/bug-2054.bib
@@ -0,0 +1,10 @@
+@phdthesis{foo,
+  author  = {Doe, John and Smith, Jane and Bob and Alice},
+  title   = {A Tale of Four Authors},
+}
+
+@book{bar,
+  author     = {Doe, John and Smith, Jane},
+  translator = {Pumpernickel, Peter},
+  title      = {A Book},
+}
diff --git a/tests/bug-2054.expected b/tests/bug-2054.expected
new file mode 100644
index 000000000..f77f64d7f
--- /dev/null
+++ b/tests/bug-2054.expected
@@ -0,0 +1,114 @@
+Set paper size 	297.6377985	419.5275636
+Begin page
+Mx 	14.8819
+My 	28.5447
+Set font 	Gentium Plus;10;400;;normal;;;LTR
+T	39 82 72 w=15.5762	(Doe)
+Mx 	30.4581
+T	15 w=2.2900	(,)
+Mx 	34.9513
+T	45 w=3.0518	(J)
+Mx 	38.0031
+T	17 w=2.2900	(.)
+Mx 	42.4963
+T	68 81 71 w=15.3076	(and)
+Mx 	60.0072
+T	54 80 76 87 75 w=24.4629	(Smith)
+Mx 	84.4701
+T	15 w=2.2900	(,)
+Mx 	88.9633
+T	45 w=3.0518	(J)
+Mx 	92.0151
+T	17 w=2.2900	(.)
+Mx 	94.3051
+T	15 w=2.2900	(,)
+Mx 	98.7984
+Set font 	Gentium Plus;10;400;Italic;normal;;;LTR
+T	36 w=5.6006	(A)
+Mx 	106.4019
+T	37 82 82 78 w=17.7783	(Book)
+Mx 	124.1802
+Set font 	Gentium Plus;10;400;;normal;;;LTR
+T	17 w=2.2900	(.)
+Mx 	128.6735
+T	55 85 68 81 86 79 68 87 72 71 w=43.9990	(Translated)
+Mx 	174.8758
+T	69 92 w=10.0098	(by)
+Mx 	187.0887
+T	51 88 80 83 72 85 81 76 70 78 72 79 w=57.2412	(Pumpernickel)
+Mx 	244.3300
+T	15 w=2.2900	(,)
+Mx 	248.8232
+T	51 72 87 72 85 w=21.8701	(Peter)
+Mx 	270.6933
+T	17 w=2.2900	(.)
+Mx 	14.8819
+My 	40.5447
+T	39 82 72 w=15.5762	(Doe)
+Mx 	32.6737
+T	72 87 w=8.0615	(et)
+Mx 	42.9508
+T	68 79 w=7.2998	(al)
+Mx 	50.2506
+T	17 w=2.2900	(.)
+Mx 	52.5406
+T	15 w=2.2900	(,)
+Mx 	57.0463
+T	180 w=4.3604	(“)
+Mx 	61.4066
+T	36 w=5.9521	(A)
+Mx 	69.5744
+T	55 68 79 72 w=17.4268	(Tale)
+Mx 	89.2167
+T	82 73 w=8.1396	(of)
+Mx 	99.5720
+T	41 82 88 85 w=19.0674	(Four)
+Mx 	120.8550
+T	36 88 87 75 82 85 86 w=33.0615	(Authors)
+Mx 	153.9165
+T	17 w=2.2900	(.)
+Mx 	156.2065
+T	181 w=4.3604	(”)
+Mx 	14.8819
+My 	52.5447
+T	39 82 72 w=15.5762	(Doe)
+Mx 	30.4581
+T	15 w=2.2900	(,)
+Mx 	34.9513
+T	45 w=3.0518	(J)
+Mx 	38.0031
+T	17 w=2.2900	(.)
+Mx 	42.4963
+T	68 81 71 w=15.3076	(and)
+Mx 	60.0072
+T	54 80 76 87 75 w=24.4629	(Smith)
+Mx 	84.4701
+T	15 w=2.2900	(,)
+Mx 	88.9633
+T	45 w=3.0518	(J)
+Mx 	92.0151
+T	17 w=2.2900	(.)
+Mx 	94.3051
+T	15 w=2.2900	(,)
+Mx 	98.7984
+Set font 	Gentium Plus;10;400;Italic;normal;;;LTR
+T	36 w=5.6006	(A)
+Mx 	106.4019
+T	37 82 82 78 w=17.7783	(Book)
+Mx 	124.1802
+Set font 	Gentium Plus;10;400;;normal;;;LTR
+T	17 w=2.2900	(.)
+Mx 	128.6735
+T	55 85 68 81 86 79 68 87 72 71 w=43.9990	(Translated)
+Mx 	174.8758
+T	69 92 w=10.0098	(by)
+Mx 	187.0887
+T	51 88 80 83 72 85 81 76 70 78 72 79 w=57.2412	(Pumpernickel)
+Mx 	244.3300
+T	15 w=2.2900	(,)
+Mx 	248.8232
+T	51 72 87 72 85 w=21.8701	(Peter)
+Mx 	270.6933
+T	17 w=2.2900	(.)
+End page
+Finish
diff --git a/tests/bug-2054.sil b/tests/bug-2054.sil
new file mode 100644
index 000000000..d8b381879
--- /dev/null
+++ b/tests/bug-2054.sil
@@ -0,0 +1,15 @@
+\begin[papersize=a6]{document}
+\nofolios
+\neverindent
+\language[main=en]
+\use[module=packages.bibtex]
+\loadbibliography[file=tests/bug-2054.bib]
+
+\reference{bar}
+
+\reference{foo}
+
+% Should not use Latin localization, should match first use of reference
+\reference{bar}
+
+\end{document}