diff --git a/src/control/XournalMain.cpp b/src/control/XournalMain.cpp
index 3770faf2c1db..4c7e77129666 100644
--- a/src/control/XournalMain.cpp
+++ b/src/control/XournalMain.cpp
@@ -80,6 +80,12 @@ void initLocalisation() {
                   "xournalpp with msvc",
                   e.what());
     }
+    /**
+     * Force numbers to be printed out and parsed by C libraries (cairo) in the "classic" locale.
+     * This avoids issue with tags when exporting to PDF, see #3551
+     */
+    setlocale(LC_NUMERIC, "C");
+
     std::cout.imbue(std::locale());
 }
 
diff --git a/src/control/jobs/PdfExportJob.cpp b/src/control/jobs/PdfExportJob.cpp
index a0c9b55f113d..44f5a7dcc240 100644
--- a/src/control/jobs/PdfExportJob.cpp
+++ b/src/control/jobs/PdfExportJob.cpp
@@ -33,10 +33,9 @@ void PdfExportJob::run() {
     doc->unlock();
 
     if (!pdfe->createPdf(this->filepath, false)) {
+        this->errorMsg = pdfe->getLastError();
         if (control->getWindow()) {
             callAfterRun();
-        } else {
-            this->errorMsg = pdfe->getLastError();
         }
     }
 
diff --git a/src/pdf/base/XojCairoPdfExport.cpp b/src/pdf/base/XojCairoPdfExport.cpp
index 5355f4109d89..da6820758fae 100644
--- a/src/pdf/base/XojCairoPdfExport.cpp
+++ b/src/pdf/base/XojCairoPdfExport.cpp
@@ -38,7 +38,7 @@ auto XojCairoPdfExport::startPdf(const fs::path& file) -> bool {
     this->populatePdfOutline(tocModel);
 #endif
 
-    return true;
+    return cairo_surface_status(this->surface) == CAIRO_STATUS_SUCCESS;
 }
 
 #if CAIRO_VERSION >= CAIRO_VERSION_ENCODE(1, 16, 0)
@@ -68,6 +68,7 @@ void XojCairoPdfExport::populatePdfOutline(GtkTreeModel* tocModel) {
         auto pageDest = pdfBgPage == npos ? npos : doc->findPdfPage(pdfBgPage);  // Destination in document
         if (pageDest != npos) {
             std::ostringstream linkAttrBuf;
+            linkAttrBuf.imbue(std::locale::classic());
             linkAttrBuf << "page=" << pageDest + 1;
             if (dest->shouldChangeLeft() && dest->shouldChangeTop()) {
                 linkAttrBuf << " pos=[" << dest->getLeft() << " " << dest->getTop() << "]";
@@ -150,6 +151,9 @@ auto XojCairoPdfExport::createPdf(fs::path const& file, PageRangeVector& range,
     }
 
     if (!startPdf(file)) {
+        this->lastError = _("Failed to initialize PDF Cairo surface");
+        this->lastError += "\nCairo error: ";
+        this->lastError += cairo_status_to_string(cairo_surface_status(this->surface));
         return false;
     }
 
@@ -192,6 +196,9 @@ auto XojCairoPdfExport::createPdf(fs::path const& file, bool progressiveMode) ->
     }
 
     if (!startPdf(file)) {
+        this->lastError = _("Failed to initialize PDF Cairo surface");
+        this->lastError += "\nCairo error: ";
+        this->lastError += cairo_status_to_string(cairo_surface_status(this->surface));
         return false;
     }