fixed security issues; added pdftk option; added development site is …

…protected Configuration directive; altered error message display so that end users will not see certain types of error messages, but they will be displayed in the log

CHANGELOG.md

@@ -1,5 +1,32 @@
 # Change Log
 
+## [1.4.97] - 2024-02-29
+
+### Added
+- The `pdftk` option under `attachment` and `features` for filling in
+  `pdf template file` attachments using pdftk instead of pikepdf.
+### Changed
+- During the Docker image build process, `pandoc` will run once, so
+  that the first user to assemble a document with `pandoc` will not
+  experience slowness due to LaTeX needing to generate files.
+- Appearance streams will be generated when using `pdf template file`.
+- Error messages related to problems in the source code will no longer
+  be displayed to the user unless the user is an administrator or
+  developer. If you want these error messages to appear to all users,
+  set `debug: True` and `development site is protected: True` in the
+  Configuration. The error messages will be available in
+  `docassemble.log`.
+### Fixed
+- Fixed security issue identified by Riyush Ghimire, affecting
+  versions 1.4.53 to 1.4.96, that could cause contents of files in the
+  filesystem to be revealed. This is a high severity issue and
+  upgrading as soon as possible is recommended.
+- Fixed security issue identified by Riyush Ghimire, affecting
+  versions up to 1.4.96, that allowed an open redirect URL to be formed.
+- Fixed security issue identified by Riyush Ghimire, affecting
+  versions up to 1.4.96, that would allow HTML or JavaScript
+  injection.
+
 ## [1.4.96] - 2024-02-14
 
 ### Fixed

Dockerfile

@@ -122,7 +122,13 @@ RUN bash -c \
 && python /tmp/docassemble/Docker/nltkdownload.py \
 && cd /var/www/nltk_data/corpora \
 && unzip -o wordnet.zip \
-&& unzip -o omw-1.4.zip"
+&& unzip -o omw-1.4.zip \
+&& cd /tmp \
+&& mkdir -p /tmp/conv \
+&& pandoc --pdf-engine=lualatex -M latextmpdir=./conv -M pdfa=false /usr/share/docassemble/local3.10/lib/python3.10/site-packages/docassemble/base/data/templates/Legal-Template.yml --template=/usr/share/docassemble/local3.10/lib/python3.10/site-packages/docassemble/base/data/templates/Legal-Template.tex --from=markdown+raw_tex-latex_macros -s -o /tmp/temp.pdf /usr/share/docassemble/local3.10/lib/python3.10/site-packages/docassemble/base/data/templates/hello.md \
+&& rm /tmp/temp.pdf \
+&& pandoc --pdf-engine=lualatex -M latextmpdir=./conv -M pdfa=false --template=/usr/share/docassemble/local3.10/lib/python3.10/site-packages/docassemble/base/data/templates/Legal-Template.rtf -s -o /tmp/temp.rtf /usr/share/docassemble/local3.10/lib/python3.10/site-packages/docassemble/base/data/templates/hello.md \
+&& rm /tmp/temp.rtf"
 
 USER root
 RUN rm -rf /tmp/docassemble

docassemble_base/docassemble/base/data/sources/base-words.yml

@@ -311,6 +311,7 @@
 "Falkland Islands (Malvinas)": Null
 "false": Null
 "Faroe Islands": Null
+"Field cannot contain HTML": Null
 "Fiji": Null
 "File could not be converted: ": Null
 "File deleted.": Null
@@ -1031,6 +1032,7 @@
 "There is already a username and password on this system with the e-mail address": Null
 "The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.": Null
 "There was an error.": Null
+"There was an error. Please contact the system administrator.": Null
 "There was an error updating the packages.": Null
 "There was an error with the synchronization.": Null
 "There was a problem connecting to GitHub. Please check your GitHub configuration and try again.": Null

docassemble_base/docassemble/base/error.py

@@ -27,6 +27,9 @@ def __str__(self):
         return str(self.value)
 
 
+class DASourceError(DAError):
+    pass
+
 class DANotFoundError(Exception):
     pass
 

docassemble_base/docassemble/base/pandoc.py

@@ -10,16 +10,16 @@
 import random
 import mimetypes
 import urllib.request
+import convertapi
+import requests
+from pikepdf import Pdf
 import docassemble.base.filter
 import docassemble.base.functions
 from docassemble.base.config import daconfig
 from docassemble.base.logger import logmessage
 from docassemble.base.pdfa import pdf_to_pdfa
 from docassemble.base.pdftk import pdf_encrypt
 from docassemble.base.error import DAError, DAException
-import convertapi
-import requests
-from pikepdf import Pdf
 
 style_find = re.compile(r'{\s*(\\s([1-9])[^\}]+)\\sbasedon[^\}]+heading ([0-9])', flags=re.DOTALL)
 PANDOC_PATH = daconfig.get('pandoc', 'pandoc')
@@ -802,13 +802,31 @@ def concatenate_files(path_list, pdfa=False, password=None, owner_password=None)
             new_path_list.append(path)
     if len(new_path_list) == 0:
         raise DAError("concatenate_files: no valid files to concatenate")
+
     if len(new_path_list) == 1:
         shutil.copyfile(new_path_list[0], pdf_file.name)
     else:
         with Pdf.open(new_path_list[0]) as original:
+            need_appearances = False
+            try:
+                if original.Root.AcroForm.NeedAppearances:
+                    need_appearances = True
+            except:
+                pass
             for additional_file in new_path_list[1:]:
                 with Pdf.open(additional_file) as additional_pdf:
+                    if need_appearances is False:
+                        try:
+                            if additional_pdf.Root.AcroForm.NeedAppearances:
+                                need_appearances = True
+                        except:
+                            pass
                     original.pages.extend(additional_pdf.pages)
+            if need_appearances:
+                try:
+                    original.Root.AcroForm.NeedAppearances = True
+                except:
+                    logmessage("concatenate_files: an additional file had an AcroForm with NeedAppearances but setting NeedAppearances on the final document resulted in an error")
             original.save(pdf_file.name)
     if pdfa:
         pdf_to_pdfa(pdf_file.name)