moved images to new format, updated with dockerfile and new processing scripts

Author: russbiggs

.build_scripts/clean.py

@@ -1,11 +1,13 @@
 import os
 import re
-from shutil import copyfile
+import shutil
+import pathlib
 
 import frontmatter
 import toml
 
 
+
 def main():
     extensions = {}
     with open('config.toml', 'r') as f:
@@ -31,19 +33,27 @@ def main():
                 if ext != '.md':
                     copyfile(os.path.join('content', item), os.path.join('pdf-build',item))
         # move image files
-        if 'images' in os.listdir('content'):
-            for path, _, files in os.walk('content/images'):
-                for filename in files:
-                    ext = os.path.splitext(filename)[1]
-                    if ext != '.md':
-                        copyfile(os.path.join(path, filename), os.path.join('pdf-build',filename))
-        else:
-            for path, _, files in os.walk('content/pages'):
-                for filename in files:
-                    ext = os.path.splitext(filename)[1]
-                    if ext != '.md':
-                        copyfile(os.path.join(path, filename), os.path.join('pdf-build',filename))
-        images = [f for f in os.listdir('pdf-build') if re.search(r'.*\.(jpe?g|png|gif)$', f)]
+        for path, _, files in os.walk('content/pages'):
+            for filename in files:
+                ext = os.path.splitext(filename)[1]
+                if ext != '.md':
+                    copyfile(os.path.join(path, filename), os.path.join('pdf-build',filename))
+        for path, _, files in os.walk('static/'):
+            p = pathlib.Path(path)
+            cleaned_path = p.relative_to(*p.parts[:2])
+            pdf_path = pathlib.Path('pdf-build').joinpath(p.relative_to(*p.parts[:2]))
+            if not os.path.exists(pdf_path):
+                os.makedirs(pdf_path)
+            for filename in files:
+                ext = os.path.splitext(filename)[1]
+                if ext.lower() in ['.jpg','.jpeg','.png','.gif']:
+                    copy_parents(os.path.join(path, filename),'pdf-build', 1)
+            images = []
+        for path, d, f in os.walk('pdf-build'):
+            p = pathlib.Path(*pathlib.Path(path).parts[1:])
+            for file in f:
+                if re.search(r'.*\.(jpe?g|png|gif)$', file):
+                    images.append(os.path.join(p, file))
         for path, _, files in os.walk('content/pages'):
             for filename in files:
                 ext = os.path.splitext(filename)[1]
@@ -53,15 +63,15 @@ def main():
                         contents.append({'name':filename, 'content': content})
                 else:
                     continue
-        full_pdf_content = ""
+        full_pdf_content = "\n\n\pagebreak\n\n"
         content = clean_markdown("content", "_index.md", images)
         if content:
             full_pdf_content += content
             full_pdf_content += "\n\n\pagebreak\n\n"
         for item in sorted(contents, key=lambda k: k['name']):
             full_pdf_content += item['content']
             full_pdf_content += "\n\n\pagebreak\n\n"
-        with open('pdf-build/' + site_name + ".md", 'w') as f:
+        with open('pdf-build/' + site_name + ".fullsite.md", 'w') as f:
             f.write(full_pdf_content)
 
     else:
@@ -76,18 +86,28 @@ def main():
                 for filename in files:
                     ext = os.path.splitext(filename)[1]
                     if ext == '.md':
-                        content = clean_markdown(path, filename, lang_key ,default_lang)
+                        content = clean_markdown(path, filename, lang_key, default_lang)
                         if content:
                             contents.append({'name':filename, 'content': content})
                     else:
                         copyfile(os.path.join(path,filename), os.path.join('pdf-build',lang_key,filename))
-            full_pdf_content = ""
+            full_pdf_content = "\n\n\pagebreak\n\n"
             for item in sorted(contents, key=lambda k: k['name']):
                 full_pdf_content += item['content']
                 full_pdf_content += "\n\n\pagebreak\n\n"
             with open('pdf-build/' + site_name + ".md", 'w') as f:
                 f.write(full_pdf_content)
 
+def copy_parents(src, dest_folder, dir_offset=0):
+    prev_offset = 0 if dir_offset == 0 else src.replace('/', '%', dir_offset - 1).find('/') + 1
+    post_offset = src.rfind('/')
+
+    src_dirs = '' if post_offset == -1 else src[prev_offset:post_offset]
+    src_filename = src[post_offset + 1:]
+
+    os.makedirs(f'{dest_folder}/{src_dirs}', exist_ok=True)
+    shutil.copy(src, f'{dest_folder}/{src_dirs}/{src_filename}')
+
 
 def clean_markdown(path, filename, images, lang="", default_lang = "en"):
     post = frontmatter.load(os.path.join(path, filename))
@@ -97,9 +117,8 @@ def clean_markdown(path, filename, images, lang="", default_lang = "en"):
         guide['filename'] = filename
         content = post.content
         for image in images:
-            replace_regex = r'(\!\[.*\]).*(\().*\/(' + re.escape(image) + r')([A-Za-z\s\"\'\-\,\.\;\:]*)(\))'
-            print(replace_regex)
-            content = re.sub(replace_regex, r'\1\2\3\5', content)
+            replace_regex = r'(\!\[.*\]).*(\()(.*\/)(' + re.escape(str(image)) + r')([A-Za-z\s\"\'\-\,\.\;\:]*)(\))'
+            content = re.sub(replace_regex, r'\1\2\4\5\6', content)
         guide['content'] = ''
         if title:
             guide['content'] += '# {0} \n\n'.format(title)

.build_scripts/pandoc.sh

@@ -0,0 +1,22 @@
+cd $HOME/pdf-build
+ls . 
+mkdir -p $HOME/public/pdfs
+mkdir -p $HOME/tmp
+for d in $(find . -mindepth 1 -type d); do
+    mkdir $HOME/public/pdfs/"$d"
+done &&
+
+for i in $(find . -mindepth 1); do
+    if [ -f "$i" ] && [ "${i: -3}" == ".md" ]; then
+        pdfreplace=${i//\.md/.pdf}
+        pdffilename=${pdfreplace/\.\//}
+        filepath=${i//\.\//$HOME/pdf-build/}
+        if [[ "${filepath: -12}" == ".fullsite.md" ]]; then
+            pandoc "$filepath" -o "$HOME/tmp/$pdffilename" --pdf-engine=xelatex -V geometry:margin=1in -V papersize:a4 -V mainfont:Archivo-Regular --toc --toc-depth=1
+            gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/ebook -dPrinted=false -dNOPAUSE -dQUIET -dBATCH -sOutputFile="$HOME/public/pdfs/${pdffilename//\.fullsite/}" "$HOME/tmp/$pdffilename"
+        else
+            pandoc "$filepath" -o "$HOME/tmp/$pdffilename" --pdf-engine=xelatex -V geometry:margin=1in -V papersize:a4 -V mainfont:Archivo-Regular
+            gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/ebook -dPrinted=false -dNOPAUSE -dQUIET -dBATCH -sOutputFile="$HOME/public/pdfs/$pdffilename" "$HOME/tmp/$pdffilename"
+        fi 
+    fi
+done

Dockerfile

@@ -0,0 +1,33 @@
+FROM node AS search 
+COPY . /data
+WORKDIR /data
+RUN npm install 
+RUN node .build_scripts/build_index.js
+
+FROM jojomi/hugo AS hugo
+COPY --from=search /data /data
+WORKDIR /data
+RUN hugo
+
+FROM python:3.7 AS clean
+COPY --from=hugo /data /data
+WORKDIR /data
+RUN mkdir pdf-build
+RUN pip3 install -r requirements.txt
+RUN python3 .build_scripts/clean.py
+
+FROM dalibo/pandocker AS pandoc
+COPY --from=clean /data /data
+WORKDIR /data
+ENV HOME=/data
+RUN apt-get update
+RUN apt-get install ghostscript --yes
+RUN wget https://fonts.google.com/download?family=Archivo -O $HOME/archivo.zip 
+RUN unzip $HOME/archivo.zip -d $HOME/.fonts
+RUN bash $HOME/.build_scripts/pandoc.sh
+
+FROM python:3.7
+COPY --from=pandoc /data /data
+WORKDIR /data/public
+EXPOSE 7000
+CMD python3 -m http.server 7000