Nouveau processus de publication

- on arrête avec watchdog
- on prend une simple boucle de polling
- un peu de paralellisme

requirements.txt

@@ -34,7 +34,6 @@ dry-rest-permissions==0.1.10
 oauth2client==4.1.3
 
 # Zep 12 dependency
-watchdog==0.8.3 # use last non-bc-breaking version
 # next versions of this libraries break previous behavior for slug generation from string with single quotes
 django-uuslug==1.0.3
 python-slugify==1.1.4

zds/settings/abstract_base/zds.py

@@ -164,9 +164,9 @@
         'repo_private_path': join(BASE_DIR, 'contents-private'),
         'repo_public_path': join(BASE_DIR, 'contents-public'),
         'extra_contents_dirname': 'extra_contents',
-        # can also be 'extra_content_generation_policy': 'WATCHDOG'
+        # can also be 'extra_content_generation_policy': 'SYNC'
         # or 'extra_content_generation_policy': 'NOTHING'
-        'extra_content_generation_policy': 'SYNC',
+        'extra_content_generation_policy': 'WATCHDOG',
         'extra_content_watchdog_dir': join(BASE_DIR, 'watchdog-build'),
         'max_tree_depth': 3,
         'default_licence_pk': 7,

zds/tutorialv2/management/commands/publication_watchdog.py

@@ -1,79 +1,54 @@
-from os.path import dirname, join
-import os
+import logging
 import time
 
-import shutil
-from django.core.management import BaseCommand
-from pathtools.path import listdir
-from watchdog.observers import Observer
-from watchdog.events import FileCreatedEvent, FileSystemEventHandler, LoggingEventHandler
-from django.conf import settings
-from zds.tutorialv2.publication_utils import generate_external_content
-from codecs import open
-
-
-class TutorialIsPublished(FileSystemEventHandler):
-    prepare_callbacks = []  # because we can imagine we will create far more than test directory existence
-    finish_callbacks = []  # because we can imagine we will send a PM on success or failure one day
-
-    @staticmethod
-    def __create_dir(extra_contents_path):
-        if not os.path.exists(extra_contents_path):
-                os.makedirs(extra_contents_path)
-
-    @staticmethod
-    def __cleanup_build_and_watchdog(extra_contents_path, watchdog_file_path):
-        for listed in listdir(extra_contents_path, recursive=False):
-            try:
-                shutil.copy(join(extra_contents_path, listed), extra_contents_path.replace('__building', ''))
-            except Exception:
-                pass
-        shutil.rmtree(extra_contents_path)
-        os.remove(watchdog_file_path)
-
-    def __init__(self):
-        self.prepare_callbacks = [TutorialIsPublished.__create_dir]
-        self.finish_callbacks = [TutorialIsPublished.__cleanup_build_and_watchdog]
+from pathlib import Path
 
-    def on_created(self, event):
-        super(TutorialIsPublished, self).on_created(event)
-
-        if isinstance(event, FileCreatedEvent):
-            with open(event.src_path, encoding='utf-8') as f:
-                infos = f.read().strip().split(';')
-            md_file_path = infos[1]
-            base_name = infos[0]
-            extra_contents_path = dirname(md_file_path)
-            self.prepare_generation(extra_contents_path)
-            try:
-                generate_external_content(base_name, extra_contents_path, md_file_path, overload_settings=True,
-                                          excluded=['watchdog'])
-            finally:
-                self.finish_generation(extra_contents_path, event.src_path)
-
-    def prepare_generation(self, extra_contents_path):
-
-        for callback in self.prepare_callbacks:
-            callback(extra_contents_path)
+from django.core.management import BaseCommand
+from concurrent.futures import Future, ProcessPoolExecutor
 
-    def finish_generation(self, extra_contents_path, watchdog_file_path):
-        for callback in self.finish_callbacks:
-            callback(extra_contents_path, watchdog_file_path)
+from zds.tutorialv2.models.database import PublicationEvent, STATE_CHOICES
+from zds.tutorialv2.publication_utils import PublicatorRegistry
 
 
 class Command(BaseCommand):
     help = 'Launch a watchdog that generate all exported formats (epub, pdf...) files without blocking request handling'
 
     def handle(self, *args, **options):
-        path = settings.ZDS_APP['content']['extra_content_watchdog_dir']
-        event_handler = TutorialIsPublished()
-        observer = Observer()
-        observer.schedule(event_handler, path, recursive=True)
-        observer.schedule(LoggingEventHandler(), path)
-        observer.start()
-        try:
-            while True:
-                time.sleep(1)
-        except KeyboardInterrupt:
-            observer.stop()
-        observer.join()
+        logger = logging.getLogger(Command.__name__)
+        with ProcessPoolExecutor(5) as executor:
+            try:
+                while True:
+                    self.__launch_publicators(executor, logger)
+                    time.sleep(10)
+            except KeyboardInterrupt:
+                executor.shutdown(wait=False)
+
+    @staticmethod
+    def get_callback_of(publication_event: PublicationEvent):
+        def callback(future: Future):
+            if future.done():
+                publication_event.state_of_processing = 'SUCCESS'
+            elif future.cancelled():
+                publication_event.state_of_processing = 'FAILURE'
+            publication_event.save()
+        return callback
+
+    def __launch_publicators(self, executor, logger):
+        for publication_event in PublicationEvent.objects.filter(state_of_processing=STATE_CHOICES[0][0]):
+            logger.info('Export %s -- format=%s', publication_event.published_object.title(),
+                        publication_event.format_requested)
+            content = publication_event.published_object
+            publicator = PublicatorRegistry.get(publication_event.format_requested)
+
+            extra_content_dir = content.get_extra_contents_directory()
+            building_extra_content_path = Path(str(Path(extra_content_dir).parent) + '__building',
+                                               'extra_contents', content.content_public_slug)
+            if not building_extra_content_path.exists():
+                building_extra_content_path.mkdir(parents=True)
+            base_name = str(Path(str(building_extra_content_path), content.content_public_slug))
+            md_file_path = base_name + '.md'
+
+            future = executor.submit(publicator.publish, md_file_path, base_name)
+            publication_event.state_of_processing = STATE_CHOICES[1][0]
+            publication_event.save()
+            future.add_done_callback(Command.get_callback_of(publication_event))

zds/tutorialv2/migrations/0024_publicationevent.py

@@ -0,0 +1,27 @@
+# Generated by Django 2.1.5 on 2019-04-11 12:43
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tutorialv2', '0023_auto_20190114_1301'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PublicationEvent',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('state_of_processing', models.CharField(choices=[('REQUESTED', 'Export demandé'), ('RUNNING', 'Export en cours'), ('SUCCESS', 'Export réalisé'), ('FAILURE', 'Export échoué')], max_length=20)),
+                ('format_requested', models.CharField(max_length=25)),
+                ('published_object', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tutorialv2.PublishedContent', verbose_name='contenu publié')),
+            ],
+            options={
+                'verbose_name': 'Événement de publication',
+                'verbose_name_plural': 'Événements de publication',
+            },
+        ),
+    ]

zds/tutorialv2/models/database.py

@@ -1313,6 +1313,26 @@ def save(self, **kwargs):
         raise ValueError('Content cannot be null or something else than opinion.', self.content)
 
 
+STATE_CHOICES = [
+    ('REQUESTED', 'Export demandé'),
+    ('RUNNING', 'Export en cours'),
+    ('SUCCESS', 'Export réalisé'),
+    ('FAILURE', 'Export échoué'),
+]
+
+
+class PublicationEvent(models.Model):
+    class Meta:
+        verbose_name = 'Événement de publication'
+        verbose_name_plural = 'Événements de publication'
+
+    published_object = models.ForeignKey(PublishedContent, null=False, on_delete=models.CASCADE,
+                                         verbose_name='contenu publié')
+    state_of_processing = models.CharField(choices=STATE_CHOICES, null=False, blank=False, max_length=20)
+    # 25 for formats such as "printable.pdf", if tomorrow we want other "long" formats this will be ready
+    format_requested = models.CharField(blank=False, null=False, max_length=25)
+
+
 @receiver(models.signals.pre_delete, sender=User)
 def transfer_paternity_receiver(sender, instance, **kwargs):
     """
@@ -1322,4 +1342,5 @@ def transfer_paternity_receiver(sender, instance, **kwargs):
     PublishableContent.objects.transfer_paternity(instance, external, UserGallery)
     PublishedContent.objects.transfer_paternity(instance, external)
 
+
 import zds.tutorialv2.receivers  # noqa

zds/tutorialv2/publication_utils.py

@@ -17,7 +17,7 @@
 from django.conf import settings
 from zds.notification import signals
 from zds.tutorialv2.epub_utils import build_ebook
-from zds.tutorialv2.models.database import ContentReaction, PublishedContent
+from zds.tutorialv2.models.database import ContentReaction, PublishedContent, PublicationEvent
 from zds.tutorialv2.publish_container import publish_container
 from zds.tutorialv2.signals import content_unpublished
 from zds.utils.templatetags.emarkdown import render_markdown, MD_PARSING_ERROR
@@ -173,16 +173,14 @@ def generate_external_content(base_name, extra_contents_path, md_file_path, over
     :param base_name: base nae of file (without extension)
     :param extra_contents_path: internal directory where all files will be pushed
     :param md_file_path: bundled markdown file path
-    :param pandoc_debug_str: *specific to pandoc publication : avoid subprocess to be errored*
     :param overload_settings: this option force the function to generate all registered formats even when settings \
     ask for PDF not to be published
-    :return:
+    :param excluded: list of excluded format, None if no exclusion
     """
-    excluded = excluded or []
+    excluded = excluded or ['watchdog']
     excluded.append('md')
     if not settings.ZDS_APP['content']['build_pdf_when_published'] and not overload_settings:
         excluded.append('pdf')
-    # TODO: exclude watchdog
     for publicator_name, publicator in PublicatorRegistry.get_all_registered(excluded):
         try:
             publicator.publish(md_file_path, base_name, change_dir=extra_contents_path)
@@ -443,7 +441,7 @@ def tex_compiler(self, texfile, draftmode: str=''):
         command_process = subprocess.Popen(command,
                                            shell=True, cwd=path.dirname(texfile),
                                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        command_process.communicate()
+        command_process.communicate(timeout=120)
 
         with contextlib.suppress(ImportError):
             from raven import breadcrumbs
@@ -519,6 +517,8 @@ def publish(self, md_file_path, base_name, **kwargs):
                 os.remove(str(epub_path))
             if not epub_path.parent.exists():
                 epub_path.parent.mkdir(parents=True)
+            logger.info('created %s. moving it to %s', epub_file_path,
+                        published_content_entity.get_extra_contents_directory())
             shutil.move(str(epub_file_path), published_content_entity.get_extra_contents_directory())
 
 
@@ -536,10 +536,10 @@ def __init__(self, watched_dir):
     def publish(self, md_file_path, base_name, silently_pass=True, **kwargs):
         if silently_pass:
             return
-        filename = base_name.replace(path.dirname(base_name), self.watched_directory)
-        with open(filename, 'w', encoding='utf-8') as w_file:
-            w_file.write(';'.join([base_name, md_file_path]))
-        self.__logger.debug('Registered {} for generation'.format(md_file_path))
+        published_content = self.get_published_content_entity(md_file_path)
+        for requested_format in PublicatorRegistry.get_all_registered(['md', 'watchdog']):
+            PublicationEvent.objects.create(state_of_processing='REQUESTED', published_object=published_content,
+                                            format_requested=requested_format[0])
 
 
 class FailureDuringPublication(Exception):