Feature/646 restore email reply

backend/README.md

@@ -88,12 +88,13 @@ create wsgidav configuration file for webdav:
 
     cp wsgidav.conf.sample wsgidav.conf
 
+
+## Run Tracim_backend web services With Uwsgi : great for production ##
+
 if not did before, you need to create a color.json file at root of tracim_v2 :
 
     cp ../color.json.sample ../color.json
 
-## Run Tracim_backend With Uwsgi : great for production ##
-
 #### Install Uwsgi
 
 You can either install uwsgi with pip or with you distrib package manager:
@@ -109,15 +110,17 @@ or on debian 9 :
 ### All in terminal way ###
 
 
-Run all services with uwsgi
-
+Run all web services with uwsgi
+
+    ## UWSGI SERVICES
     # set tracim_conf_file path
     export TRACIM_CONF_PATH="$(pwd)/development.ini"
     export TRACIM_WEBDAV_CONF_PATH="$(pwd)/wsgidav.conf"
     # pyramid webserver
     uwsgi -d /tmp/tracim_web.log --http-socket :6543 --plugin python3 --wsgi-file wsgi/web.py -H env --pidfile /tmp/tracim_web.pid
     # webdav wsgidav server
     uwsgi -d /tmp/tracim_webdav.log --http-socket :3030 --plugin python3 --wsgi-file wsgi/webdav.py -H env --pidfile /tmp/tracim_webdav.pid
+
 
 to stop them:
 
@@ -166,6 +169,65 @@ run wsgidav server:
 
     tracimcli webdav start
 
+## Run daemons according to your config
+
+Feature such as async email notification and email reply system need additional
+daemons to work correctly.
+
+### python way
+
+#### Run
+    # set tracim_conf_file path
+    export TRACIM_CONF_PATH="$(pwd)/development.ini"
+    ## DAEMONS SERVICES
+    # email notifier (if async email notification is enabled)
+    python3 daemons/mail_notifier.py &
+    # email fetcher (if email reply is enabled)
+    python3 daemons/mail_fetcher.py &
+
+### STOP
+
+    # email notifier
+    killall python3 daemons/mail_notifier.py
+    # email fetcher
+    killall python3 daemons/mail_fetcher.py
+
+### Using Supervisor
+
+#### Install supervisor
+
+    sudo apt install supervisor
+
+#### Configure supervisord.conf file
+
+example of supervisord.conf file
+
+    [supervisord]
+    ; You need to replace <PATH> with correct absolute path
+
+    ; email notifier (if async email notification is enabled)
+    [program:tracim_mail_notifier]
+    directory=<PATH>/tracim_v2/backend/
+    command=<PATH>/tracim_v2/backend/env/bin/python <PATH>/tracim_v2/backend/daemons/mail_notifier.py
+    stdout_logfile =/tmp/mail_notifier.log
+    redirect_stderr=true
+    autostart=true
+    autorestart=true
+    environment=TRACIM_CONF_PATH=<PATH>/tracim_v2/backend/development.ini
+
+    ; email fetcher (if email reply is enabled)
+    [program:tracim_mail_fetcher]
+    directory=<PATH>/tracim_v2/backend/
+    command=<PATH>/tracim_v2/backend/env/bin/python <PATH>/tracim_v2/backend/daemons/mail_fetcher.py
+    stdout_logfile =/tmp/mail_fetcher.log
+    redirect_stderr=true
+    autostart=true
+    autorestart=true
+    environment=TRACIM_CONF_PATH=<PATH>/tracim_v2/backend/development.ini
+
+run with (supervisord.conf should be provided, see [supervisord.conf default_paths](http://supervisord.org/configuration.html):
+
+    supervisord
 
 ## Run Tests and others checks ##
 

backend/daemons/mail_fetcher.py

@@ -0,0 +1,18 @@
+# coding=utf-8
+# Runner for daemon
+import os
+from pyramid.paster import get_appsettings
+from pyramid.paster import setup_logging
+from tracim_backend import CFG
+from tracim_backend.lib.mail_fetcher.daemon import MailFetcherDaemon
+
+config_uri = os.environ['TRACIM_CONF_PATH']
+
+setup_logging(config_uri)
+settings = get_appsettings(config_uri)
+settings.update(settings.global_conf)
+app_config = CFG(settings)
+app_config.configure_filedepot()
+
+daemon = MailFetcherDaemon(app_config, burst=False)
+daemon.run()

backend/daemons/mail_notifier.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+# Runner for daemon
+import os
+
+from pyramid.paster import get_appsettings
+from pyramid.paster import setup_logging
+from tracim_backend import CFG
+from tracim_backend.lib.mail_notifier.daemon import MailSenderDaemon
+
+config_uri = os.environ['TRACIM_CONF_PATH']
+
+setup_logging(config_uri)
+settings = get_appsettings(config_uri)
+settings.update(settings.global_conf)
+app_config = CFG(settings)
+app_config.configure_filedepot()
+
+daemon = MailSenderDaemon(app_config, burst=False)
+daemon.run()

backend/setup.py

@@ -42,6 +42,11 @@
     'lxml',
     'redis',
     'rq',
+    # mail-fetcher
+    'markdown',
+    'email_reply_parser',
+    'filelock',
+    'imapclient',
     # auth
     'pyramid_multiauth',
     'beaker',
@@ -59,6 +64,8 @@
     'pep8',
     'mypy',
     'requests',
+    'responses',
+    'mock',
     'Pillow',
     'freezegun'
 ]

backend/tracim_backend/config.py

@@ -132,7 +132,6 @@ def __init__(self, settings):
             'session.reissue_time',
             120
         ))
-
         self.WEBSITE_TITLE = settings.get(
             'website.title',
             'TRACIM',
@@ -330,61 +329,61 @@ def __init__(self, settings):
             None,
         )
 
-        # self.EMAIL_REPLY_ACTIVATED = asbool(settings.get(
-        #     'email.reply.activated',
-        #     False,
-        # ))
-        #
-        # self.EMAIL_REPLY_IMAP_SERVER = settings.get(
-        #     'email.reply.imap.server',
-        # )
-        # self.EMAIL_REPLY_IMAP_PORT = settings.get(
-        #     'email.reply.imap.port',
-        # )
-        # self.EMAIL_REPLY_IMAP_USER = settings.get(
-        #     'email.reply.imap.user',
-        # )
-        # self.EMAIL_REPLY_IMAP_PASSWORD = settings.get(
-        #     'email.reply.imap.password',
-        # )
-        # self.EMAIL_REPLY_IMAP_FOLDER = settings.get(
-        #     'email.reply.imap.folder',
-        # )
-        # self.EMAIL_REPLY_CHECK_HEARTBEAT = int(settings.get(
-        #     'email.reply.check.heartbeat',
-        #     60,
-        # ))
-        # self.EMAIL_REPLY_TOKEN = settings.get(
-        #     'email.reply.token',
-        # )
-        # self.EMAIL_REPLY_IMAP_USE_SSL = asbool(settings.get(
-        #     'email.reply.imap.use_ssl',
-        # ))
-        # self.EMAIL_REPLY_IMAP_USE_IDLE = asbool(settings.get(
-        #     'email.reply.imap.use_idle',
-        #     True,
-        # ))
-        # self.EMAIL_REPLY_CONNECTION_MAX_LIFETIME = int(settings.get(
-        #     'email.reply.connection.max_lifetime',
-        #     600,  # 10 minutes
-        # ))
-        # self.EMAIL_REPLY_USE_HTML_PARSING = asbool(settings.get(
-        #     'email.reply.use_html_parsing',
-        #     True,
-        # ))
-        # self.EMAIL_REPLY_USE_TXT_PARSING = asbool(settings.get(
-        #     'email.reply.use_txt_parsing',
-        #     True,
-        # ))
-        # self.EMAIL_REPLY_LOCKFILE_PATH = settings.get(
-        #     'email.reply.lockfile_path',
-        #     ''
-        # )
-        # if not self.EMAIL_REPLY_LOCKFILE_PATH and self.EMAIL_REPLY_ACTIVATED:
-        #     raise Exception(
-        #         mandatory_msg.format('email.reply.lockfile_path')
-        #     )
-        #
+        self.EMAIL_REPLY_ACTIVATED = asbool(settings.get(
+            'email.reply.activated',
+            False,
+        ))
+
+        self.EMAIL_REPLY_IMAP_SERVER = settings.get(
+            'email.reply.imap.server',
+        )
+        self.EMAIL_REPLY_IMAP_PORT = settings.get(
+            'email.reply.imap.port',
+        )
+        self.EMAIL_REPLY_IMAP_USER = settings.get(
+            'email.reply.imap.user',
+        )
+        self.EMAIL_REPLY_IMAP_PASSWORD = settings.get(
+            'email.reply.imap.password',
+        )
+        self.EMAIL_REPLY_IMAP_FOLDER = settings.get(
+            'email.reply.imap.folder',
+        )
+        self.EMAIL_REPLY_CHECK_HEARTBEAT = int(settings.get(
+            'email.reply.check.heartbeat',
+            60,
+        ))
+        self.EMAIL_REPLY_TOKEN = settings.get(
+            'email.reply.token',
+        )
+        self.EMAIL_REPLY_IMAP_USE_SSL = asbool(settings.get(
+            'email.reply.imap.use_ssl',
+        ))
+        self.EMAIL_REPLY_IMAP_USE_IDLE = asbool(settings.get(
+            'email.reply.imap.use_idle',
+            True,
+        ))
+        self.EMAIL_REPLY_CONNECTION_MAX_LIFETIME = int(settings.get(
+            'email.reply.connection.max_lifetime',
+            600,  # 10 minutes
+        ))
+        self.EMAIL_REPLY_USE_HTML_PARSING = asbool(settings.get(
+            'email.reply.use_html_parsing',
+            True,
+        ))
+        self.EMAIL_REPLY_USE_TXT_PARSING = asbool(settings.get(
+            'email.reply.use_txt_parsing',
+            True,
+        ))
+        self.EMAIL_REPLY_LOCKFILE_PATH = settings.get(
+            'email.reply.lockfile_path',
+            ''
+        )
+        if not self.EMAIL_REPLY_LOCKFILE_PATH and self.EMAIL_REPLY_ACTIVATED:
+            raise Exception(
+                mandatory_msg.format('email.reply.lockfile_path')
+            )
+
         self.EMAIL_PROCESSING_MODE = settings.get(
             'email.processing_mode',
             'sync',

backend/tracim_backend/exceptions.py

@@ -193,6 +193,26 @@ class EmptyCommentContentNotAllowed(EmptyValueNotAllowed):
     pass
 
 
+class EmptyEmailBody(EmptyValueNotAllowed):
+    pass
+
+
+class NoSpecialKeyFound(EmptyValueNotAllowed):
+    pass
+
+
+class UnsupportedRequestMethod(TracimException):
+    pass
+
+
+class CommentRequestCreationFailed(TracimException):
+    pass
+
+
+class BadStatusCode(TracimException):
+    pass
+
+
 class UserNotActive(TracimException):
     pass
 

backend/tracim_backend/lib/core/userworkspace.py

@@ -129,6 +129,8 @@ def create_one(
         with_notif: bool,
         flush: bool=True
     ) -> UserRoleInWorkspace:
+
+        # TODO - G.M - 2018-08-24 - Check if role already exist
         role = UserRoleInWorkspace()
         role.user_id = user.user_id
         role.workspace = workspace

backend/tracim_backend/lib/mail_fetcher/daemon.py

@@ -0,0 +1,50 @@
+import typing
+
+from tracim_backend import BASE_API_V2
+from tracim_backend.lib.mail_fetcher.email_fetcher import MailFetcher
+from tracim_backend.lib.utils.daemon import FakeDaemon
+from tracim_backend.lib.utils.logger import logger
+
+
+class MailFetcherDaemon(FakeDaemon):
+    """
+    Thread containing a daemon who fetch new mail from a mailbox and
+    send http request to a tracim endpoint to handle them.
+    """
+    def __init__(self, config: 'CFG', burst=True, *args, **kwargs):
+        """
+        :param config: Tracim Config
+        :param burst: if true, run one time, if false, run continously
+        """
+        super().__init__(*args, **kwargs)
+        self.config = config
+        self._fetcher = None  # type: MailFetcher
+        self.burst = burst
+
+    def append_thread_callback(self, callback: typing.Callable) -> None:
+        logger.warning('MailFetcherrDaemon not implement append_thread_callback')  # nopep8
+        pass
+
+    def stop(self) -> None:
+        if self._fetcher:
+            self._fetcher.stop()
+
+    def run(self) -> None:
+        self._fetcher = MailFetcher(
+            host=self.config.EMAIL_REPLY_IMAP_SERVER,
+            port=self.config.EMAIL_REPLY_IMAP_PORT,
+            user=self.config.EMAIL_REPLY_IMAP_USER,
+            password=self.config.EMAIL_REPLY_IMAP_PASSWORD,
+            use_ssl=self.config.EMAIL_REPLY_IMAP_USE_SSL,
+            folder=self.config.EMAIL_REPLY_IMAP_FOLDER,
+            heartbeat=self.config.EMAIL_REPLY_CHECK_HEARTBEAT,
+            use_idle=self.config.EMAIL_REPLY_IMAP_USE_IDLE,
+            connection_max_lifetime=self.config.EMAIL_REPLY_CONNECTION_MAX_LIFETIME,  # nopep8
+            api_base_url=self.config.WEBSITE_BASE_URL + BASE_API_V2,
+            api_key=self.config.API_KEY,
+            use_html_parsing=self.config.EMAIL_REPLY_USE_HTML_PARSING,
+            use_txt_parsing=self.config.EMAIL_REPLY_USE_TXT_PARSING,
+            lockfile_path=self.config.EMAIL_REPLY_LOCKFILE_PATH,
+            burst=self.burst
+        )
+        self._fetcher.run()