Add 'ftp' and 'sftp' service modules

These are simpler, more flexible, replacements for parts of the pywws.towebsite module.

Signed-off-by: Jim Easterbrook <jim@jim-easterbrook.me.uk>

src/doc/api_index.rst

@@ -50,16 +50,18 @@ Upload data to online "services"
 .. autosummary::
    :toctree: api
 
+   pywws.service.ftp
+   pywws.service.sftp
    pywws.service.cwop
-   pywws.service.mastodon
    pywws.service.metoffice
    pywws.service.mqtt
    pywws.service.openweathermap
    pywws.service.pwsweather
    pywws.service.temperaturnu
-   pywws.service.twitter
    pywws.service.underground
    pywws.service.wetterarchivde
+   pywws.service.twitter
+   pywws.service.mastodon
 
 "Internal" modules
 ------------------

src/doc/guides/integration.rst

@@ -52,16 +52,18 @@ The remaining weather service uploads are handled by modules in the :ref:`pywws.
 
 .. autosummary::
 
+   pywws.service.ftp
+   pywws.service.sftp
    pywws.service.cwop
-   pywws.service.mastodon
    pywws.service.metoffice
    pywws.service.mqtt
    pywws.service.openweathermap
    pywws.service.pwsweather
    pywws.service.temperaturnu
-   pywws.service.twitter
    pywws.service.underground
    pywws.service.wetterarchivde
+   pywws.service.twitter
+   pywws.service.mastodon
 
 These each use a separate thread to upload the data so that a slow or not responding service doesn't delay other processing or uploads.
 

src/pywws/regulartasks.py

@@ -109,6 +109,34 @@ def __init__(self, context):
             if changed:
                 self.params.set(section, 'text', repr(templates))
                 self.params.set(section, 'services', repr(services))
+        # convert to use pywws.service.{ftp,sftp}
+        if not eval(self.params.get('ftp', 'local site', 'False')):
+            if eval(self.params.get('ftp', 'secure', 'False')):
+                for key in ('site', 'user', 'directory', 'port',
+                            'password', 'privkey'):
+                    self.params.set('sftp', key, self.params.get('ftp', key, ''))
+                mod = 'sftp'
+            else:
+                mod = 'ftp'
+            for section in list(self.cron.keys()) + [
+                           'live', 'logged', 'hourly', '12 hourly', 'daily']:
+                for t_p in ('text', 'plot'):
+                    templates = eval(self.params.get(section, t_p, '[]'))
+                    services = eval(self.params.get(section, 'services', '[]'))
+                    changed = False
+                    for n, template in enumerate(templates):
+                        if isinstance(template, (list, tuple)):
+                            continue
+                        templates[n] = (template, 'L')
+                        if t_p == 'plot':
+                            template = os.path.splitext(template)[0]
+                        task = (mod, template)
+                        if task not in services:
+                            services.append(task)
+                        changed = True
+                    if changed:
+                        self.params.set(section, t_p, repr(templates))
+                        self.params.set(section, 'services', repr(services))
         # create service uploader objects
         self.services = {}
         for section in list(self.cron.keys()) + [

src/pywws/service/ftp.py

@@ -0,0 +1,121 @@
+# pywws - Python software for USB Wireless Weather Stations
+# http://github.com/jim-easterbrook/pywws
+# Copyright (C) 2008-18  pywws contributors
+
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+"""Upload files to a web server by FTP.
+
+This module uploads files to (typically) a website *via* FTP. Details of
+the upload destination are stored in the file ``weather.ini`` in your
+data directory. You should be able to get the required information from
+your web space provider. If your provider allows SFTP then you could use
+:py:mod:`pywws.service.sftp` for greater security.
+
+* Example ``weather.ini`` configuration::
+
+    [ftp]
+    site = ftp.xxxx.yyyy.co.uk
+    user = xxxxxxx
+    password = zzzzzzzzz
+    directory = public_html/weather/data/
+    port = 21
+
+    [hourly]
+    plot = [('24hrs.png.xml', 'L'), ('rose_12hrs.png.xml', 'L')]
+    text = [('24hrs.txt', 'L')]
+    services = [('ftp', '24hrs.txt'), ('ftp', '24hrs.png'),
+                ('ftp', 'rose_12hrs.png')]
+
+Run :py:mod:`pywws.service.ftp` once to set the default configuration,
+which you can then change. ``directory`` is the name of a directory in
+which all the uploaded files will be put. This will depend on the
+structure of your web site and the sort of host you use. ``port`` is the
+port number to use. 21 is the standard value but your web space provider
+may require a different port.
+
+You can upload any files you like, as often as you like, but typical
+usage is to update a website once an hour. Each file to be uploaded
+needs a service entry like ``('ftp', 'filename')``. If the file is not
+in your ``local files`` directory then ``filename`` should be the full
+path.
+
+"""
+
+from __future__ import absolute_import
+
+from contextlib import contextmanager
+from datetime import timedelta
+import ftplib
+import logging
+import os
+import sys
+
+import pywws.service
+
+__docformat__ = "restructuredtext en"
+service_name = os.path.splitext(os.path.basename(__file__))[0]
+logger = logging.getLogger(__name__)
+
+
+class ToService(pywws.service.FileService):
+    logger = logger
+    service_name = service_name
+
+    def __init__(self, context):
+        # base class init
+        super(ToService, self).__init__(context)
+        # get config
+        self.params = {}
+        for key in ('site', 'user', 'password', 'directory'):
+            self.params[key] = context.params.get(service_name, key, '')
+        self.params['port'] = eval(context.params.get(service_name, 'port', '21'))
+        for key in self.params:
+            if not self.params[key]:
+                raise RuntimeError('No {} specified in weather.ini'.format(key))
+
+    @contextmanager
+    def session(self):
+        logger.info("Uploading to web site with FTP")
+        session = ftplib.FTP()
+        session.connect(self.params['site'], self.params['port'])
+        logger.debug('welcome message\n' + session.getwelcome())
+        session.login(self.params['user'], self.params['password'])
+        session.cwd(self.params['directory'])
+        try:
+            yield session
+        finally:
+            session.close()
+
+    def upload_file(self, session, path):
+        target = os.path.basename(path)
+        text_file = os.path.splitext(target)[1] in ('.txt', '.xml', '.html')
+        if text_file and sys.version_info[0] < 3:
+            mode = 'r'
+        else:
+            mode = 'rb'
+        try:
+            with open(path, mode) as f:
+                if text_file:
+                    session.storlines('STOR %s' % (target), f)
+                else:
+                    session.storbinary('STOR %s' % (target), f)
+        except Exception as ex:
+            return False, str(ex)
+        return True, 'OK'
+
+
+if __name__ == "__main__":
+    sys.exit(pywws.service.main(ToService))

src/pywws/service/sftp.py

@@ -0,0 +1,141 @@
+# pywws - Python software for USB Wireless Weather Stations
+# http://github.com/jim-easterbrook/pywws
+# Copyright (C) 2008-18  pywws contributors
+
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+"""Upload files to a web server by SFTP.
+
+This module uploads files to (typically) a website *via* SFTP. Details
+of the upload destination are stored in the file ``weather.ini`` in your
+data directory. You should be able to get the required information from
+your web space provider. If your provider doesn't allow SFTP then use
+:py:mod:`pywws.service.ftp` instead.
+
+* Additional dependency: https://www.paramiko.org/
+* Example ``weather.ini`` configuration::
+
+    [sftp]
+    site = ftp.xxxx.yyyy.co.uk
+    user = xxxxxxx
+    directory = public_html/weather/data/
+    port = 22
+    password =
+    privkey = /home/pywws/.ssh/webhost_rsa
+
+    [hourly]
+    plot = [('24hrs.png.xml', 'L'), ('rose_12hrs.png.xml', 'L')]
+    text = [('24hrs.txt', 'L')]
+    services = [('sftp', '24hrs.txt'), ('sftp', '24hrs.png'),
+                ('sftp', 'rose_12hrs.png')]
+
+Paramiko can be installed with ``pip``::
+
+    sudo pip install paramiko
+
+Run :py:mod:`pywws.service.sftp` once to set the default configuration,
+which you can then change. ``directory`` is the name of a directory in
+which all the uploaded files will be put. This will depend on the
+structure of your web site and the sort of host you use. ``port`` is the
+port number to use. 22 is the standard value but your web space provider
+may require a different port.
+
+Authentication can be by password or RSA public key. To use a key you
+first need to create a passwordless key pair using ``ssh-keygen``, then
+copy the public key to your web space provider. For example::
+
+    ssh-keygen -t rsa -f webhost_rsa
+    ssh-copy-id -i webhost_rsa.pub xxxxxxx@ftp.xxxx.yyyy.co.uk
+
+Move both key files to somewhere convenient, such as
+``/home/pywws/.ssh/`` and set ``privkey`` to the full path of the
+private key.
+
+You can upload any files you like, as often as you like, but typical
+usage is to update a website once an hour. Each file to be uploaded
+needs a service entry like ``('sftp', 'filename')``. If the file is not
+in your ``local files`` directory then ``filename`` should be the full
+path.
+
+"""
+
+from __future__ import absolute_import
+
+from contextlib import contextmanager
+from datetime import timedelta
+import logging
+import os
+import sys
+
+import paramiko
+
+import pywws.service
+
+__docformat__ = "restructuredtext en"
+service_name = os.path.splitext(os.path.basename(__file__))[0]
+logger = logging.getLogger(__name__)
+
+
+class ToService(pywws.service.FileService):
+    logger = logger
+    service_name = service_name
+
+    def __init__(self, context):
+        # base class init
+        super(ToService, self).__init__(context)
+        # get config
+        self.params = {}
+        for key in ('site', 'user', 'directory'):
+            self.params[key] = context.params.get(service_name, key, '')
+        self.params['port'] = eval(context.params.get(service_name, 'port', '22'))
+        for key in self.params:
+            if not self.params[key]:
+                raise RuntimeError('No {} specified in weather.ini'.format(key))
+        self.params['password'] = context.params.get(service_name, 'password', '')
+        self.params['privkey'] = context.params.get(service_name, 'privkey', '')
+
+    @contextmanager
+    def session(self):
+        logger.info("Uploading to web site with SFTP")
+        address = (self.params['site'], self.params['port'])
+        try:
+            with paramiko.Transport(address) as transport:
+                transport.start_client(timeout=30)
+                if self.params['privkey']:
+                    transport.auth_publickey(
+                        username=self.params['user'],
+                        key=paramiko.RSAKey.from_private_key_file(
+                            self.params['privkey']))
+                else:
+                    transport.auth_password(
+                        username=self.params['user'],
+                        password=self.params['password'])
+                with paramiko.SFTPClient.from_transport(transport) as session:
+                    session.chdir(self.params['directory'])
+                    yield session
+        except Exception as ex:
+            logger.exception(ex)
+
+    def upload_file(self, session, path):
+        target = os.path.basename(path)
+        try:
+            session.put(path, target)
+        except Exception as ex:
+            return False, str(ex)
+        return True, 'OK'
+
+
+if __name__ == "__main__":
+    sys.exit(pywws.service.main(ToService))