Add pywws.service.copy and remove pywws.towebsite

This completes making the file copying/uploading simpler and more
flexible.

Signed-off-by: Jim Easterbrook <jim@jim-easterbrook.me.uk>

src/doc/api_index.rst

@@ -52,6 +52,7 @@ Upload data to online "services"
 
    pywws.service.ftp
    pywws.service.sftp
+   pywws.service.copy
    pywws.service.cwop
    pywws.service.metoffice
    pywws.service.mqtt
@@ -77,8 +78,6 @@ Upload data to online "services"
    pywws.windrose
    pywws.template
    pywws.forecast
-   pywws.towebsite
-   pywws.totwitter
    pywws.weatherstation
    pywws.device_libusb1
    pywws.device_pyusb1

src/doc/guides/integration.rst

@@ -54,6 +54,7 @@ The remaining weather service uploads are handled by modules in the :ref:`pywws.
 
    pywws.service.ftp
    pywws.service.sftp
+   pywws.service.copy
    pywws.service.cwop
    pywws.service.metoffice
    pywws.service.mqtt

src/pywws/__init__.py

@@ -1,3 +1,3 @@
 __version__ = '18.8.0'
-_release = '1582'
-_commit = 'bf291f6'
+_release = '1583'
+_commit = '1e2219e'

src/pywws/regulartasks.py

@@ -30,7 +30,6 @@
 import pywws.plot
 import pywws.template
 from pywws.timezone import timezone
-import pywws.towebsite
 import pywws.windrose
 
 logger = logging.getLogger(__name__)
@@ -70,8 +69,6 @@ def __init__(self, context):
         self.uploads_directory = os.path.join(self.work_dir, 'uploads')
         if not os.path.isdir(self.uploads_directory):
             os.makedirs(self.uploads_directory)
-        # delay creation of uploader object until we know it's needed
-        self.uploader = None
         # get daytime end hour
         self.day_end_hour, self.use_dst = pywws.process.get_day_end_hour(
                                                                 self.params)
@@ -107,17 +104,26 @@ def __init__(self, context):
                     services.append(task)
                 changed = True
             if changed:
+                logger.error('updating Twitter in [%s]', section)
                 self.params.set(section, 'text', repr(templates))
                 self.params.set(section, 'services', repr(services))
-        # convert to use pywws.service.{ftp,sftp}
-        if not eval(self.params.get('ftp', 'local site', 'False')):
-            if eval(self.params.get('ftp', 'secure', 'False')):
-                for key in ('site', 'user', 'directory', 'port',
-                            'password', 'privkey'):
-                    self.params.set('sftp', key, self.params.get('ftp', key, ''))
-                mod = 'sftp'
-            else:
-                mod = 'ftp'
+        # convert to use pywws.service.{copy,ftp,sftp}
+        if self.params.get('ftp', 'local site') == 'True':
+            self.params.set(
+                'copy', 'directory', self.params.get('ftp', 'directory', ''))
+            mod = 'copy'
+        elif self.params.get('ftp', 'secure') == 'True':
+            for key in ('site', 'user', 'directory', 'port',
+                        'password', 'privkey'):
+                self.params.set('sftp', key, self.params.get('ftp', key, ''))
+            mod = 'sftp'
+        elif self.params.get('ftp', 'secure') == 'False':
+            mod = 'ftp'
+        else:
+            mod = None
+        for key in ('local site', 'secure', 'privkey'):
+            self.params.unset('ftp', key)
+        if mod:
             for section in list(self.cron.keys()) + [
                            'live', 'logged', 'hourly', '12 hourly', 'daily']:
                 for t_p in ('text', 'plot'):
@@ -135,6 +141,7 @@ def __init__(self, context):
                             services.append(task)
                         changed = True
                     if changed:
+                        logger.error('updating %s in [%s]', t_p, section)
                         self.params.set(section, t_p, repr(templates))
                         self.params.set(section, 'services', repr(services))
         # create service uploader objects
@@ -219,16 +226,6 @@ def _do_common(self, now, sections, live_data=None):
             self.services[name].do_catchup()
         for name, option in service_tasks:
             self.services[name].upload(live_data=live_data, option=option)
-        # upload non local files
-        upload_files = []
-        for name in os.listdir(self.uploads_directory):
-            path = os.path.join(self.uploads_directory, name)
-            if os.path.isfile(path):
-                upload_files.append(path)
-        if upload_files:
-            if not self.uploader:
-                self.uploader = pywws.towebsite.ToWebSite(self.context)
-            self.uploader.upload(upload_files, delete=True)
         # update status
         for section in sections:
             self.status.set('last update', section, now.isoformat(' '))
@@ -298,8 +295,6 @@ def do_tasks(self):
             # cleanly shut down upload threads
             for name in self.services:
                 self.services[name].stop()
-            if self.uploader:
-                self.uploader.stop()
 
     def do_plot(self, template, local=False):
         logger.info("Graphing %s", template)

src/pywws/service/__init__.py

@@ -36,90 +36,6 @@
 import pywws.template
 
 
-class UploadThread(threading.Thread):
-    def __init__(self, parent, context):
-        super(UploadThread, self).__init__()
-        self.parent = parent
-        self.context = context
-        self.queue = deque()
-
-    def run(self):
-        self.parent.logger.debug('thread started ' + self.name)
-        self.old_message = ''
-        if self.context.live_logging:
-            polling_interval = self.parent.interval.total_seconds() / 20
-            polling_interval = min(max(polling_interval, 4.0), 40.0)
-        else:
-            polling_interval = 4.0
-        while not self.context.shutdown.is_set():
-            try:
-                OK = self.upload_batch()
-            except Exception as ex:
-                self.log(str(ex))
-                OK = False
-            if OK:
-                pause = polling_interval
-            elif self.context.live_logging:
-                # upload failed, wait before trying again
-                pause = 40.0
-            else:
-                # upload failed or nothing more to do
-                break
-            self.context.shutdown.wait(pause)
-
-    def stop(self):
-        if self.is_alive():
-            self.parent.logger.debug('stopping thread ' + self.name)
-            self.queue.append(None)
-
-    def upload_batch(self):
-        if not self.queue:
-            return True
-        OK = True
-        count = 0
-        with self.parent.session() as session:
-            while self.queue and not self.context.shutdown.is_set():
-                if self.parent.catchup == 0:
-                    # "live only" service, so ignore old records
-                    drop = len(self.queue) - 1
-                    if self.queue[-1] is None:
-                        drop -= 1
-                    if drop > 0:
-                        for i in range(drop):
-                            self.queue.popleft()
-                        self.parent.logger.warning(
-                            '{:d} record(s) dropped'.format(drop))
-                # send upload without taking it off queue
-                upload = self.queue[0]
-                if upload is None:
-                    OK = False
-                    break
-                timestamp, kwds = upload
-                OK, message = self.parent.upload_data(session, **kwds)
-                self.log(message)
-                if not OK:
-                    break
-                count += 1
-                if timestamp:
-                    self.context.status.set(
-                        'last update', self.parent.service_name, str(timestamp))
-                # finally remove upload from queue
-                self.queue.popleft()
-        if self.parent.log_count:
-            if count > 1:
-                self.parent.logger.warning('{:d} records sent'.format(count))
-            elif count:
-                self.parent.logger.info('1 record sent')
-        return OK
-
-    def log(self, message):
-        if message == self.old_message:
-            self.parent.logger.debug(message)
-        else:
-            self.parent.logger.error(message)
-            self.old_message = message
-
-
 class ServiceBase(threading.Thread):
     interval = timedelta(seconds=40)
 

src/pywws/service/copy.py

@@ -0,0 +1,91 @@
+# pywws - Python software for USB Wireless Weather Stations
+# http://github.com/jim-easterbrook/pywws
+# Copyright (C) 2008-18  pywws contributors
+
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+"""Copy files to another directory.
+
+This module can be used to copy template and graph results to another
+directory on your computer. This could be useful if you are running a
+web server on the same machine as pywws (or on a machine that's
+accessible as a network share).
+
+* Example ``weather.ini`` configuration::
+
+    [copy]
+    directory = public_html/weather/data/
+
+    [hourly]
+    plot = [('24hrs.png.xml', 'L'), ('rose_12hrs.png.xml', 'L')]
+    text = [('24hrs.txt', 'L')]
+    services = [('copy', '24hrs.txt'), ('copy', '24hrs.png'),
+                ('copy', 'rose_12hrs.png')]
+
+Run :py:mod:`pywws.service.copy` once to set the default configuration,
+which you can then change. ``directory`` is the name of a directory in
+which all the copied files will be put.
+
+You can copy any files you like, as often as you like, but typical usage
+is to update a website once an hour. Each file to be copied needs a
+service entry like ``('copy', 'filename')``. If the file is not in your
+``local files`` directory then ``filename`` should be the full path.
+
+"""
+
+from __future__ import absolute_import
+
+from contextlib import contextmanager
+from datetime import timedelta
+import logging
+import os
+import shutil
+import sys
+
+import pywws.service
+
+__docformat__ = "restructuredtext en"
+service_name = os.path.splitext(os.path.basename(__file__))[0]
+logger = logging.getLogger(__name__)
+
+
+class ToService(pywws.service.FileService):
+    logger = logger
+    service_name = service_name
+
+    def __init__(self, context):
+        # base class init
+        super(ToService, self).__init__(context)
+        # get config
+        self.directory = context.params.get(service_name, 'directory', '')
+        if not self.directory:
+            raise RuntimeError('No directory specified in weather.ini')
+
+    @contextmanager
+    def session(self):
+        yield None
+
+    def upload_file(self, session, path):
+        try:
+            if not os.path.isdir(self.directory):
+                os.makedirs(self.directory)
+            shutil.copy(path, self.directory)
+        except Exception as ex:
+            return False, str(ex)
+        return True, 'OK'
+
+
+if __name__ == "__main__":
+    sys.exit(pywws.service.main(ToService))