Replace dict with dataclass for locally cached data

gh2slack.py

@@ -140,7 +140,7 @@ def main():
                 slack_token, args.slack_base_url, args.slack_timeout
             )
             for html_url in to_publish:
-                cache_item = cache[html_url]
+                cache_item = cache.items[html_url]
                 try:
                     message = assembly_slack_message(
                         logger, args.gh_owner, args.gh_repo,
@@ -151,7 +151,7 @@ def main():
                     )
                 except Exception:
                     logger.error(traceback.format_exc())
-                    cache.pop(html_url)
+                    cache.items.pop(html_url)
                 finally:
                     time.sleep(args.sleep)
 
@@ -235,7 +235,7 @@ def parse_args() -> argparse.Namespace:
 
 
 def process_page_items(
-        logger: logging.Logger, cache: Dict, pages: List,
+        logger: logging.Logger, cache: rss2irc.CachedData, pages: List,
         expiration: int, repository_url: str
 ) -> Set:
     """Parse page items, update cache and return items to publish.
@@ -256,8 +256,8 @@ def process_page_items(
                 logger.debug("Item doesn't have required fields: %s", item)
                 continue
 
-            if item['html_url'] in cache:
-                cache[item['html_url']]['expiration'] = expiration
+            if item['html_url'] in cache.items:
+                cache.items[item['html_url']]['expiration'] = expiration
                 continue
 
             try:
@@ -266,7 +266,7 @@ def process_page_items(
                 logger.error('Failed to convert %s to int.', item['number'])
                 item_number = 0
 
-            cache[item['html_url']] = {
+            cache.items[item['html_url']] = {
                 'expiration': expiration,
                 'number': item_number,
                 'repository_url': repository_url,
@@ -277,22 +277,22 @@ def process_page_items(
     return to_publish
 
 
-def scrub_cache(logger: logging.Logger, cache: Dict) -> None:
+def scrub_cache(logger: logging.Logger, cache: rss2irc.CachedData) -> None:
     """Scrub cache and remove expired items."""
     time_now = int(time.time())
-    for key in list(cache.keys()):
+    for key in list(cache.items.keys()):
         try:
-            expiration = int(cache[key]['expiration'])
+            expiration = int(cache.items[key]['expiration'])
         except (KeyError, ValueError):
             logger.error(traceback.format_exc())
             logger.error("Invalid cache entry will be removed: '%s'",
-                         cache[key])
-            cache.pop(key)
+                         cache.items[key])
+            cache.items.pop(key)
             continue
 
         if expiration < time_now:
             logger.debug('URL %s has expired.', key)
-            cache.pop(key)
+            cache.items.pop(key)
 
 
 if __name__ == '__main__':

migrations/convert_cache_to_dataclass_v1.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Convert original cache file(simple dict) to data class v1.
+
+Migration:
+* disable all 2IRC/2Slack scripts in eg. cron/systemd/runit/etc.
+* migrate cache files with this script
+* enable 2IRC/2Slack scripts again
+* if everything is ok, remove bak files
+"""
+import argparse
+import logging
+import pickle
+import shutil
+import sys
+from dataclasses import dataclass, field
+
+
+@dataclass
+class CachedData:
+    """CachedData represents locally cached data and state."""
+
+    items: dict = field(default_factory=dict)
+
+
+def main():
+    """Open cache file, convert it and overwrite it.
+
+    Backup file is created in the process. Manual cleanup is required after
+    migration.
+    """
+    logging.basicConfig(stream=sys.stdout)
+    logger = logging.getLogger('migrate-to-dataclass')
+    args = parse_args()
+    if args.verbosity:
+        logger.setLevel(logging.DEBUG)
+
+    logger.info("Read cache from file '%s'.", args.cache)
+    with open(args.cache, 'rb') as fhandle:
+        cache = pickle.load(fhandle)
+
+    if not isinstance(cache, dict):
+        logger.error(
+            "Cache file '%s' has invalid format, dict is expected.", args.cache
+        )
+        sys.exit(1)
+
+    bak_file = '{}.bak'.format(args.cache)
+    logger.info("Create backup file '%s' from '%s'.", bak_file, args.cache)
+    shutil.copy2(args.cache, bak_file)
+
+    new_cache = CachedData()
+    for key, value in cache.items():
+        new_cache.items[key] = value
+
+    logger.info("Write converted cache into file '%s'.", args.cache)
+    with open(args.cache, 'wb') as fhandle:
+        pickle.dump(new_cache, fhandle, pickle.HIGHEST_PROTOCOL)
+
+    logger.info("Migration complete and '%s' can be removed.", bak_file)
+
+
+def parse_args() -> argparse.Namespace:
+    """Return parsed CLI args."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-v', '--verbose',
+        dest='verbosity', action='store_true', default=False,
+        help='Increase logging verbosity.'
+    )
+    parser.add_argument(
+        '--cache',
+        dest='cache', type=str, default=None, required=True,
+        help='File which contains cache.'
+    )
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    main()

migrations/tests/test_convert_cache_to_dataclass_v1.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Unit tests for convert_cache_to_dataclass_v1.py."""
+import io
+import os
+import pickle
+import sys
+import tempfile
+from unittest.mock import patch
+
+import pytest
+
+import migrations.convert_cache_to_dataclass_v1 as migration  # noqa:I202
+
+
+@pytest.fixture
+def fixture_bak_cleanup():
+    """Cleanup bak file which is created during migration."""
+    bak_fnames = []
+
+    def _fixture_back_cleanup(bak_fname):
+        bak_fnames.append(bak_fname)
+        return bak_fname
+
+    yield _fixture_back_cleanup
+
+    for bak_fname in bak_fnames:
+        print("teardown of '{}'".format(bak_fname))
+        os.remove(bak_fname)
+
+
+@pytest.fixture
+def fixture_cache_file():
+    """Create tmpfile and return its file name."""
+    file_desc, fname = tempfile.mkstemp()
+    os.fdopen(file_desc).close()
+    yield fname
+    # Cleanup
+    try:
+        os.unlink(fname)
+    except FileNotFoundError:
+        pass
+
+
+def test_migration(fixture_cache_file, fixture_bak_cleanup):
+    """Test migration under ideal conditions."""
+    bak_file = '{}.bak'.format(fixture_cache_file)
+    _ = fixture_bak_cleanup(bak_file)
+    expected_cache = migration.CachedData(
+        items={
+            'test1': 1234,
+            'test2': 0,
+        }
+    )
+
+    test_data = {
+        'test1': 1234,
+        'test2': 0,
+    }
+    with open(fixture_cache_file, 'wb') as fhandle:
+        pickle.dump(test_data, fhandle, pickle.HIGHEST_PROTOCOL)
+
+    exception = None
+    args = [
+        './convert_cache_to_dataclass_v1.py',
+        '--cache',
+        fixture_cache_file,
+    ]
+
+    saved_stdout = sys.stdout
+    out = io.StringIO()
+    sys.stdout = out
+
+    with patch.object(sys, 'argv', args):
+        try:
+            migration.main()
+        except Exception as exc:
+            exception = exc
+        finally:
+            sys.stdout = saved_stdout
+
+    assert exception is None
+    assert os.path.exists(bak_file) is True
+
+    with open(fixture_cache_file, 'rb') as fhandle:
+        migrated_cache = pickle.load(fhandle)
+    assert migrated_cache == expected_cache

phpbb2slack.py

@@ -87,14 +87,14 @@ def main():
         scrub_cache(logger, cache)
 
         for key in list(news.keys()):
-            if key not in cache:
+            if key not in cache.items:
                 continue
 
             logger.debug('Key %s found in cache', key)
-            comments_cached = int(cache[key]['comments_cnt'])
+            comments_cached = int(cache.items[key]['comments_cnt'])
             comments_actual = int(news[key]['comments_cnt'])
             if comments_cached == comments_actual:
-                cache[key]['expiration'] = (
+                cache.items[key]['expiration'] = (
                     int(time.time()) + args.cache_expiration
                 )
                 news.pop(key)
@@ -235,28 +235,31 @@ def parse_news(data: str, authors: List[str]) -> Dict:
     return news
 
 
-def scrub_cache(logger: logging.Logger, cache: Dict) -> None:
+def scrub_cache(logger: logging.Logger, cache: rss2irc.CachedData) -> None:
     """Scrub cache and remove expired items."""
     time_now = int(time.time())
-    for key in list(cache.keys()):
+    for key in list(cache.items.keys()):
         try:
-            expiration = int(cache[key]['expiration'])
+            expiration = int(cache.items[key]['expiration'])
         except (KeyError, ValueError):
             logger.error(traceback.format_exc())
-            logger.error("Invalid cache entry will be removed: '%s'",
-                         cache[key])
-            cache.pop(key)
+            logger.error(
+                "Invalid cache entry will be removed: '%s'", cache.items[key]
+            )
+            cache.items.pop(key)
             continue
 
         if expiration < time_now:
             logger.debug('URL %s has expired.', key)
-            cache.pop(key)
+            cache.items.pop(key)
 
 
-def update_cache(cache: Dict, news: Dict, expiration: int) -> None:
+def update_cache(
+        cache: rss2irc.CachedData, news: Dict, expiration: int
+) -> None:
     """Update cache contents."""
     for key in list(news.keys()):
-        cache[key] = {
+        cache.items[key] = {
             'expiration': expiration,
             'comments_cnt': int(news[key]['comments_cnt']),
         }

requirements.txt

@@ -1,3 +1,3 @@
-feedparser==5.2.1
-requests==2.22.0
-slackclient>=2.5.0,<2.6
+feedparser==6.0.2
+requests==2.25.1
+slackclient==2.9.3