Replace unused feed.keep with used time based keep

In addition, I've added feed.keep_unread which will avoid discarding
items with unread status.

Signed-off-by: Jack Miller <jack@codezen.org>

canto_next/config.py

@@ -27,7 +27,8 @@
         "defaults" :
         {
             "rate" : 10,
-            "keep" : 0,
+            "keep_time" : 86400,
+            "keep_unread" : False,
             "global_transform" : "None"
         },
 
@@ -54,22 +55,25 @@ def __init__(self, filename, shelf):
         self.json = {}
 
         self.defaults_validators = [
-                ("rate", self.validate_int, True),
-                ("keep", self.validate_int, True),
-                ("global_transform", self.validate_set_transform, True),
+                ("rate", self.validate_int, False),
+                ("keep_time", self.validate_int, False),
+                ("keep_unread", self.validate_bool, False),
+                ("global_transform", self.validate_set_transform, False),
         ]
 
         self.defaults_defaults = {
                 "rate" : 10,
-                "keep" : 0,
+                "keep_time" : 86400,
+                "keep_unread" : False,
                 "global_transform" : "None",
         }
 
         self.feed_validators = [
                 ("name", self.validate_unique_feed_name, True),
                 ("url", self.validate_unique_url, True),
                 ("rate", self.validate_int, False),
-                ("keep", self.validate_int, False),
+                ("keep_time", self.validate_int, False),
+                ("keep_unread", self.validate_bool, False),
                 ("username", self.validate_string, False),
                 ("password", self.validate_string, False),
         ]
@@ -136,6 +140,12 @@ def validate_unique_url(self, ident, value):
     def validate_unique_feed_name(self, ident, value):
         return self._validate_unique(ident, value, self.feed_names, "Feed name")
 
+    def validate_bool(self, ident, value):
+        if type(value) != bool:
+            self.error(ident, value, "Not boolean!")
+            return False
+        return (True, value)
+
     def validate_int(self, ident, value):
         if type(value) != int:
             self.error(ident, value, "Not integer!")
@@ -217,6 +227,9 @@ def validate_dict(self, ident_prefix, d, validators):
 
         return not section_invalidated
 
+    # Validate validates only what exists in self.final, it does not make
+    # substitutions for defaults. That's done on instantiation.
+
     def validate(self):
         # Because we have to ensure that all items in the JSON are
         # simple, we can do this cheap deepcopy intead of importing
@@ -229,8 +242,6 @@ def validate(self):
                     self.defaults_validators)
             if not good:
                 del self.final["defaults"]
-        else:
-            self.final["defaults"] = self.defaults_defaults.copy()
 
         if "tags" in self.final and not self.errors:
             for tag in list(self.final["tags"].keys()):
@@ -261,6 +272,13 @@ def validate(self):
 
     def instantiate(self):
 
+        if "defaults" in self.final:
+            for k in self.defaults_defaults.keys():
+                if k not in self.final["defaults"]:
+                    self.final["defaults"][k] = self.defaults_defaults[k]
+        else:
+            self.final["defaults"] = self.defaults_defaults.copy()
+
         if "tags" in self.final:
             for tag in self.final["tags"]:
                 defs = self.final["tags"][tag]
@@ -284,10 +302,9 @@ def instantiate(self):
             for feed in self.final["feeds"]:
 
                 # Mandatory arguments to CantoFeed
-                if "rate" not in feed:
-                    feed["rate"] = self.final["defaults"]["rate"]
-                if "keep" not in feed:
-                    feed["keep"] = self.final["defaults"]["keep"]
+                for k in [ "rate", "keep_time", "keep_unread" ]:
+                    if k not in feed:
+                        feed[k] = self.final["defaults"][k]
 
                 # Optional arguments in kwargs
                 kws = {}
@@ -296,7 +313,7 @@ def instantiate(self):
                         kws[k] = feed[k]
 
                 feed = CantoFeed(self.shelf, feed["name"],\
-                        feed["url"], feed["rate"], feed["keep"], **kws)
+                        feed["url"], feed["rate"], feed["keep_time"], feed["keep_unread"], **kws)
 
         # Now that feeds have instantiated the tags, trim out the tags that
         # have configuration but are unused.

canto_next/feed.py

@@ -79,7 +79,7 @@ class DaemonFeedPlugin(Plugin):
     pass
 
 class CantoFeed(PluginHandler):
-    def __init__(self, shelf, name, URL, rate, keep, **kwargs):
+    def __init__(self, shelf, name, URL, rate, keep_time, keep_unread, **kwargs):
         PluginHandler.__init__(self)
 
         self.plugin_class = DaemonFeedPlugin
@@ -90,7 +90,8 @@ def __init__(self, shelf, name, URL, rate, keep, **kwargs):
         self.name = name
         self.URL = URL
         self.rate = rate
-        self.keep = keep
+        self.keep_time = keep_time
+        self.keep_unread = keep_unread
 
         self.username = None
         if "username" in kwargs:
@@ -329,16 +330,28 @@ def index(self):
         ref_time = time.time()
         for idx, item in unprotected_old:
             # Old item
-            if "canto_update" not in item:
-                item["canto_update"] = ref_time
+            if "canto_update" not in old_contents["entries"][idx]:
+                old_contents["entries"][idx]["canto_update"] = ref_time
                 log.debug("Subbing item time %s" % item)
 
-            if (ref_time - item["canto_update"]) < 60*60*24:
-                self.update_contents["entries"].append(\
-                        old_contents["entries"][idx])
-                self.items.append(item)
+            item_time = old_contents["entries"][idx]["canto_update"]
+            if "canto-state" in old_contents["entries"][idx]:
+                item_state = old_contents["entries"][idx]["canto-state"]
+            else:
+                item_state = []
+
+            if (ref_time - item_time) < self.keep_time:
+                log.debug("Item not over keep_time (%d): %s" %
+                        (self.keep_time, item))
+            elif self.keep_unread and "read" not in item_state:
+                log.debug("Keeping unread item: %s\n" % item)
             else:
                 log.debug("Discarding: %s", item)
+                continue
+
+            self.update_contents["entries"].append(\
+                    old_contents["entries"][idx])
+            self.items.append(item)
 
         # Allow plugins DaemonFeedPlugins defining edit_* functions to have a
         # crack at the contents before we commit to disk.