Merge branch 'datkey_unixtime'

doc/2ch-interface.README.md

@@ -31,7 +31,9 @@
   例えば「テスト」スレを新規作成のつもりで投稿しても、既存のスレがネットワークにある場合1番目の投稿にはなりません。
 
 * 通常スレッドキーはそれが作られた時間を表していますが、
-  このインターフェイスでは無意味な数値になっています。
+  このインターフェイスではそのノードが初めてそのスレッドで手に入れた書き込みを元にしています。
+  そのためスレタイが同じでもスレッドキーはノードごとに異なるものになります。
+  また>>1以前に書き込みが追加されると新たな>>1の書き込み時間とスレッドキーはずれます。
 
 板
 -----------------

shingetsu/mch/datd.py

@@ -282,7 +282,7 @@ def __init__(self, *args, **kwds):
 
     def run(self):
         utils.log('start 2ch interface')
-        keylib.load(config.cache_dir)
+        keylib.load()
         try:
             import waitress
         except ImportError:

shingetsu/mch/keylib.py

@@ -1,50 +1,126 @@
+import threading
 
+from shingetsu import cache
+from shingetsu import config
+from shingetsu import util
 
-import threading
-import os
+from . import utils
+
+
+class _DatkeyTable:
+    def __init__(self, file):
+        self.file = file
+        self.datkey2filekey = {}  # epoch stamp -> `Cache.datfile`
+        self.filekey2datkey = {}  # `Cache.datfile` -> epoch stamp
+        self.lock = threading.Lock()
+
+
+    def load(self):
+        try:
+            with util.opentext(self.file, 'r') as f:
+                for line in f:
+                    try:
+                        # format -> `12345<>thread_FFFF\n`
+                        stamp, filekey = line.strip().split('<>', 1)
+                        stamp = int(stamp)
+                    except ValueError:
+                        continue
+                    self.set_entry(stamp, filekey)
+
+        except IOError:
+            pass
+
+
+    def save(self):
+        try:
+            with util.opentext(self.file, 'w') as f:
+                for stamp, filekey in self.datkey2filekey.items():
+                    f.write(str(stamp) + '<>' + filekey + '\n')
+
+        except IOError:
+            utils.log('keylib._DatkeyTable.save fails')
+
+
+    def set_entry(self, stamp, filekey):
+        with self.lock:
+            self.datkey2filekey[stamp] = filekey
+            self.filekey2datkey[filekey] = stamp
+
+
+    def set_from_cache(self, cache):
+        if cache.datfile in self.filekey2datkey:
+            return
+
+        try:
+            rec = cache[cache.keys()[0]]  # first record
+            rec.load()
+            first_stamp = int(rec.get('stamp', 0))
+        except IndexError:
+            first_stamp = cache.recent_stamp  # if don't have recent_stamp, it's 0
+
+        # avoid duplication
+        while first_stamp in self.datkey2filekey:
+            first_stamp += 1
 
+        self.set_entry(first_stamp, cache.datfile)
+
+
+    def get_datkey(self, filekey):
+        if filekey in self.filekey2datkey:
+            return self.filekey2datkey[filekey]
+
+        # because saku don't have this cache at start
+        c = cache.Cache(filekey)
+        c.load()
+        self.set_from_cache(c)
+
+        self.save()  # don't lose entry
+
+        if filekey in self.filekey2datkey:
+            return self.filekey2datkey[filekey]
+
+        raise DatkeyNotFound(filekey + ' not found')
+
+
+    def get_filekey(self, datkey):
+        datkey = int(datkey)
+        if datkey in self.datkey2filekey:
+            return self.datkey2filekey[datkey]
+
+        raise DatkeyNotFound(datkey + ' not found')
 
-keycache = {}
-_lock = threading.Lock()
 
 class DatkeyNotFound(Exception):
     pass
 
-def calc(filekey):
-    if '_' in filekey:
-        type, key = filekey.split('_', 1)
-    else:
-        return False
 
-    try:
-        n = int(key, 16) % 9999999999
-    except (TypeError, ValueError):
-        return False
+_datkey_table = _DatkeyTable(config.run_dir + '/datkey.txt')
 
-    datkey = str(n).rjust(10, '0')  # 10 digits
-    with _lock:
-        keycache[filekey] = datkey
-        keycache[datkey] = filekey
-    return True
 
 def get_datkey(filekey):
-    if filekey in keycache:
-        return keycache[filekey]
-
-    if not calc(filekey):
-        raise DatkeyNotFound
+    return _datkey_table.get_datkey(filekey)
 
-    return keycache[filekey]
 
 def get_filekey(datkey):
-    if datkey in keycache:
-        return keycache[datkey]
-    raise DatkeyNotFound
+    return _datkey_table.get_filekey(datkey)
+
+
+def load():
+    _datkey_table.load()
+
+    for c in cache.CacheList():
+        c.load()
+        _datkey_table.set_from_cache(c)
+
+    # Because CacheList don't list all cache
+    for rec in cache.RecentList():
+        c = cache.Cache(rec.datfile)
+        c.load()
+
+        c.recent_stamp = rec.stamp  # if cache has no records, use recent_stamp
+        _datkey_table.set_from_cache(c)
 
-def load(cache_dir):
-    for k in os.listdir(cache_dir):
-        calc(k)
+    save()
 
-def clear():
-    with _lock:
-        keycache.clear()
+def save():
+    _datkey_table.save()