Permalink
Browse files

first cut of a Flask version of Temboz (bugs in the Cheetah ImportMan…

…ager

would cause Temboz to hang randomly)
  • Loading branch information...
Fazal Majid
Fazal Majid committed Jul 23, 2016
1 parent b38c4e8 commit 03409874b5d646bf694847693e90085a05dc1df1
View

Large diffs are not rendered by default.

Oops, something went wrong.
View
183 dbop.py
@@ -0,0 +1,183 @@
import sqlite3
import param
def db():
return sqlite3.connect('rss.db', 120.0)
def rebuild_v_feed_stats(c):
sql = c.execute("select sql from sqlite_master where name='v_feeds_snr'")
if sql:
c.executescript("""
drop view v_feeds_snr;
create view v_feeds_snr as
select feed_uid, feed_title, feed_html, feed_xml, feed_pubxml,
julianday('now') - last_modified as last_modified,
ifnull(interesting, 0) as interesting,
ifnull(unread, 0) as unread,
ifnull(uninteresting, 0) as uninteresting,
ifnull(filtered, 0) as filtered,
ifnull(total, 0) as total,
ifnull(snr, 0) as snr,
feed_status, feed_private, feed_exempt, feed_dupcheck, feed_errors,
feed_desc, feed_filter
from fm_feeds
left outer join mv_feed_stats on feed_uid=snr_feed_uid
group by feed_uid, feed_title, feed_html, feed_xml;""")
def snr_mv(c):
"""SQLite does not have materialized views, so we use a conventional table
instead. The side-effect of this is that new feeds may not be reflected
immediately. The SNR will also lag by up to a day, which should not matter in
practice"""
sql = c.execute("select sql from sqlite_master where name='update_stat_mv'")
if sql:
c.execute('drop trigger update_stat_mv')
sql = c.execute("select sql from sqlite_master where name='insert_stat_mv'")
if sql:
c.execute('drop trigger insert_stat_mv')
sql = c.execute("select sql from sqlite_master where name='delete_stat_mv'")
if sql:
c.execute('drop trigger delete_stat_mv')
sql = c.execute("select sql from sqlite_master where name='insert_feed_mv'")
if sql:
c.execute('drop trigger insert_feed_mv')
sql = c.execute("select sql from sqlite_master where name='delete_feed_mv'")
if sql:
c.execute('drop trigger delete_feed_mv')
sql = c.execute("select sql from sqlite_master where name='mv_feed_stats'")
if sql:
c.execute('drop table mv_feed_stats')
c.execute("""create table mv_feed_stats (
snr_feed_uid integer primary key,
interesting integer default 0,
unread integer default 0,
uninteresting integer default 0,
filtered integer default 0,
total integer default 0,
last_modified timestamp,
snr real default 0.0)""")
c.execute("""insert into mv_feed_stats
select feed_uid,
sum(case when item_rating=1 then 1 else 0 end),
sum(case when item_rating=0 then 1 else 0 end),
sum(case when item_rating=-1 then 1 else 0 end),
sum(case when item_rating=-2 then 1 else 0 end),
sum(1),
max(item_modified),
snr_decay(item_rating, item_created, ?)
from fm_feeds left outer join (
select item_rating, item_feed_uid, item_created,
ifnull(
julianday(item_modified),
julianday(item_created)
) as item_modified
from fm_items
) on feed_uid=item_feed_uid
group by feed_uid, feed_title, feed_html, feed_xml""",
[getattr(param, 'decay', 30)])
rebuild_v_feed_stats(c)
c.executescript("""
create trigger update_stat_mv after update on fm_items
begin
update mv_feed_stats set
interesting = interesting
+ case new.item_rating when 1 then 1 else 0 end
- case old.item_rating when 1 then 1 else 0 end,
unread = unread
+ case new.item_rating when 0 then 1 else 0 end
- case old.item_rating when 0 then 1 else 0 end,
uninteresting = uninteresting
+ case new.item_rating when -1 then 1 else 0 end
- case old.item_rating when -1 then 1 else 0 end,
filtered = filtered
+ case new.item_rating when -2 then 1 else 0 end
- case old.item_rating when -2 then 1 else 0 end,
last_modified = max(ifnull(last_modified, 0),
ifnull(julianday(new.item_modified),
julianday(new.item_created)))
where snr_feed_uid=new.item_feed_uid;
end;
create trigger insert_stat_mv after insert on fm_items
begin
update mv_feed_stats set
interesting = interesting
+ case new.item_rating when 1 then 1 else 0 end,
unread = unread
+ case new.item_rating when 0 then 1 else 0 end,
uninteresting = uninteresting
+ case new.item_rating when -1 then 1 else 0 end,
filtered = filtered
+ case new.item_rating when -2 then 1 else 0 end,
total = total + 1,
last_modified = max(ifnull(last_modified, 0),
ifnull(julianday(new.item_modified),
julianday(new.item_created)))
where snr_feed_uid=new.item_feed_uid;
end;""")
# XXX there is a possibility last_modified will not be updated if we purge
# XXX the most recent item. There are no use cases where this could happen
# XXX since garbage-collection works from the oldest item, and purge-reload
# XXX will reload the item anyway
c.executescript("""
create trigger delete_stat_mv after delete on fm_items
begin
update mv_feed_stats set
interesting = interesting
- case old.item_rating when 1 then 1 else 0 end,
unread = unread
- case old.item_rating when 0 then 1 else 0 end,
uninteresting = uninteresting
- case old.item_rating when -1 then 1 else 0 end,
filtered = filtered
- case old.item_rating when -2 then 1 else 0 end,
total = total - 1
where snr_feed_uid=old.item_feed_uid;
end;
create trigger insert_feed_mv after insert on fm_feeds
begin
insert into mv_feed_stats (snr_feed_uid) values (new.feed_uid);
end;
create trigger delete_feed_mv after delete on fm_feeds
begin
delete from mv_feed_stats
where snr_feed_uid=old.feed_uid;
end;""")
c.commit()
def mv_on_demand(c):
"""creating the materialized view is not more expensive than running the
slow full table scan way, so we do so on demand (rather than at startup)"""
sql = c.execute("select sql from sqlite_master where name='mv_feed_stats'")
if not sql:
print >> param.log, 'WARNING: rebuilding mv_feed_stats...',
snr_mv(c)
print >> param.log, 'done'
def view_sql(c, where, sort, params, overload_threshold):
mv_on_demand(c)
c.execute('drop table if exists articles')
c.execute("""create table articles as select
item_uid, item_creator, item_title, item_link, item_content,
datetime(item_loaded), date(item_created) as item_created,
datetime(item_rated) as item_rated,
julianday('now') - julianday(item_created) as delta_created, item_rating,
item_rule_uid, item_feed_uid, feed_title, feed_html, feed_xml, snr
from fm_items, v_feeds_snr
where item_feed_uid=feed_uid and """ + where + """
order by """ + sort + """, item_uid DESC limit ?""",
params + [overload_threshold])
c.execute("""create index articles_i on articles(item_uid)""")
tag_dict = dict()
for item_uid, tag_name, tag_by in c.execute(
"""select tag_item_uid, tag_name, tag_by
from fm_tags, articles where tag_item_uid=item_uid"""):
tag_dict.setdefault(item_uid, []).append(tag_name)
return tag_dict, c.execute("""select * from articles
order by """ + sort + """, item_uid DESC""")
c = db()
mv_on_demand(c)
rebuild_v_feed_stats(c)
c.commit()
c.close()
View
@@ -215,18 +215,18 @@ def highlight_title(self, html):
def highlight_content(self, html):
return html + '<br><p>Filtered by Python rule %d</p>' % self.uid
def load_rules(db, c):
def load_rules(c):
global loaded, rules, feed_rules
if loaded: return
rules = []
feed_rules = dict()
try:
try:
c.execute("""select rule_uid, rule_type, rule_text, rule_feed_uid,
strftime('%s', rule_expires)
from fm_rules
where rule_expires is null or rule_expires > julianday('now')""")
for uid, rtype, rule, feed_uid, expires in c:
for uid, rtype, rule, feed_uid, expires in \
c.execute("""select rule_uid, rule_type, rule_text, rule_feed_uid,
strftime('%s', rule_expires)
from fm_rules
where rule_expires is null or rule_expires > julianday('now')"""):
if expires: expires = int(expires)
if feed_uid:
container = feed_rules.setdefault(feed_uid, list())
@@ -250,9 +250,9 @@ def load_rules(db, c):
uid, expires, rule, rtype.replace('union_', 'content_')))
else:
container.append(KeywordRule(uid, expires, rule, rtype))
c.execute("""select feed_uid, feed_filter from fm_feeds
where feed_filter is not null""")
for feed_uid, rule in c:
for feed_uid, rule in \
c.execute("""select feed_uid, feed_filter from fm_feeds
where feed_filter is not null"""):
rule = PythonRule('feed_%d' % feed_uid, None, rule)
feed_rules.setdefault(feed_uid, list()).append(rule)
except:
View
@@ -420,7 +420,7 @@ def http_error_302(self, req, fp, code, msg, headers):
raise Redirect(code, newurl)
http_error_301 = http_error_303 = http_error_307 = http_error_302
redirect_opener = urllib2.build_opener(DontHandleRedirect)
socket.setdefaulttimeout(10)
#socket.setdefaulttimeout(10)
def dereference(url, seen=None, level=0):
"""Recursively dereference a URL"""
Oops, something went wrong.

0 comments on commit 0340987

Please sign in to comment.