Skip to content
Permalink
Browse files

first working Python 3 version. Still needs a lot more testing

and dogfooding
  • Loading branch information
fazalmajid committed May 22, 2019
1 parent 1bb9e8c commit 0662a537ad68fd327fcb82eeda47d74d6fe79b46
Showing with 249 additions and 4,217 deletions.
  1. +20 −0 Dockerfile
  2. +6 −0 Makefile
  3. +7 −7 etc/norm_url.py
  4. +2 −2 etc/resolveguid.py
  5. +1 −0 setup.py
  6. +11 −5 temboz
  7. +47 −20 tembozapp/bootstrap.py
  8. +7 −6 tembozapp/dbop.py
  9. +3 −3 tembozapp/degunk.py
  10. +0 −3,987 tembozapp/feedparser.py
  11. +7 −6 tembozapp/filters.py
  12. +24 −41 tembozapp/normalize.py
  13. +7 −7 tembozapp/opml.py
  14. +52 −34 tembozapp/server.py
  15. +0 −49 tembozapp/social.py
  16. +46 −43 tembozapp/update.py
  17. +9 −7 tembozapp/util.py
@@ -0,0 +1,20 @@
FROM alpine
RUN apk add --update python3-dev py3-cffi gcc linux-headers musl-dev sqlite
RUN python3 -m ensurepip
RUN pip3 install --upgrade pip
RUN pip3 install flask
RUN pip3 install requests
RUN pip3 install html5lib
RUN pip3 install passlib
RUN pip3 install argon2_cffi
RUN pip3 install translitcodec
RUN pip3 install waitress
RUN pip3 install feedparser
#RUN pip3 install yappi
COPY . /temboz
RUN rm -f /temboz/tembozapp/feedparser.py
VOLUME ["/temboz/data"]
WORKDIR /temboz/data
ENV DOCKER=true
EXPOSE 9999/tcp
ENTRYPOINT ["python3", "-v", "/temboz/temboz", "--server"]
@@ -65,6 +65,12 @@ opml:
wget -c https://majid.info/temboz/temboz.opml
mv temboz.opml me.opml

docker:
docker build -t fazalmajid/temboz .

docker-run:
docker run -p 9999:9999 -v `pwd`:/temboz/data fazalmajid/temboz

sdist:
python setup.py sdist

@@ -1,13 +1,13 @@
import sys, os, threading, Queue
import sys, os, threading, queue
sys.path.append('.')
os.chdir('..')
import normalize
from singleton import db

num_workers = 64

in_q = Queue.Queue()
out_q = Queue.Queue()
in_q = queue.Queue()
out_q = queue.Queue()
class Worker(threading.Thread):
def run(self):
while True:
@@ -30,8 +30,8 @@ def run(self):
from fm_items
where item_rating>0
order by item_uid""")
map(in_q.put, c)
map(in_q.put, [(None, None)] * num_workers)
list(map(in_q.put, c))
list(map(in_q.put, [(None, None)] * num_workers))


while True:
@@ -42,7 +42,7 @@ def run(self):
db.commit()
sys.exit(0)
continue
print uid, url
print '\t==>', new_url
print(uid, url)
print('\t==>', new_url)
c.execute('update fm_items set item_link=? where item_uid=?',
[new_url, uid])
@@ -18,8 +18,8 @@ def escape(str):
ll = [x[0] for x in ll]
assert uid in ll
if len(ll) > 2:
print 'could not resolve link', link,
print 'more than 2 instances:', ', '.join(map(str, ll))
print('could not resolve link', link, end=' ')
print('more than 2 instances:', ', '.join(map(str, ll)))
continue
if len(ll) < 2: continue
ll.remove(uid)
@@ -17,6 +17,7 @@
'passlib',
'argon2_cffi',
'translitcodec',
'feedparser',
'yappi'
],
description='The Temboz RSS/Atom feed reader and aggregator.',
16 temboz
@@ -1,16 +1,17 @@
#!/usr/bin/env python
from __future__ import print_function
import sys, os, datetime

def usage():
print """usage: %s [options]
print("""usage: %s [options]
--refresh : refresh the subscribed feeds
--import <opml_file> : import subscriptions from a OPML file
--server : run the built-in web server
--kill : kill the server
--cleanup : perform database housekeeping and cleanup
-v : be more verbose (can be repeated)
-h, --help : print this help text
""" % sys.argv[0]
""" % sys.argv[0])
sys.exit(0)

if __name__ == '__main__':
@@ -20,12 +21,17 @@ if __name__ == '__main__':
os.stat('rss.db')
except:
import tembozapp.bootstrap
tembozapp.bootstrap.do_bootstrap()
if os.getenv('DOCKER') == 'true':
tembozapp.bootstrap.docker_bootstrap()
else:
tembozapp.bootstrap.interactive_bootstrap()
import tembozapp.transform
try:
import tembozapp.transform
except ImportError:
print >> sys.stderr, 'You must create a transform.py file, possibly using',
print >> sys.stderr, 'transform.py.sample as an example'
print('You must create a transform.py file, possibly using',
end=' ', file=sys.stderr)
print('transform.py.sample as an example', file=sys.stderr)
sys.exit(0)
import tembozapp.util
opts, args = getopt.getopt(
@@ -1,13 +1,13 @@
import sys, os, socket, string, getpass, passlib.hash
import sys, os, socket, string, shutil, getpass, passlib.hash

def do_bootstrap():
def interactive_bootstrap():
dir = os.path.dirname(__file__ or os.getcwd())
dir = os.getcwd() + os.sep + 'tempip' if dir == '.' else dir
print """Welcome to the Temboz initial setup wizard!
"""
print("""Welcome to the Temboz initial setup wizard!
""")
ip, port = None, None
while not ip or not port:
bind = raw_input(
bind = input(
"""What IP address and TCP port should the server run on?
Choose 127.0.0.1 to only allow connections from this machine (default)
Choose 0.0.0.0 to allow connections from outside machines
@@ -19,37 +19,39 @@ def do_bootstrap():
# IPv6 addresses can have colons too
ip, port_s = bind.rsplit(':', 1)
except ValueError:
print >> sys.stderr, 'Invalid bind specification', bind,
print >> sys.stderr, '- it should be a of the form <IP>:<port>.'
print('Invalid bind specification', bind, end=' ', file=sys.stderr)
print('- it should be a of the form <IP>:<port>.', file=sys.stderr)
continue
try:
port = int(port_s)
except ValueError:
print >> sys.stderr, 'Invalid port number', port_s,
print >> sys.stderr, '- it should be a number between 1 and 65535.'
print('Invalid port number', port_s, end=' ', file=sys.stderr)
print('- it should be a number between 1 and 65535.', file=sys.stderr)
continue
if port < 1 or port > 65535:
print >> sys.stderr, 'Invalid port number:', port,
print >> sys.stderr, '- it should be a number between 1 and 65535.'
print('Invalid port number:', port, end=' ', file=sys.stderr)
print('- it should be a number between 1 and 65535.', file=sys.stderr)
port = None
try:
s = socket.socket()
s.bind((ip, port))
s.close()
except socket.error as e:
print >> sys.stderr, 'Cannot bind to', bind, '-', str(e)
print('Cannot bind to', bind, '-', str(e), file=sys.stderr)
ip, port = None, None
continue

login = None
while not login:
login = raw_input(
login = input(
'Choose a username: ')
login = login.strip()
if not set(login).issubset(set(string.letters + string.digits + '_.')):
print >> sys.stderr, 'Invalid username', login,
print >> sys.stderr, '- it should only have alphanumeric characters,',
print >> sys.stderr, 'underscore or dot'
if not set(login).issubset(set(
string.ascii_letters + string.digits + '_.'
)):
print('Invalid username', login, end=' ', file=sys.stderr)
print('- it should only have alphanumeric characters,', end=' ', file=sys.stderr)
print('underscore or dot', file=sys.stderr)
login = None
continue

@@ -60,11 +62,11 @@ def do_bootstrap():
while not passwd:
passwd = getpass.getpass('Enter password: ')
if len(passwd) < 8:
print >> sys.stderr, 'The password must have at least 8 characters'
print('The password must have at least 8 characters', file=sys.stderr)
passwd = None
continue
if passwd != getpass.getpass('Confirm password: '):
print >> sys.stderr, 'The passwords do not match'
print('The passwords do not match', file=sys.stderr)
passwd = None
continue
hash = passlib.hash.argon2.using(
@@ -74,9 +76,34 @@ def do_bootstrap():
digest_size=32).hash(passwd)

os.system('sqlite3 rss.db < %s/ddl.sql' % dir)
import dbop
from . import dbop
with dbop.db() as db:
dbop.setting(db, 'login', login)
dbop.setting(db, 'passwd', hash)
dbop.setting(db, 'ip', ip)
dbop.setting(db, 'port', str(port))


def docker_bootstrap():
ip, port = '127.0.0.1', 9999
login = 'temboz'
hash = passlib.hash.argon2.using(
rounds=64,
memory_cost=65536,
parallelism=1,
digest_size=32).hash('temboz')

dir = os.path.dirname(__file__ or os.getcwd())
assert dir == '/temboz/tembozapp'
os.system('sqlite3 rss.db < %s/ddl.sql' % dir)
from . import dbop
with dbop.db() as db:
dbop.setting(db, 'login', login)
dbop.setting(db, 'passwd', hash)
dbop.setting(db, 'ip', ip)
dbop.setting(db, 'port', str(port))
print("""\033[1;34mInitialized with defaults for docker:
URL: http://localhost:9999/
login: temboz
password: temboz
\033[0m\n""")
@@ -1,5 +1,6 @@
from __future__ import print_function
import sys, time, sqlite3, string, json
import param
from . import param

def db():
conn = sqlite3.connect('rss.db', 60.0)
@@ -168,15 +169,15 @@ def mv_on_demand(db):
sql = c.execute("select sql from sqlite_master where name='mv_feed_stats'")
status = c.fetchone()
if not status:
print >> param.log, 'WARNING: rebuilding mv_feed_stats...',
print('WARNING: rebuilding mv_feed_stats...', end=' ', file=param.log)
snr_mv(db, c)
db.commit()
print >> param.log, 'done'
print('done', file=param.log)
c.close()

def elapsed(t, what):
t2 = time.time()
print >> param.log, what, (t2-t)* 1000, 'ms'
print(what, (t2-t)* 1000, 'ms', file=param.log)
return t2

use_json = None
@@ -348,12 +349,12 @@ def item(db, uid):

def setting(db, *args, **kwargs):
c = db.cursor()
for name, value in zip(args[::2], args[1::2]) + kwargs.items():
for name, value in list(zip(args[::2], args[1::2])) + list(kwargs.items()):
param.settings[name] = str(value)
try:
c.execute("insert into fm_settings (name, value) values (?, ?)",
[name, str(value)])
except sqlite3.IntegrityError, e:
except sqlite3.IntegrityError as e:
c.execute("update fm_settings set value=? where name=?",
[value, name])
db.commit()
@@ -1,8 +1,8 @@
# this module defines classes that can be used to massage the content of an
# article, mostly to remove gunk like ads

from __future__ import print_function
import sys, re, requests, sqlite3
import param, util, dbop
from . import param, util, dbop

class Filter:
"""Virtual class with the interface for all degunking filters"""
@@ -125,7 +125,7 @@ def apply(self, content, *args, **kwargs):
link = c.fetchone()
c.close()
if link:
print >> param.log, 'not dereferencing', guid, '->', link[0]
print('not dereferencing', guid, '->', link[0], file=param.log)
item['link'] = link[0]
return content
# we haven't seen this article before, buck up and load it

0 comments on commit 0662a53

Please sign in to comment.
You can’t perform that action at this time.