Skip to content

Commit

Permalink
ran the black code formatter, just to try it
Browse files Browse the repository at this point in the history
  • Loading branch information
snarfed committed Sep 4, 2021
1 parent 4d325fd commit 760fb13
Show file tree
Hide file tree
Showing 55 changed files with 19,135 additions and 16,134 deletions.
197 changes: 110 additions & 87 deletions admin.py
Expand Up @@ -15,103 +15,126 @@
from flask_app import app
import models
import util

# Import source class files so their metaclasses are initialized.
from models import BlogPost, Response, Source
import blogger, flickr, github, instagram, mastodon, medium, tumblr, twitter, wordpress_rest

NUM_ENTITIES = 10


@app.route('/admin/responses')
@app.route("/admin/responses")
def responses():
"""Find the most recently attempted responses and blog posts with error URLs."""
entities = []

for cls in (Response,): # BlogPost
for e in cls.query().order(-cls.updated):
if (len(entities) >= NUM_ENTITIES or
e.updated < datetime.datetime.now() - datetime.timedelta(hours=1)):
break
elif (not e.error and not e.unsent) or e.status == 'complete':
continue

e.links = [util.pretty_link(u, new_tab=True) for u in e.error + e.failed]
if e.key.kind() == 'Response':
e.response = json_loads(e.response_json)
e.activities = [json_loads(a) for a in e.activities_json]
else:
e.response = {'content': '[BlogPost]'}
e.activities = [{'url': e.key.id()}]

entities.append(e)

return render_template('admin_responses.html', responses=entities, logs=logs)


@app.route('/admin/sources')
"""Find the most recently attempted responses and blog posts with error URLs."""
entities = []

for cls in (Response,): # BlogPost
for e in cls.query().order(-cls.updated):
if len(
entities
) >= NUM_ENTITIES or e.updated < datetime.datetime.now() - datetime.timedelta(
hours=1
):
break
elif (not e.error and not e.unsent) or e.status == "complete":
continue

e.links = [util.pretty_link(u, new_tab=True) for u in e.error + e.failed]
if e.key.kind() == "Response":
e.response = json_loads(e.response_json)
e.activities = [json_loads(a) for a in e.activities_json]
else:
e.response = {"content": "[BlogPost]"}
e.activities = [{"url": e.key.id()}]

entities.append(e)

return render_template("admin_responses.html", responses=entities, logs=logs)


@app.route("/admin/sources")
def sources():
"""Find sources whose last poll errored out."""
CLASSES = (flickr.Flickr, github.GitHub, twitter.Twitter,
instagram.Instagram, mastodon.Mastodon)
queries = [cls.query(Source.status == 'enabled',
Source.poll_status == 'error',
Source.rate_limited.IN((False, None)),
Source.features == 'listen',
).fetch_async(NUM_ENTITIES)
for cls in CLASSES]

return render_template(
'admin_sources.html',
sources=itertools.chain(*[q.get_result() for q in queries]),
logs=logs,
)


@app.route('/admin/mark_complete', methods=['POST'])
"""Find sources whose last poll errored out."""
CLASSES = (
flickr.Flickr,
github.GitHub,
twitter.Twitter,
instagram.Instagram,
mastodon.Mastodon,
)
queries = [
cls.query(
Source.status == "enabled",
Source.poll_status == "error",
Source.rate_limited.IN((False, None)),
Source.features == "listen",
).fetch_async(NUM_ENTITIES)
for cls in CLASSES
]

return render_template(
"admin_sources.html",
sources=itertools.chain(*[q.get_result() for q in queries]),
logs=logs,
)


@app.route("/admin/mark_complete", methods=["POST"])
def mark_complete():
entities = ndb.get_multi(ndb.Key(urlsafe=u)
for u in request.values.getlist('key'))
for e in entities:
e.status = 'complete'
ndb.put_multi(entities)
return util.redirect('/admin/responses')
entities = ndb.get_multi(ndb.Key(urlsafe=u) for u in request.values.getlist("key"))
for e in entities:
e.status = "complete"
ndb.put_multi(entities)
return util.redirect("/admin/responses")


@app.route('/admin/stats')
@app.route("/admin/stats")
def stats():
"""Collect and report misc lifetime stats.
https://developers.google.com/appengine/docs/python/ndb/admin#Statistics_queries
Used to be on the front page, dropped them during the Flask port in August 2021.
"""
def count(query):
stat = query.get() # no datastore stats in dev_appserver
return stat.count if stat else 0

def kind_count(kind):
return count(KindStat.query(KindStat.kind_name == kind))

num_users = sum(kind_count(cls.__name__) for cls in models.sources.values())
link_counts = {
property: sum(count(KindPropertyNamePropertyTypeStat.query(
KindPropertyNamePropertyTypeStat.kind_name == kind,
KindPropertyNamePropertyTypeStat.property_name == property,
# specify string because there are also >2M Response entities with null
# values for some of these properties, as opposed to missing altogether,
# which we don't want to include.
KindPropertyNamePropertyTypeStat.property_type == 'String'))
for kind in ('BlogPost', 'Response'))
for property in ('sent', 'unsent', 'error', 'failed', 'skipped')}

return render_template('admin_stats.html', **{
# add comma separator between thousands
k: '{:,}'.format(v) for k, v in {
'users': num_users,
'responses': kind_count('Response'),
'links': sum(link_counts.values()),
'webmentions': link_counts['sent'] + kind_count('BlogPost'),
'publishes': kind_count('Publish'),
'blogposts': kind_count('BlogPost'),
'webmentions_received': kind_count('BlogWebmention'),
}.items()})
"""Collect and report misc lifetime stats.
https://developers.google.com/appengine/docs/python/ndb/admin#Statistics_queries
Used to be on the front page, dropped them during the Flask port in August 2021.
"""

def count(query):
stat = query.get() # no datastore stats in dev_appserver
return stat.count if stat else 0

def kind_count(kind):
return count(KindStat.query(KindStat.kind_name == kind))

num_users = sum(kind_count(cls.__name__) for cls in models.sources.values())
link_counts = {
property: sum(
count(
KindPropertyNamePropertyTypeStat.query(
KindPropertyNamePropertyTypeStat.kind_name == kind,
KindPropertyNamePropertyTypeStat.property_name == property,
# specify string because there are also >2M Response entities with null
# values for some of these properties, as opposed to missing altogether,
# which we don't want to include.
KindPropertyNamePropertyTypeStat.property_type == "String",
)
)
for kind in ("BlogPost", "Response")
)
for property in ("sent", "unsent", "error", "failed", "skipped")
}

return render_template(
"admin_stats.html",
**{
# add comma separator between thousands
k: "{:,}".format(v)
for k, v in {
"users": num_users,
"responses": kind_count("Response"),
"links": sum(link_counts.values()),
"webmentions": link_counts["sent"] + kind_count("BlogPost"),
"publishes": kind_count("Publish"),
"blogposts": kind_count("BlogPost"),
"webmentions_received": kind_count("BlogWebmention"),
}.items()
}
)
3 changes: 2 additions & 1 deletion appengine_config.py
Expand Up @@ -3,6 +3,7 @@
# Needed because I originally generated tag URIs with the current year, which
# resulted in different URIs for the same objects when the year changed. :/
from oauth_dropins.webutil import util
if not hasattr(util, '_orig_tag_uri'):

if not hasattr(util, "_orig_tag_uri"):
util._orig_tag_uri = util.tag_uri
util.tag_uri = lambda domain, name: util._orig_tag_uri(domain, name, year=2013)

0 comments on commit 760fb13

Please sign in to comment.