Permalink
Browse files

added stats page showing top post sizes, added a blacklist and public…

… blacklist page, added a header view showing post headers, added links to feed for discovery, allowing arbitrary subpaths on a bin, wrote a cleanup handler that deletes posts older than 90 days, other minor misc
  • Loading branch information...
1 parent 9da6dce commit 7bdb8b4adcad918bafd99418054abccfb65b3e4f @progrium progrium committed Aug 7, 2010
Showing with 269 additions and 32 deletions.
  1. +6 −0 app.yaml
  2. +13 −2 bin.py
  3. +4 −0 cron.yaml
  4. +5 −0 dos.yaml
  5. +43 −4 main.py
  6. +1 −0 models.py
  7. +154 −0 static/jquery.min.js
  8. +5 −5 static/style.css
  9. +3 −1 templates/base.html
  10. +17 −19 templates/bin.html
  11. +10 −0 templates/blacklist.html
  12. +2 −1 templates/main.html
  13. +6 −0 templates/stats.html
View
@@ -6,6 +6,12 @@ api_version: 1
handlers:
- url: /
script: main.py
+- url: /stats
+ script: main.py
+- url: /tasks.*
+ script: main.py
+- url: /blacklist
+ script: main.py
- url: /favicon.ico
static_files: static/favicon.ico
upload: static/favicon.ico
View
15 bin.py
@@ -8,6 +8,9 @@
import re
import hashlib
from cgi import FieldStorage
+import logging
+
+class NotFound(Exception): pass
class BinHandler(webapp.RequestHandler):
def get(self):
@@ -44,6 +47,12 @@ def head(self):
else:
self._record_post(bin)
+ def handle_exception(self, exception, debug_mode):
+ if isinstance(exception, NotFound):
+ self.error(404)
+ else:
+ super(BinHandler, self).handle_exception(exception, debug_mode)
+
def _record_post(self, bin, use_get=False):
post = Post(bin=bin, remote_addr=self.request.remote_addr)
post.headers = dict(self.request.headers)
@@ -55,6 +64,7 @@ def _record_post(self, bin, use_get=False):
post.query_string = self.request.query_string
post.form_data = []
data_source = self.request.GET if use_get else self.request.POST
+ post.size = len(post.body) if post.body else 0
for k,v in data_source.items():
if isinstance(v, FieldStorage):
file_body = v.file.read()
@@ -64,17 +74,18 @@ def _record_post(self, bin, use_get=False):
'file_digest': hashlib.md5(file_body).hexdigest(),
'file_size': round(len(file_body) / 1024.0, 1),
}])
+ post.size += len(file_body)
else:
post.form_data.append([k,v])
post.put()
def _get_bin(self, path):
- name = path.replace('/', '')
+ name = path[1:].split('/')[0]
bin = Bin.all().filter('name =', name).get()
if bin:
return bin
else:
- self.redirect('/')
+ raise NotFound()
View
@@ -0,0 +1,4 @@
+cron:
+- description: cleanup
+ url: /tasks/cleanup
+ schedule: every 1 minutes
View
@@ -0,0 +1,5 @@
+blacklist:
+- subnet: 204.236.228.148
+ description: mftkxc - Uploading 2MB/min and pushing over quota - 22 reqs/min - Aug 7 2010
+- subnet: 67.202.28.20
+ description: tenu8x - Posts too big for datastore API - 9 reqs/min - Aug 7 2010
View
47 main.py
@@ -1,8 +1,11 @@
import wsgiref.handlers
-
-from google.appengine.ext import webapp
+import datetime
+from google.appengine.ext import webapp, db
from google.appengine.ext.webapp import template
-from models import Bin
+from models import Bin, Post
+from google.appengine.api import datastore_errors
+from google.appengine.runtime import DeadlineExceededError
+import time, yaml
class MainHandler(webapp.RequestHandler):
def get(self):
@@ -13,5 +16,41 @@ def post(self):
bin.put()
self.redirect('/%s' % bin.name)
+class StatsHandler(webapp.RequestHandler):
+ def get(self):
+ posts = Post.all().order('-size').fetch(20)
+ try:
+ current_post = None
+ for post in posts:
+ current_post = post
+ bin = post.bin
+ self.response.out.write(template.render('templates/stats.html', locals()))
+ except datastore_errors.Error, e:
+ if e.args[0] == "ReferenceProperty failed to be resolved":
+ current_post.delete()
+ self.redirect('/stats')
+
+class BlacklistHandler(webapp.RequestHandler):
+ def get(self):
+ blacklist = yaml.load(open('dos.yaml').read())
+ blacklist = [(b['subnet'], b['description'].split(' - ')) for b in blacklist['blacklist']]
+ self.response.out.write(template.render('templates/blacklist.html', locals()))
+
+class CleanupHandler(webapp.RequestHandler):
+ def get(self):
+ posts = Post.all().filter('created <', datetime.datetime.now() - datetime.timedelta(days=90))
+ assert posts.count()
+ try:
+ while True:
+ db.delete(posts.fetch(500))
+ time.sleep(0.1)
+ except DeadlineExceededError:
+ self.response.clear()
+ self.response.set_status(200)
+
if __name__ == '__main__':
- wsgiref.handlers.CGIHandler().run(webapp.WSGIApplication([('/', MainHandler)], debug=True))
+ wsgiref.handlers.CGIHandler().run(webapp.WSGIApplication([
+ ('/', MainHandler),
+ ('/stats', StatsHandler),
+ ('/tasks/cleanup', CleanupHandler),
+ ('/blacklist', BlacklistHandler)], debug=True))
View
@@ -43,6 +43,7 @@ class Post(db.Model):
query_string = db.StringProperty()
form_data = ObjectProperty()
body = db.TextProperty()
+ size = db.IntegerProperty()
#body_binary = db.BlobProperty()
def id(self):
Oops, something went wrong.

0 comments on commit 7bdb8b4

Please sign in to comment.