Rate limit new listings and inquiries.

caravel/__init__.py

@@ -22,7 +22,7 @@
     testbed.activate()
     testbed.init_all_stubs()
     app.testing = True
-    
+
 # Ensure that the Recaptcha field is small.
 app.config["RECAPTCHA_DATA_ATTRS"] = {"size": "compact"}
 
@@ -31,7 +31,12 @@
 CsrfProtect(app)
 
 # Imported for side effects:
-from caravel import app, model, utils
-from caravel.storage import config #, photos
-from caravel.controllers import listings, api, moderation
-from caravel.daemons import migration, delete_old_photos, nag_moderators
+import caravel.model
+import caravel.utils
+import caravel.storage.config
+import caravel.controllers.listings
+import caravel.controllers.api
+import caravel.controllers.moderation
+import caravel.daemons.migration
+import caravel.daemons.delete_old_photos
+import caravel.daemons.nag_moderators

caravel/model/inquiry.py

@@ -1,18 +1,25 @@
 from google.appengine.ext import ndb
+
+from caravel import utils
 from caravel.model import listing
+
 from caravel.model.moderation import ModeratedMixin
 from caravel.model.temporal import TimeOrderMixin
 from caravel.model.principal import PrincipalMixin
 from caravel.model.side_effects import SideEffectsMixin
-from caravel import utils
+from caravel.model.rate_limits import RateLimitMixin
 
 from flask import render_template
 
-class _Inquiry(TimeOrderMixin, PrincipalMixin, ModeratedMixin, ndb.Model):
+
+class _Inquiry(TimeOrderMixin, PrincipalMixin, ModeratedMixin,
+               ndb.Model):
     message = ndb.StringProperty()
     listing = ndb.KeyProperty(kind=listing.Listing)
 
+
 class Inquiry(SideEffectsMixin, _Inquiry):
+
     def side_effects(self):
         """
         Sends an email to the owner of this listing letting them know of the
@@ -25,10 +32,12 @@ def side_effects(self):
             reply_to=self.principal,
             subject=u"Re: Marketplace Listing \"{}\"".format(listing.title),
             html=render_template("email/inquiry.html",
-                    listing=listing, inquiry=self),
+                                 listing=listing, inquiry=self),
             text=render_template("email/inquiry.txt",
-                    listing=listing, inquiry=self)
+                                 listing=listing, inquiry=self)
         )
 
+
 class UnapprovedInquiry(_Inquiry):
+    MAX_DAILY_LIMIT = 5
     TYPE_ONCE_APPROVED = Inquiry

caravel/model/listing.py

@@ -9,21 +9,27 @@
 from caravel.model.principal import PrincipalMixin
 from caravel.model.side_effects import SideEffectsMixin
 from caravel.model.full_text import FullTextMixin
+from caravel.model.rate_limits import RateLimitMixin
+
 from caravel import utils
 
 import datetime
 import re
 from flask import render_template
 
+
 class _Listing(CategoriesMixin, PhotosMixin, PrincipalMixin, TimeOrderMixin,
-               SchemaMixin, PriceMixin, ModeratedMixin, ndb.Model):
+               SchemaMixin, PriceMixin, RateLimitMixin, ModeratedMixin,
+               ndb.Model):
 
     SCHEMA_VERSION = 11
 
     title = ndb.StringProperty()
     body = ndb.TextProperty()
 
+
 class Listing(SideEffectsMixin, FullTextMixin, _Listing):
+
     def side_effects(self):
         """
         Sends an email to the creator of this listing.
@@ -35,7 +41,7 @@ def side_effects(self):
             html=render_template("email/listing_verified.html", listing=self),
             text=render_template("email/listing_verified.txt", listing=self)
         )
-    
+
     def _keywords(self):
         """Generates keywords for this listing."""
 
@@ -48,9 +54,12 @@ def _keywords(self):
             keywords.append("price:free")
         return keywords
 
+
 class UnapprovedListing(_Listing):
+    MAX_DAILY_LIMIT = 4
     TYPE_ONCE_APPROVED = Listing
 
+
 @Listing.migration(to_version=11)
 def to_ndb_schema(listing):
     if hasattr(listing, "details"):
@@ -72,7 +81,7 @@ def to_ndb_schema(listing):
         listing.posted_at = datetime.datetime(month=9, day=15, year=2015)
 
     listing.categories = [re.sub(r'^category:', '', x) for x in
-                             listing.categories]
+                          listing.categories]
 
     listing.photos = [
         Photo(re.sub(r'-large$', '', p.path)) for p in listing.photos]

caravel/model/migration.py

@@ -1,17 +1,19 @@
 from google.appengine.ext import ndb
 
+
 class SchemaMixin(ndb.Expando):
+
     """
     A SchemaMixin tracks changes to the schema of an entity.
-    
+
     >>> class F(SchemaMixin):
     ...   SCHEMA_VERSION = 1
     >>> x = F()
-    
+
     >>> @F.migration(to_version=2)
     ... def bump(ent): print "bumping version for " + ent.__class__.__name__
     >>> F.SCHEMA_VERSION = 2
-    
+
     >>> y = x.put().get()
     bumping version for F
     >>> y.version
@@ -62,4 +64,3 @@ def _from_pb(klass, *vargs, **kwargs):
             entity.version = 0
         entity.run_migrations()
         return entity
-

caravel/model/rate_limits.py

@@ -0,0 +1,74 @@
+from google.appengine.ext import ndb
+from caravel.storage import dos
+
+
+class RateLimitMixin(ndb.Model):
+
+    """
+    A RateLimitMixin ensures that the given model will only be created a fixed
+    number of times per minute. If the rate limit is violated, the entity is
+    treated as though the principal creating it is unviolated.
+
+    >>> from caravel.model.moderation import ModeratedMixin
+    >>> from caravel.model.principal import PrincipalMixin
+    >>> from caravel.utils import Principal, Device
+
+    >>> class G(RateLimitMixin, ModeratedMixin, PrincipalMixin):
+    ...   MAX_BURST_LIMIT = 2
+
+    >>> device = Device('', '', '')
+    >>> person_a = Principal('rlt-foo@uchicago.edu', device, 'GOOGLE_APPS')
+    >>> person_b = Principal('rlt-bar@uchicago.edu', device, 'GOOGLE_APPS')
+
+    >>> a = G(principal=person_a); a.put() and a.approved()
+    True
+    >>> a = G(principal=person_a); a.put() and a.approved()
+    True
+    >>> a = G(principal=person_a); a.put() and a.approved()
+    False
+
+    >>> b = G(principal=person_b); b.put() and b.approved()
+    True
+    """
+
+    MAX_BURST_LIMIT = 0
+    MAX_DAILY_LIMIT = 0
+
+    burst_count = ndb.IntegerProperty()
+    daily_count = ndb.IntegerProperty()
+
+    def _pre_put_hook(self):
+        """
+        Compute the current rate and store it in the burst/daily limits.
+        """
+
+        if not self.burst_count:
+            self.burst_count = dos.current_rate(self.principal.email,
+                                                self.MAX_BURST_LIMIT, 60)
+
+        if not self.daily_count:
+            self.daily_count = dos.current_rate(self.principal.email,
+                                                self.MAX_DAILY_LIMIT,
+                                                3600 * 24)
+
+        return super(RateLimitMixin, self)._pre_put_hook()
+
+    def approved(self):
+        """
+        Ensure that listings must be manually approved.
+        """
+
+        if super(RateLimitMixin, self).approved():
+            # Allow manual approval.
+            if self.principal.validated_by:
+                return True
+
+            if self.MAX_BURST_LIMIT and self.burst_count > self.MAX_BURST_LIMIT:
+                return False
+
+            if self.MAX_DAILY_LIMIT and self.burst_count > self.MAX_DAILY_LIMIT:
+                return False
+
+            return True
+
+        return False

caravel/storage/dos.py

@@ -4,6 +4,18 @@
 
 
 def current_rate(entity, limit, duration):
+    """
+    Stores a counter with the given name. This function reports rate limit
+    denials based on the given limit.
+
+    >>> current_rate('rlt_test', 2, 60)
+    1L
+    >>> current_rate('rlt_test', 2, 60)
+    2L
+    >>> current_rate('rlt_test', 2, 60)
+    3L
+    """
+
     key = "ratelimit:{}:{}".format(int(time.time() / duration), entity)
     value = memcache.incr(key, initial_value=0)
     if value > limit:
@@ -13,8 +25,20 @@ def current_rate(entity, limit, duration):
     else:
         logging.info(
             "RateLimitAllowed({!r}, value={!r}, limit={!r}, duration={!r})"
-                .format(entity, value, limit, duration))
+            .format(entity, value, limit, duration))
     return value
 
+
 def rate_limit(entity, limit, duration=60):
+    """
+    Runs a rate limit with the given duration.
+
+    >>> rate_limit('rlt_test2', 2)
+    False
+    >>> rate_limit('rlt_test2', 2)
+    False
+    >>> rate_limit('rlt_test2', 2)
+    True
+    """
+
     return current_rate(entity, limit, duration) > limit

caravel/utils/emails.py

@@ -6,10 +6,10 @@
 import logging
 
 from caravel.utils import principals
-from caravel.storage import config
 
 SENDER = "Marketplace Team <marketplace@lists.uchicago.edu>"
 
+
 def send_mail(to, subject, html, text, reply_to=None, sender=SENDER):
     """
     Sends an email to the given principals.
@@ -23,7 +23,7 @@ def send_mail(to, subject, html, text, reply_to=None, sender=SENDER):
     if reply_to:
         if not (isinstance(reply_to, principals.Principal) and reply_to.valid):
             raise ValueError("{!r} has not consented to send email."
-                .format(reply_to))
+                             .format(reply_to))
 
     # Actually send the message to the user.
     _send_raw_mail(
@@ -35,9 +35,12 @@ def send_mail(to, subject, html, text, reply_to=None, sender=SENDER):
         sender=sender
     )
 
+
 def _send_raw_mail(to, subject, html, text, reply_to=None, sender=SENDER):
     logging.debug("SendMail(to={!r}, subject={!r})".format(to, subject))
 
+    from caravel.storage import config  # prevent import cycle
+
     email = sendgrid.Mail()
     email.set_from(sender)
     email.add_to(to)

caravel/utils/principals.py

@@ -71,7 +71,7 @@ def explain(self):
         if self.auth_method in (self.GOOGLE_APPS, self.LEGACY):
             return "Validated by {}".format(self.auth_method)
         else:
-            return self.validated_by 
+            return self.validated_by
 
     def __repr__(self):
         """