February 2011 Merge

* Upgrade discount to 1.6.8
* Start reading Votes, Saves, and Hides from Cassandra (while still dual-writing all three)
* Some XSS fixes
* Significant Gold buying and gifting improvements
  - Move /api/ipn to /ipn
* Allow non-US countries to buy sponsored links
* Increase embed.ly scope
* redd.it support
* Allow postgres port number to be specified in ini file (this changes the format of the .ini file)
* Upgrade Cassandra to 0.7
  - Change g.urlcache to LinksByURL
  - Translate storage-conf.xml to cassandra.yaml
  - TTL support (and enable on Hides)
  - Move permacache keyspace to inside reddit keyspace
* The stalecache: a local memcached that contains slightly old information to speed up some lookups
* Switch to patched Paste that is hopefully download.gz-proof
* Don't store votes on things > 30 days old
* Many many bugfixes/small features

.gitignore

@@ -35,6 +35,7 @@ r2/_builder.egg-info/
 r2/_normalized_hot.egg-info/
 r2/_sorts.egg-info/
 r2/r2/lib/_normalized_hot.c
+r2/r2/lib/mr_tools/_mr_tools.c
 r2/r2/lib/db/_sorts.c
 r2/r2/lib/sgm.c
 r2/r2/lib/utils/_utils.c

r2/example.ini

@@ -32,9 +32,11 @@ MODSECRET = abcdefghijklmnopqrstuvwxyz0123456789
 # secret for /prefs/feeds
 FEEDSECRET = abcdefghijklmnopqrstuvwxyz0123456789
 
+INDEXTANK_API_URL =
+
 # -- important settings -- 
 # the domain that this app serves itself up as
-domain = localhost
+domain = reddit.local
 # if you use www for the old-timey feel, put it here
 domain_prefix = 
 # the user used for "system" operations and messages
@@ -80,6 +82,7 @@ admin_message_acct = reddit
 # data cache (used for caching Thing objects)
 num_mc_clients = 5
 memcaches = 127.0.0.1:11211
+stalecaches =
 # render caches (the second is "remote" and the local is optional but in the same format)
 local_rendercache =
 rendercaches = 127.0.0.1:11211
@@ -94,7 +97,7 @@ permacache_memcaches = 127.0.0.1:11211
 cassandra_seeds = 127.0.0.1:9160
 # read/write consistency levels for Cassandra
 cassandra_rcl = ONE
-cassandra_wcl = QUORUM
+cassandra_wcl = ONE
 
 # -- url cache options --
 url_caches = 127.0.0.1:11211
@@ -117,6 +120,8 @@ adtracker_url = /static/pixel.png
 adframetracker_url = http://pixel.reddit.com/pixel/of_defenestration.png
 # open redirector to bounce clicks off of on sponsored links for tracking
 clicktracker_url = /static/pixel.png
+# new pixel
+newtracker_url = 
 
 # amqp
 amqp_host = localhost:5672
@@ -128,17 +133,23 @@ amqp_virtual_host = /
 # list of all databases named in the subsequent table
 databases = main, comment, vote, email, authorize, award, hc
 
-#db name         db           host       user, pass
-main_db =        reddit,   127.0.0.1, reddit,   password
-comment_db =     reddit,   127.0.0.1, reddit,   password
-comment2_db =    reddit,   127.0.0.1, reddit,   password
-vote_db =        reddit,   127.0.0.1, reddit,   password
-email_db =       reddit,   127.0.0.1, reddit,   password
-authorize_db =   reddit,   127.0.0.1, reddit,   password
-award_db =       reddit,   127.0.0.1, reddit,   password
-hc_db =          reddit,   127.0.0.1, reddit,   password
+db_user = reddit
+db_pass = password
+db_port = 5432
+db_pool_size = 3
+db_pool_overflow_size = 3
+
+#db name       db         host      user, pass, port, conn, overflow_conn
+main_db =      reddit,   127.0.0.1, *,    *,    *,    *,    *
+comment_db =   reddit,   127.0.0.1, *,    *,    *,    *,    *
+comment2_db =  reddit,   127.0.0.1, *,    *,    *,    *,    *
+vote_db =      reddit,   127.0.0.1, *,    *,    *,    *,    *
+email_db =     reddit,   127.0.0.1, *,    *,    *,    *,    *
+authorize_db = reddit,   127.0.0.1, *,    *,    *,    *,    *
+award_db =     reddit,   127.0.0.1, *,    *,    *,    *,    *
+hc_db =        reddit,   127.0.0.1, *,    *,    *,    *,    *
 
-hardcache_categories = *:hc
+hardcache_categories = *:hc:hc
 
 # this setting will prefix all of the table names
 db_app_name = reddit
@@ -197,13 +208,17 @@ tracking_secret = abcdefghijklmnopqrstuvwxyz0123456789
 
 ## -- Self-service sponsored link stuff --
 # (secure) payment domain
-payment_domain = http://pay.localhost/
-ad_domain = http://localhost
+payment_domain = http://reddit.local/
+ad_domain = http://reddit.local
+allowed_pay_countries = United States, United Kingdom, Canada
 sponsors =
-# authorize.net credentials
+
+# authorize.net credentials (blank authorizenetapi to disable)
+authorizenetapi =
+# authorizenetapi = https://api.authorize.net/xml/v1/request.api
 authorizenetname = 
 authorizenetkey = 
-authorizenetapi = https://api.authorize.net/xml/v1/request.api
+
 min_promote_bid = 20
 max_promote_bid = 9999
 min_promote_future = 2
@@ -227,7 +242,7 @@ authorized_cnames =
 num_query_queue_workers = 5
 query_queue_worker = http://cslowe.local:8000
 enable_doquery = True
-use_query_cache = False
+use_query_cache = True
 write_query_queue = True
 
 # -- stylesheet editor --
@@ -243,6 +258,9 @@ stylesheet_rtl = reddit-rtl.css
 # location of the static directory
 static_path = /static/
 
+# make frontpage 100% dart
+frontpage_dart = false
+
 #  -- translator UI -- 
 # enable/disable access to the translation UI in /admin/i18n
 translator = true

r2/r2/config/middleware.py

@@ -255,7 +255,7 @@ def filter(self, execution_func, prof_arg = None):
         return [res]
 
 class DomainMiddleware(object):
-    lang_re = re.compile(r"^\w\w(-\w\w)?$")
+    lang_re = re.compile(r"\A\w\w(-\w\w)?\Z")
 
     def __init__(self, app):
         self.app = app
@@ -371,7 +371,7 @@ def __call__(self, environ, start_response):
         return self.app(environ, start_response)
 
 class DomainListingMiddleware(object):
-    domain_pattern = re.compile(r'^/domain/(([-\w]+\.)+[\w]+)')
+    domain_pattern = re.compile(r'\A/domain/(([-\w]+\.)+[\w]+)')
 
     def __init__(self, app):
         self.app = app
@@ -386,7 +386,7 @@ def __call__(self, environ, start_response):
         return self.app(environ, start_response)
 
 class ExtensionMiddleware(object):
-    ext_pattern = re.compile(r'\.([^/]+)$')
+    ext_pattern = re.compile(r'\.([^/]+)\Z')
 
     extensions = (('rss' , ('xml', 'text/xml; charset=UTF-8')),
                   ('xml' , ('xml', 'text/xml; charset=UTF-8')),

r2/r2/config/rewrites.py

@@ -23,9 +23,9 @@
 
 rewrites = (#these first two rules prevent the .embed rewrite from
             #breaking other js that should work
-            ("^/_(.*)", "/_$1"),
-            ("^/static/(.*\.js)", "/static/$1"),
+            ("\A/_(.*)", "/_$1"),
+            ("\A/static/(.*\.js)", "/static/$1"),
             #This next rewrite makes it so that all the embed stuff works.
-            ("^(.*)(?<!button)(?<!buttonlite)(\.js)$", "$1.embed"))
+            ("\A(.*)(?<!button)(?<!buttonlite)(\.js)\Z", "$1.embed"))
 
 rewrites = tuple((re.compile(r[0]), r[1]) for r in rewrites)

r2/r2/config/routing.py

@@ -173,6 +173,8 @@ def make_map(global_conf={}, app_conf={}):
     mc('/thanks', controller='forms', action="thanks", secret = '')
     mc('/thanks/:secret', controller='forms', action="thanks")
 
+    mc('/gold', controller='forms', action="gold")
+
     mc('/password', controller='forms', action="password")
     mc('/:action', controller='front',
        requirements=dict(action="random|framebuster|selfserviceoatmeal"))
@@ -206,15 +208,19 @@ def make_map(global_conf={}, app_conf={}):
        requirements=dict(action="options|over18|unlogged_options|optout|optin|login|reg"))
 
     mc('/api/distinguish/:how', controller='api', action="distinguish")
-    mc('/api/ipn/:secret', controller='api', action='ipn')
+    # wherever this is, google has to agree.
+    mc('/api/gcheckout', controller='ipn', action='gcheckout')
+    mc('/api/spendcreddits', controller='ipn', action="spendcreddits")
+    mc('/api/ipn/:secret', controller='ipn', action='ipn')
+    mc('/ipn/:secret',     controller='ipn', action='ipn')
     mc('/api/:action/:url_user', controller='api',
        requirements=dict(action="login|register"))
     mc('/api/gadget/click/:ids', controller = 'api', action='gadget', type='click')
     mc('/api/gadget/:type', controller = 'api', action='gadget')
     mc('/api/:action', controller='promote',
        requirements=dict(action="promote|unpromote|edit_promo|link_thumb|freebie|promote_note|update_pay|refund|traffic_viewer|rm_traffic_viewer|edit_campaign|delete_campaign|meta_promo|add_roadblock|rm_roadblock"))
     mc('/api/:action', controller='apiminimal',
-       requirements=dict(action="onload"))
+       requirements=dict(action="onload|new_captcha"))
     mc('/api/:action', controller='api')
 
     mc("/button_info", controller="api", action="info", limit = 1)

r2/r2/controllers/__init__.py

@@ -63,4 +63,4 @@
 from api import ApiminimalController
 from admin import AdminController
 from redirect import RedirectController
-
+from ipn import IpnController