Skip to content
This repository
Browse code

* Combine cassandra clusters into a single one

* Add a new Cassandra CF for comment-sort caching
* Add JSONP support
* Support non-auto-renewing PayPal IPNs
* Gold accounting
* Gold features: comments tracking, larger comment limit
* Autorenewing gold
* Google checkout support
* Profile-page sorting for all
* Title-text on the logo
* Hardcache sharding and profiling
* Self serve enhancements
* Add /r/foo/faq
* Make self-centred reddits allow for more verbose selftext
* Much better down-handling of databases
* Add the ability to take a thread-dump from a running process
* Remove the comscore tracker
* Add comments to modqueue (does not back-populate)
  • Loading branch information...
commit 37e2ba9892d1742f78ab496d8fc1d17797731078 1 parent 0ae8f2f
David King authored October 18, 2010

Showing 96 changed files with 3,044 additions and 893 deletions. Show diff stats Hide diff stats

  1. 228  config/cassandra/storage-conf.xml
  2. 4  r2/draw_load.py
  3. 12  r2/example.ini
  4. 16  r2/r2/config/middleware.py
  5. 5  r2/r2/config/routing.py
  6. 276  r2/r2/controllers/api.py
  7. 11  r2/r2/controllers/awards.py
  8. 16  r2/r2/controllers/embed.py
  9. 13  r2/r2/controllers/error.py
  10. 225  r2/r2/controllers/front.py
  11. 68  r2/r2/controllers/health.py
  12. 49  r2/r2/controllers/listingcontroller.py
  13. 4  r2/r2/controllers/post.py
  14. 27  r2/r2/controllers/promotecontroller.py
  15. 49  r2/r2/controllers/reddit_base.py
  16. 8  r2/r2/controllers/toolbar.py
  17. 26  r2/r2/controllers/validator/validator.py
  18. 2  r2/r2/lib/_normalized_hot.pyx
  19. 6  r2/r2/lib/amqp.py
  20. 43  r2/r2/lib/app_globals.py
  21. 2  r2/r2/lib/base.py
  22. 4  r2/r2/lib/c/filters.c
  23. 5  r2/r2/lib/cache.py
  24. 10  r2/r2/lib/captcha.py
  25. 228  r2/r2/lib/comment_tree.py
  26. 1  r2/r2/lib/db/operators.py
  27. 200  r2/r2/lib/db/queries.py
  28. 216  r2/r2/lib/db/tdb_cassandra.py
  29. 1  r2/r2/lib/db/tdb_sql.py
  30. 2  r2/r2/lib/db/thing.py
  31. 20  r2/r2/lib/emailer.py
  32. 299  r2/r2/lib/hardcachebackend.py
  33. 8  r2/r2/lib/indextank.py
  34. 2  r2/r2/lib/jsonresponse.py
  35. 6  r2/r2/lib/jsontemplates.py
  36. 2  r2/r2/lib/lock.py
  37. 32  r2/r2/lib/manager/db_manager.py
  38. 45  r2/r2/lib/menus.py
  39. 49  r2/r2/lib/migrate/comment_sorts.py
  40. 51  r2/r2/lib/migrate/mr_urls.py
  41. 186  r2/r2/lib/mr_account.py
  42. 18  r2/r2/lib/mr_gold.py
  43. 2  r2/r2/lib/mr_top.py
  44. 226  r2/r2/lib/pages/pages.py
  45. 17  r2/r2/lib/promote.py
  46. 4  r2/r2/lib/scraper.py
  47. 14  r2/r2/lib/services.py
  48. 4  r2/r2/lib/strings.py
  49. 44  r2/r2/lib/template_helpers.py
  50. 13  r2/r2/lib/traffic.py
  51. 61  r2/r2/lib/utils/_utils.pyx
  52. 4  r2/r2/lib/utils/trial_utils.py
  53. 21  r2/r2/lib/utils/utils.py
  54. 1  r2/r2/lib/wrapped.pyx
  55. 21  r2/r2/models/_builder.pyx
  56. 38  r2/r2/models/account.py
  57. 133  r2/r2/models/admintools.py
  58. 12  r2/r2/models/award.py
  59. 12  r2/r2/models/builder.py
  60. 242  r2/r2/models/gold.py
  61. 39  r2/r2/models/link.py
  62. 78  r2/r2/models/subreddit.py
  63. 2  r2/r2/models/vote.py
  64. 84  r2/r2/public/static/css/reddit.css
  65. BIN  r2/r2/public/static/flaptor.png
  66. 19  r2/r2/public/static/js/blogbutton.js
  67. 10  r2/r2/public/static/js/jquery.reddit.js
  68. 19  r2/r2/public/static/js/reddit.js
  69. 13  r2/r2/public/static/robots.txt
  70. 4  r2/r2/templates/adminawardgive.html
  71. 4  r2/r2/templates/base.htmllite
  72. 2  r2/r2/templates/base.xml
  73. 0  r2/r2/templates/commentvisitsbox.compact
  74. 56  r2/r2/templates/commentvisitsbox.html
  75. 10  r2/r2/templates/createsubreddit.html
  76. 28  r2/r2/templates/dart_ad.html
  77. 2  r2/r2/templates/feedbackblurb.html
  78. 7  r2/r2/templates/frame.html
  79. 3  r2/r2/templates/link.html
  80. 1  r2/r2/templates/link.mobile
  81. 4  r2/r2/templates/listing.html
  82. 5  r2/r2/templates/morechildren.html
  83. 7  r2/r2/templates/panestack.html
  84. 4  r2/r2/templates/paymentform.html
  85. 20  r2/r2/templates/prefoptions.html
  86. 4  r2/r2/templates/printable.html
  87. 20  r2/r2/templates/profilebar.html
  88. 43  r2/r2/templates/promotelinkform.html
  89. 40  r2/r2/templates/promotion_summary.email
  90. 10  r2/r2/templates/reddit.html
  91. 8  r2/r2/templates/redditheader.html
  92. 11  r2/r2/templates/searchbar.html
  93. 13  r2/r2/templates/searchform.html
  94. 16  r2/r2/templates/thanks.html
  95. 4  r2/setup.py
  96. 3  scripts/usage_q.py
228  config/cassandra/storage-conf.xml
... ...
@@ -1,24 +1,154 @@
  1
+<!--
  2
+ ~ Licensed to the Apache Software Foundation (ASF) under one
  3
+ ~ or more contributor license agreements.  See the NOTICE file
  4
+ ~ distributed with this work for additional information
  5
+ ~ regarding copyright ownership.  The ASF licenses this file
  6
+ ~ to you under the Apache License, Version 2.0 (the
  7
+ ~ "License"); you may not use this file except in compliance
  8
+ ~ with the License.  You may obtain a copy of the License at
  9
+ ~
  10
+ ~    http://www.apache.org/licenses/LICENSE-2.0
  11
+ ~
  12
+ ~ Unless required by applicable law or agreed to in writing,
  13
+ ~ software distributed under the License is distributed on an
  14
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15
+ ~ KIND, either express or implied.  See the License for the
  16
+ ~ specific language governing permissions and limitations
  17
+ ~ under the License.
  18
+-->
1 19
 <Storage>
2 20
   <!--======================================================================-->
3 21
   <!-- Basic Configuration                                                  -->
4 22
   <!--======================================================================-->
5 23
 
  24
+  <!-- 
  25
+   ~ The name of this cluster.  This is mainly used to prevent machines in
  26
+   ~ one logical cluster from joining another.
  27
+  -->
6 28
   <ClusterName>reddit</ClusterName>
7 29
 
  30
+  <!--
  31
+   ~ Turn on to make new [non-seed] nodes automatically migrate the right data 
  32
+   ~ to themselves.  (If no InitialToken is specified, they will pick one 
  33
+   ~ such that they will get half the range of the most-loaded node.)
  34
+   ~ If a node starts up without bootstrapping, it will mark itself bootstrapped
  35
+   ~ so that you can't subsequently accidently bootstrap a node with
  36
+   ~ data on it.  (You can reset this by wiping your data and commitlog
  37
+   ~ directories.)
  38
+   ~
  39
+   ~ Off by default so that new clusters and upgraders from 0.4 don't
  40
+   ~ bootstrap immediately.  You should turn this on when you start adding
  41
+   ~ new nodes to a cluster that already has data on it.  (If you are upgrading
  42
+   ~ from 0.4, start your cluster with it off once before changing it to true.
  43
+   ~ Otherwise, no data will be lost but you will incur a lot of unnecessary
  44
+   ~ I/O before your cluster starts up.)
  45
+  -->
8 46
   <AutoBootstrap>false</AutoBootstrap>
  47
+
  48
+  <!--
  49
+   ~ See http://wiki.apache.org/cassandra/HintedHandoff
  50
+  -->
9 51
   <HintedHandoffEnabled>true</HintedHandoffEnabled>
10 52
 
11  
-  <Keyspaces>
12  
-    <Keyspace Name="permacache">
13  
-      <ColumnFamily CompareWith="BytesType" Name="permacache" RowsCached="3000000" />
  53
+  <!--
  54
+   ~ Keyspaces and ColumnFamilies:
  55
+   ~ A ColumnFamily is the Cassandra concept closest to a relational
  56
+   ~ table.  Keyspaces are separate groups of ColumnFamilies.  Except in
  57
+   ~ very unusual circumstances you will have one Keyspace per application.
14 58
 
  59
+   ~ There is an implicit keyspace named 'system' for Cassandra internals.
  60
+  -->
  61
+  <Keyspaces>
  62
+    <Keyspace Name="Keyspace1">
  63
+      <!--
  64
+       ~ ColumnFamily definitions have one required attribute (Name)
  65
+       ~ and several optional ones.
  66
+       ~
  67
+       ~ The CompareWith attribute tells Cassandra how to sort the columns
  68
+       ~ for slicing operations.  The default is BytesType, which is a
  69
+       ~ straightforward lexical comparison of the bytes in each column.
  70
+       ~ Other options are AsciiType, UTF8Type, LexicalUUIDType, TimeUUIDType,
  71
+       ~ and LongType.  You can also specify the fully-qualified class
  72
+       ~ name to a class of your choice extending
  73
+       ~ org.apache.cassandra.db.marshal.AbstractType.
  74
+       ~ 
  75
+       ~ SuperColumns have a similar CompareSubcolumnsWith attribute.
  76
+       ~ 
  77
+       ~ BytesType: Simple sort by byte value.  No validation is performed.
  78
+       ~ AsciiType: Like BytesType, but validates that the input can be 
  79
+       ~            parsed as US-ASCII.
  80
+       ~ UTF8Type: A string encoded as UTF8
  81
+       ~ LongType: A 64bit long
  82
+       ~ LexicalUUIDType: A 128bit UUID, compared lexically (by byte value)
  83
+       ~ TimeUUIDType: a 128bit version 1 UUID, compared by timestamp
  84
+       ~
  85
+       ~ (To get the closest approximation to 0.3-style supercolumns, you
  86
+       ~ would use CompareWith=UTF8Type CompareSubcolumnsWith=LongType.)
  87
+       ~
  88
+       ~ An optional `Comment` attribute may be used to attach additional
  89
+       ~ human-readable information about the column family to its definition.
  90
+       ~ 
  91
+       ~ The optional KeysCached attribute specifies
  92
+       ~ the number of keys per sstable whose locations we keep in
  93
+       ~ memory in "mostly LRU" order.  (JUST the key locations, NOT any
  94
+       ~ column values.) Specify a fraction (value less than 1), a percentage
  95
+       ~ (ending in a % sign) or an absolute number of keys to cache.
  96
+       ~ KeysCached defaults to 200000 keys.
  97
+       ~
  98
+       ~ The optional RowsCached attribute specifies the number of rows
  99
+       ~ whose entire contents we cache in memory. Do not use this on
  100
+       ~ ColumnFamilies with large rows, or ColumnFamilies with high write:read
  101
+       ~ ratios. Specify a fraction (value less than 1), a percentage (ending in
  102
+       ~ a % sign) or an absolute number of rows to cache. 
  103
+       ~ RowsCached defaults to 0, i.e., row cache is off by default.
  104
+       ~
  105
+       ~ Remember, when using caches as a percentage, they WILL grow with
  106
+       ~ your data set!
  107
+      -->
  108
+      <ColumnFamily Name="Standard1" CompareWith="BytesType"/>
  109
+      <ColumnFamily Name="Standard2" 
  110
+                    CompareWith="UTF8Type"
  111
+                    KeysCached="100%"/>
  112
+      <ColumnFamily Name="StandardByUUID1" CompareWith="TimeUUIDType" />
  113
+      <ColumnFamily Name="Super1"
  114
+                    ColumnType="Super"
  115
+                    CompareWith="BytesType"
  116
+                    CompareSubcolumnsWith="BytesType" />
  117
+      <ColumnFamily Name="Super2"
  118
+                    ColumnType="Super"
  119
+                    CompareWith="UTF8Type"
  120
+                    CompareSubcolumnsWith="UTF8Type"
  121
+                    RowsCached="10000"
  122
+                    KeysCached="50%"
  123
+                    Comment="A column family with supercolumns, whose column and subcolumn names are UTF8 strings"/>
  124
+
  125
+      <!--
  126
+       ~ Strategy: Setting this to the class that implements
  127
+       ~ IReplicaPlacementStrategy will change the way the node picker works.
  128
+       ~ Out of the box, Cassandra provides
  129
+       ~ org.apache.cassandra.locator.RackUnawareStrategy and
  130
+       ~ org.apache.cassandra.locator.RackAwareStrategy (place one replica in
  131
+       ~ a different datacenter, and the others on different racks in the same
  132
+       ~ one.)
  133
+      -->
15 134
       <ReplicaPlacementStrategy>org.apache.cassandra.locator.RackUnawareStrategy</ReplicaPlacementStrategy>
16  
-      <ReplicationFactor>3</ReplicationFactor>
17  
-      <EndPointSnitch>org.apache.cassandra.locator.EndPointSnitch</EndPointSnitch>
18  
-    </Keyspace>
19 135
 
20  
-    <Keyspace Name="urls">
21  
-      <ColumnFamily CompareWith="UTF8Type" Name="urls" />
  136
+      <!-- Number of replicas of the data -->
  137
+      <ReplicationFactor>1</ReplicationFactor>
  138
+
  139
+      <!--
  140
+       ~ EndPointSnitch: Setting this to the class that implements
  141
+       ~ AbstractEndpointSnitch, which lets Cassandra know enough
  142
+       ~ about your network topology to route requests efficiently.
  143
+       ~ Out of the box, Cassandra provides org.apache.cassandra.locator.EndPointSnitch,
  144
+       ~ and PropertyFileEndPointSnitch is available in contrib/.
  145
+      -->
  146
+       <EndPointSnitch>org.apache.cassandra.locator.EndPointSnitch</EndPointSnitch>
  147
+     </Keyspace>
  148
+
  149
+    <Keyspace Name="permacache">
  150
+      <ColumnFamily CompareWith="BytesType" Name="permacache" />
  151
+      <ColumnFamily CompareWith="BytesType" Name="urls" RowsCached="100000" />
22 152
 
23 153
       <ReplicaPlacementStrategy>org.apache.cassandra.locator.RackUnawareStrategy</ReplicaPlacementStrategy>
24 154
       <ReplicationFactor>3</ReplicationFactor>
@@ -32,6 +162,7 @@
32 162
 
33 163
       <!-- Views -->
34 164
       <ColumnFamily CompareWith="UTF8Type" Name="VotesByLink" />
  165
+      <ColumnFamily CompareWith="UTF8Type" Name="CommentSortsCache" RowsCached="100000" />
35 166
 
36 167
       <ReplicaPlacementStrategy>org.apache.cassandra.locator.RackUnawareStrategy</ReplicaPlacementStrategy>
37 168
       <ReplicationFactor>3</ReplicationFactor>
@@ -40,24 +171,76 @@
40 171
 
41 172
   </Keyspaces>
42 173
 
  174
+  <!--
  175
+   ~ Authenticator: any IAuthenticator may be used, including your own as long
  176
+   ~ as it is on the classpath.  Out of the box, Cassandra provides
  177
+   ~ org.apache.cassandra.auth.AllowAllAuthenticator and,
  178
+   ~ org.apache.cassandra.auth.SimpleAuthenticator 
  179
+   ~ (SimpleAuthenticator uses access.properties and passwd.properties by
  180
+   ~ default).
  181
+   ~
  182
+   ~ If you don't specify an authenticator, AllowAllAuthenticator is used.
  183
+  -->
43 184
   <Authenticator>org.apache.cassandra.auth.AllowAllAuthenticator</Authenticator>
44 185
 
  186
+  <!--
  187
+   ~ Partitioner: any IPartitioner may be used, including your own as long
  188
+   ~ as it is on the classpath.  Out of the box, Cassandra provides
  189
+   ~ org.apache.cassandra.dht.RandomPartitioner,
  190
+   ~ org.apache.cassandra.dht.OrderPreservingPartitioner, and
  191
+   ~ org.apache.cassandra.dht.CollatingOrderPreservingPartitioner.
  192
+   ~ (CollatingOPP colates according to EN,US rules, not naive byte
  193
+   ~ ordering.  Use this as an example if you need locale-aware collation.)
  194
+   ~ Range queries require using an order-preserving partitioner.
  195
+   ~
  196
+   ~ Achtung!  Changing this parameter requires wiping your data
  197
+   ~ directories, since the partitioner can modify the sstable on-disk
  198
+   ~ format.
  199
+  -->
45 200
   <Partitioner>org.apache.cassandra.dht.RandomPartitioner</Partitioner>
46 201
 
  202
+  <!--
  203
+   ~ If you are using an order-preserving partitioner and you know your key
  204
+   ~ distribution, you can specify the token for this node to use. (Keys
  205
+   ~ are sent to the node with the "closest" token, so distributing your
  206
+   ~ tokens equally along the key distribution space will spread keys
  207
+   ~ evenly across your cluster.)  This setting is only checked the first
  208
+   ~ time a node is started. 
  209
+
  210
+   ~ This can also be useful with RandomPartitioner to force equal spacing
  211
+   ~ of tokens around the hash space, especially for clusters with a small
  212
+   ~ number of nodes.
  213
+  -->
47 214
   <InitialToken></InitialToken>
48 215
 
  216
+  <!--
  217
+   ~ Directories: Specify where Cassandra should store different data on
  218
+   ~ disk.  Keep the data disks and the CommitLog disks separate for best
  219
+   ~ performance
  220
+  -->
49 221
   <CommitLogDirectory>/cassandra/commitlog</CommitLogDirectory>
50 222
   <DataFileDirectories>
51 223
       <DataFileDirectory>/cassandra/data</DataFileDirectory>
52 224
   </DataFileDirectories>
53 225
 
  226
+  <!--
  227
+   ~ Addresses of hosts that are deemed contact points. Cassandra nodes
  228
+   ~ use this list of hosts to find each other and learn the topology of
  229
+   ~ the ring. You must change this if you are running multiple nodes!
  230
+  -->
54 231
   <Seeds>
55 232
       <Seed>pmc01</Seed>
56 233
       <Seed>pmc02</Seed>
57 234
       <Seed>pmc03</Seed>
  235
+      <Seed>pmc04</Seed>
  236
+      <Seed>pmc05</Seed>
58 237
       <Seed>pmc06</Seed>
59 238
       <Seed>pmc07</Seed>
60 239
       <Seed>pmc08</Seed>
  240
+      <Seed>pmc09</Seed>
  241
+      <Seed>pmc10</Seed>
  242
+      <Seed>pmc11</Seed>
  243
+      <Seed>pmc12</Seed>
61 244
   </Seeds>
62 245
 
63 246
   <!-- Miscellaneous -->
@@ -70,16 +253,38 @@
70 253
   <!-- Size to allow commitlog to grow to before creating a new segment -->
71 254
   <CommitLogRotationThresholdInMB>128</CommitLogRotationThresholdInMB>
72 255
 
  256
+
73 257
   <!-- Local hosts and ports -->
74 258
 
  259
+  <!-- 
  260
+   ~ Address to bind to and tell other nodes to connect to.  You _must_
  261
+   ~ change this if you want multiple nodes to be able to communicate!  
  262
+   ~
  263
+   ~ Leaving it blank leaves it up to InetAddress.getLocalHost(). This
  264
+   ~ will always do the Right Thing *if* the node is properly configured
  265
+   ~ (hostname, name resolution, etc), and the Right Thing is to use the
  266
+   ~ address associated with the hostname (it might not be).
  267
+  -->
75 268
   <ListenAddress></ListenAddress>
76 269
   <!-- internal communications port -->
77 270
   <StoragePort>7000</StoragePort>
78 271
 
  272
+  <!--
  273
+   ~ The address to bind the Thrift RPC service to. Unlike ListenAddress
  274
+   ~ above, you *can* specify 0.0.0.0 here if you want Thrift to listen on
  275
+   ~ all interfaces.
  276
+   ~
  277
+   ~ Leaving this blank has the same effect it does for ListenAddress,
  278
+   ~ (i.e. it will be based on the configured hostname of the node).
  279
+  -->
79 280
   <ThriftAddress></ThriftAddress>
80 281
   <!-- Thrift RPC port (the port clients connect to). -->
81 282
   <ThriftPort>9160</ThriftPort>
82  
-
  283
+  <!-- 
  284
+   ~ Whether or not to use a framed transport for Thrift. If this option
  285
+   ~ is set to true then you must also use a framed transport on the 
  286
+   ~ client-side, (framed and non-framed transports are not compatible).
  287
+  -->
83 288
   <ThriftFramedTransport>false</ThriftFramedTransport>
84 289
 
85 290
 
@@ -143,7 +348,7 @@
143 348
    ~ actual heap memory usage (there is some overhead in indexing the
144 349
    ~ columns).
145 350
   -->
146  
-  <MemtableThroughputInMB>64</MemtableThroughputInMB>
  351
+  <MemtableThroughputInMB>128</MemtableThroughputInMB>
147 352
   <!--
148 353
    ~ Throughput setting for Binary Memtables.  Typically these are
149 354
    ~ used for bulk load so you want them to be larger.
@@ -161,8 +366,7 @@
161 366
    ~ commit log segment, that segment cannot be deleted.)
162 367
    ~ This needs to be large enough that it won't cause a flush storm
163 368
    ~ of all your memtables flushing at once because none has hit
164  
-   ~ the size or count thresholds yet.  For production, a larger
165  
-   ~ value such as 1440 is recommended.
  369
+   ~ the size or count thresholds yet.
166 370
   -->
167 371
   <MemtableFlushAfterMinutes>60</MemtableFlushAfterMinutes>
168 372
 
4  r2/draw_load.py
@@ -26,7 +26,7 @@ def draw_load(row_size = 12, width = 200, out_file = "/tmp/load.png"):
26 26
     
27 27
     number = (len([x for x in hosts if x.services]) + 
28 28
               len([x for x in hosts if x.database]) +
29  
-              sum(len(x.queue.queues) for x in hosts if x.queue)) + 9
  29
+              sum(len(x.queue.queues) for x in hosts if x.queue)) + 14
30 30
 
31 31
     im = Image.new("RGB", (width, number * row_size + 3))
32 32
     draw = ImageDraw.Draw(im)
@@ -43,7 +43,7 @@ def draw_box(label, color, center = False):
43 43
 
44 44
     draw_box(" ==== DATABASES ==== ", "#BBBBBB", center = True)
45 45
     for host in hosts:
46  
-        if host.database:
  46
+        if host.database or host.host.startswith('vote'):
47 47
             draw_box("  %s load: %s" % (host.host, host.load()),
48 48
                      get_load_level(host))
49 49
 
12  r2/example.ini
@@ -138,6 +138,8 @@ authorize_db =   reddit,   127.0.0.1, reddit,   password
138 138
 award_db =       reddit,   127.0.0.1, reddit,   password
139 139
 hc_db =          reddit,   127.0.0.1, reddit,   password
140 140
 
  141
+hardcache_categories = *:hc
  142
+
141 143
 # this setting will prefix all of the table names
142 144
 db_app_name = reddit
143 145
 # are we allowed to create tables?
@@ -302,9 +304,11 @@ HOT_PAGE_AGE = 1000
302 304
 # how long to consider links eligible for the rising page
303 305
 rising_period = 12 hours
304 306
 # max number of comments (default)
305  
-num_comments = 200
306  
-# max number of comments (if show all is selected)
  307
+num_comments = 100
  308
+# max number of comments (non-gold)
307 309
 max_comments = 500
  310
+# max number of comments (gold)
  311
+max_comments_gold = 2500
308 312
 # list of reddits to auto-subscribe users to
309 313
 automatic_reddits = 
310 314
 # special reddit that only reddit gold subscribers can use
@@ -315,6 +319,10 @@ num_default_reddits = 10
315 319
 num_serendipity = 250
316 320
 sr_dropdown_threshold = 15
317 321
 
  322
+# Conflate visits to a comment page that happen within this many
  323
+# seconds of each other
  324
+comment_visits_period = 600
  325
+
318 326
 #user-agents to rate-limit
319 327
 agents = 
320 328
 
16  r2/r2/config/middleware.py
@@ -40,6 +40,7 @@
40 40
 from r2.lib.html_source import HTMLValidationParser
41 41
 from cStringIO import StringIO
42 42
 import sys, tempfile, urllib, re, os, sha, subprocess
  43
+from httplib import HTTPConnection
43 44
 
44 45
 #from pylons.middleware import error_mapper
45 46
 def error_mapper(code, message, environ, global_conf=None, **kw):
@@ -308,6 +309,21 @@ def __call__(self, environ, start_response):
308 309
             # subdomains to disregard completely
309 310
             if sd in ('www', 'origin', 'beta', 'pay'):
310 311
                 continue
  312
+            elif sd == 'blog':
  313
+                r = Response()
  314
+                try:
  315
+                    conn = HTTPConnection(config['global_conf']['blog_host'])
  316
+                    conn.request("GET", environ['PATH_INFO'], None,
  317
+                                 {"Host": "blog.reddit.com"})
  318
+                    res = conn.getresponse()
  319
+                    r.status_code = res.status
  320
+                    r.content = res.read()
  321
+                    conn.close()
  322
+                except:
  323
+                    r.status_code = 500
  324
+                    environ['HTTP_HOST'] = base_domain
  325
+                    r.content = "failed to load blog"
  326
+                return r(environ, start_response)
311 327
             # subdomains which change the extension
312 328
             elif sd == 'm':
313 329
                 environ['reddit-domain-extension'] = 'mobile'
5  r2/r2/config/routing.py
@@ -146,8 +146,9 @@ def make_map(global_conf={}, app_conf={}):
146 146
     mc('/promoted/', controller='promoted', action = "listing",
147 147
        sort = "")
148 148
 
149  
-    mc('/health/threads', controller='health', action='threads')
150 149
     mc('/health', controller='health', action='health')
  150
+    mc('/health/:action', controller='health',
  151
+       requirements=dict(action="threads|dump|sleep"))
151 152
     mc('/shutdown', controller='health', action='shutdown')
152 153
 
153 154
     mc('/', controller='hot', action='listing')
@@ -176,7 +177,7 @@ def make_map(global_conf={}, app_conf={}):
176 177
     mc('/:action', controller='front',
177 178
        requirements=dict(action="random|framebuster|selfserviceoatmeal"))
178 179
     mc('/:action', controller='embed',
179  
-       requirements=dict(action="help|blog"))
  180
+       requirements=dict(action="help|blog|faq"))
180 181
     mc('/help/*anything', controller='embed', action='help')
181 182
 
182 183
     mc('/goto', controller='toolbar', action='goto')
276  r2/r2/controllers/api.py
@@ -27,11 +27,10 @@
27 27
 from validator import *
28 28
 
29 29
 from r2.models import *
30  
-from r2.models.subreddit import Default as DefaultSR
31 30
 
32 31
 from r2.lib.utils import get_title, sanitize_url, timeuntil, set_last_modified
33 32
 from r2.lib.utils import query_string, timefromnow, randstr
34  
-from r2.lib.utils import timeago, tup, filter_links
  33
+from r2.lib.utils import timeago, tup, filter_links, levenshtein
35 34
 from r2.lib.pages import FriendList, ContributorList, ModList, \
36 35
     BannedList, BoringPage, FormPage, CssError, UploadedImage, \
37 36
     ClickGadget, UrlParser
@@ -47,7 +46,7 @@
47 46
 from r2.lib.db.queries import changed
48 47
 from r2.lib import promote
49 48
 from r2.lib.media import force_thumbnail, thumbnail_url
50  
-from r2.lib.comment_tree import add_comment, delete_comment
  49
+from r2.lib.comment_tree import delete_comment
51 50
 from r2.lib import tracking,  cssfilter, emailer
52 51
 from r2.lib.subreddit_search import search_reddits
53 52
 from r2.lib.log import log_text
@@ -87,13 +86,14 @@ def POST_onload(self, form, jquery, promoted, sponsorships, *a, **kw):
87 86
         if not isinstance(c.site, FakeSubreddit):
88 87
             suffix = "-" + c.site.name
89 88
         def add_tracker(dest, where, what):
  89
+            if not dest.startswith("javascript:"):
  90
+                dest = tracking.PromotedLinkClickInfo.gen_url(fullname =what + suffix,
  91
+                                                              dest = dest,
  92
+                                                              ip = request.ip)
90 93
             jquery.set_tracker(
91 94
                 where,
92 95
                 tracking.PromotedLinkInfo.gen_url(fullname=what + suffix,
93  
-                                                  ip = request.ip),
94  
-                tracking.PromotedLinkClickInfo.gen_url(fullname =what + suffix,
95  
-                                                       dest = dest,
96  
-                                                       ip = request.ip)
  96
+                                                  ip = request.ip), dest
97 97
                 )
98 98
 
99 99
         if promoted:
@@ -130,10 +130,7 @@ def GET_info(self, link1, link2, count):
130 130
         elif link1 and ('ALREADY_SUB', 'url')  in c.errors:
131 131
             links = filter_links(tup(link1), filter_spam = False)
132 132
 
133  
-        if not links:
134  
-            return abort(404, 'not found')
135  
-
136  
-        listing = wrap_links(links, num = count)
  133
+        listing = wrap_links(filter(None, links or []), num = count)
137 134
         return BoringPage(_("API"), content = listing).render()
138 135
 
139 136
     @validatedForm(VCaptcha(),
@@ -195,7 +192,7 @@ def POST_compose(self, form, jquery, to, subject, body, ip):
195 192
                    url = VUrl(['url', 'sr']),
196 193
                    title = VTitle('title'),
197 194
                    save = VBoolean('save'),
198  
-                   selftext = VMarkdown('text'),
  195
+                   selftext = VSelfText('text'),
199 196
                    kind = VOneOf('kind', ['link', 'self']),
200 197
                    then = VOneOf('then', ('tb', 'comments'),
201 198
                                  default='comments'),
@@ -264,7 +261,10 @@ def POST_submit(self, form, jquery, url, selftext, kind, title,
264 261
             elif form.has_errors("title", errors.NO_TEXT):
265 262
                 pass
266 263
 
267  
-            if check_domain:
  264
+            if url is None:
  265
+                g.log.warning("%s is trying to submit url=None (title: %r)"
  266
+                              % (request.ip, title))
  267
+            elif check_domain:
268 268
                 banmsg = is_banned_domain(url)
269 269
 
270 270
 # Uncomment if we want to let spammers know we're on to them
@@ -723,7 +723,7 @@ def POST_indict(self, thing):
723 723
     @validatedForm(VUser(),
724 724
                    VModhash(),
725 725
                    item = VByNameIfAuthor('thing_id'),
726  
-                   text = VMarkdown('text'))
  726
+                   text = VSelfText('text'))
727 727
     def POST_editusertext(self, form, jquery, item, text):
728 728
         if (not form.has_errors("text",
729 729
                                 errors.NO_TEXT, errors.TOO_LONG) and
@@ -731,9 +731,13 @@ def POST_editusertext(self, form, jquery, item, text):
731 731
 
732 732
             if isinstance(item, Comment):
733 733
                 kind = 'comment'
  734
+                old = item.body
734 735
                 item.body = text
735 736
             elif isinstance(item, Link):
736 737
                 kind = 'link'
  738
+                if not getattr(item, "is_self", False):
  739
+                    return abort(403, "forbidden")
  740
+                old = item.selftext
737 741
                 item.selftext = text
738 742
 
739 743
             if item._deleted:
@@ -743,6 +747,12 @@ def POST_editusertext(self, form, jquery, item, text):
743 747
                 or (item._ups + item._downs > 2)):
744 748
                 item.editted = True
745 749
 
  750
+            #try:
  751
+            #    lv = levenshtein(old, text)
  752
+            #    item.levenshtein = getattr(item, 'levenshtein', 0) + lv
  753
+            #except:
  754
+            #    pass
  755
+
746 756
             item._commit()
747 757
 
748 758
             changed(item)
@@ -1173,6 +1183,7 @@ def POST_upload_sr_img(self, file, header, sponsor, name, form_id):
1173 1183
                    sr = VByName('sr'),
1174 1184
                    name = VSubredditName("name"),
1175 1185
                    title = VLength("title", max_length = 100),
  1186
+                   header_title = VLength("header-title", max_length = 500),
1176 1187
                    domain = VCnameDomain("domain"),
1177 1188
                    description = VMarkdown("description", max_length = 5120),
1178 1189
                    lang = VLang("lang"),
@@ -1195,7 +1206,8 @@ def POST_site_admin(self, form, jquery, name, ip, sr,
1195 1206
         redir = False
1196 1207
         kw = dict((k, v) for k, v in kw.iteritems()
1197 1208
                   if k in ('name', 'title', 'domain', 'description', 'over_18',
1198  
-                           'show_media', 'type', 'link_type', 'lang', "css_on_cname",
  1209
+                           'show_media', 'type', 'link_type', 'lang',
  1210
+                           "css_on_cname", "header_title", 
1199 1211
                            'allow_top'))
1200 1212
 
1201 1213
         #if a user is banned, return rate-limit errors
@@ -1323,16 +1335,19 @@ def POST_distinguish(self, form, jquery, thing, how):
1323 1335
         jquery(".content").replace_things(w, True, True)
1324 1336
         jquery(".content .link .rank").hide()
1325 1337
 
1326  
-    @noresponse(paypal_secret = VPrintable('secret', 50),
  1338
+# TODO: we're well beyond the point where this function should have been
  1339
+# broken up and moved to its own file
  1340
+    @textresponse(paypal_secret = VPrintable('secret', 50),
1327 1341
                 payment_status = VPrintable('payment_status', 20),
1328 1342
                 txn_id = VPrintable('txn_id', 20),
1329 1343
                 paying_id = VPrintable('payer_id', 50),
1330 1344
                 payer_email = VPrintable('payer_email', 250),
1331 1345
                 item_number = VPrintable('item_number', 20),
1332 1346
                 mc_currency = VPrintable('mc_currency', 20),
1333  
-                mc_gross = VFloat('mc_gross'))
1334  
-    def POST_ipn(self, paypal_secret, payment_status, txn_id,
1335  
-                 paying_id, payer_email, item_number, mc_currency, mc_gross):
  1347
+                mc_gross = VFloat('mc_gross'),
  1348
+                custom = VPrintable('custom', 50))
  1349
+    def POST_ipn(self, paypal_secret, payment_status, txn_id, paying_id,
  1350
+                 payer_email, item_number, mc_currency, mc_gross, custom):
1336 1351
 
1337 1352
         if paypal_secret != g.PAYPAL_SECRET:
1338 1353
             log_text("invalid IPN secret",
@@ -1349,17 +1364,8 @@ def POST_ipn(self, paypal_secret, payment_status, txn_id,
1349 1364
             payment_status = ''
1350 1365
 
1351 1366
         psl = payment_status.lower()
1352  
-        if psl == '' and parameters['txn_type'] == 'subscr_signup':
1353  
-            return "Ok"
1354  
-        elif psl == '' and parameters['txn_type'] == 'subscr_cancel':
1355  
-            return "Ok"
1356  
-        elif parameters.get('txn_type', '') == 'send_money' and mc_gross < 3.95:
1357  
-            # Temporary block while the last of the "legacy" PWYW subscriptions
1358  
-            # roll in
1359  
-            for k, v in parameters.iteritems():
1360  
-                g.log.info("IPN: %r = %r" % (k, v))
1361  
-            return "Ok"
1362  
-        elif psl == 'completed':
  1367
+
  1368
+        if psl == 'completed':
1363 1369
             pass
1364 1370
         elif psl == 'refunded':
1365 1371
             log_text("refund", "Just got notice of a refund.", "info")
@@ -1372,12 +1378,48 @@ def POST_ipn(self, paypal_secret, payment_status, txn_id,
1372 1378
             # TODO: something useful when this happens -- and don't
1373 1379
             # forget to verify first
1374 1380
             return "Ok"
  1381
+        elif psl == 'reversed':
  1382
+            log_text("canceled_reversal",
  1383
+                     "Just got notice of a PayPal reversal.", "info")
  1384
+            # TODO: something useful when this happens -- and don't
  1385
+            # forget to verify first
  1386
+            return "Ok"
  1387
+        elif psl == 'canceled_reversal':
  1388
+            log_text("canceled_reversal",
  1389
+                     "Just got notice of a PayPal 'canceled reversal'.", "info")
  1390
+            return "Ok"
  1391
+        elif psl == '':
  1392
+            pass
1375 1393
         else:
1376 1394
             for k, v in parameters.iteritems():
1377 1395
                 g.log.info("IPN: %r = %r" % (k, v))
1378 1396
 
1379 1397
             raise ValueError("Unknown IPN status: %r" % payment_status)
1380 1398
 
  1399
+        if parameters['txn_type'] == 'subscr_signup':
  1400
+            return "Ok"
  1401
+        elif parameters['txn_type'] == 'subscr_cancel':
  1402
+            cancel_subscription(parameters['subscr_id'])
  1403
+            return "Ok"
  1404
+        elif parameters['txn_type'] == 'subscr_failed':
  1405
+            log_text("failed_subscription",
  1406
+                     "Just got notice of a failed PayPal resub.", "info")
  1407
+            return "Ok"
  1408
+        elif parameters['txn_type'] == 'subscr_modify':
  1409
+            log_text("modified_subscription",
  1410
+                     "Just got notice of a modified PayPal sub.", "info")
  1411
+            return "Ok"
  1412
+        elif parameters['txn_type'] in ('new_case',
  1413
+            'recurring_payment_suspended_due_to_max_failed_payment'):
  1414
+            return "Ok"
  1415
+        elif parameters['txn_type'] == 'subscr_payment' and psl == 'completed':
  1416
+            subscr_id = parameters['subscr_id']
  1417
+        elif parameters['txn_type'] == 'web_accept' and psl == 'completed':
  1418
+            subscr_id = None
  1419
+        else:
  1420
+            raise ValueError("Unknown IPN txn_type / psl %r" %
  1421
+                             ((parameters['txn_type'], psl),))
  1422
+
1381 1423
         if mc_currency != 'USD':
1382 1424
             raise ValueError("Somehow got non-USD IPN %r" % mc_currency)
1383 1425
 
@@ -1404,41 +1446,111 @@ def POST_ipn(self, paypal_secret, payment_status, txn_id,
1404 1446
 
1405 1447
         pennies = int(mc_gross * 100)
1406 1448
 
1407  
-        if item_number and item_number == 'rgsub':
  1449
+        days = None
  1450
+        if item_number and item_number in ('rgsub', 'rgonetime'):
1408 1451
             if pennies == 2999:
1409 1452
                 secret_prefix = "ys_"
  1453
+                days = 366
1410 1454
             elif pennies == 399:
1411 1455
                 secret_prefix = "m_"
  1456
+                days = 31
1412 1457
             else:
1413  
-                log_text("weird IPN subscription",
1414  
-                         "Got %d pennies via PayPal?" % pennies, "error")
1415  
-                secret_prefix = "w_"
  1458
+                raise ValueError("Got %d pennies via PayPal?" % pennies)
  1459
+                # old formula: days = 60 + int (31 * pennies / 250.0)
1416 1460
         else:
1417  
-            secret_prefix = "o_"
  1461
+            raise ValueError("Got item number %r via PayPal?" % item_number)
  1462
+
  1463
+        account_id = accountid_from_paypalsubscription(subscr_id)
  1464
+
  1465
+        if account_id:
  1466
+            try:
  1467
+                account = Account._byID(account_id)
  1468
+            except NotFound:
  1469
+                g.log.info("Just got IPN renewal for deleted account #%d"
  1470
+                           % account_id)
  1471
+                return "Ok"
  1472
+
  1473
+            create_claimed_gold ("P" + txn_id, payer_email, paying_id,
  1474
+                                 pennies, days, None, account_id,
  1475
+                                 c.start_time, subscr_id)
  1476
+            admintools.engolden(account, days)
  1477
+
  1478
+            g.log.info("Just applied IPN renewal for %s, %d days" %
  1479
+                       (account.name, days))
  1480
+            return "Ok"
  1481
+
  1482
+        if custom:
  1483
+            gold_dict = g.hardcache.get("gold_dict-" + custom)
  1484
+            if gold_dict is None:
  1485
+                raise ValueError("No gold_dict for %r" % custom)
  1486
+
  1487
+            buyer_name = gold_dict['buyer']
  1488
+            try:
  1489
+                buyer = Account._by_name(buyer_name)
  1490
+            except NotFound:
  1491
+                g.log.info("Just got IPN for unknown buyer %s" % buyer_name)
  1492
+                return "Ok" # nothing we can do until they complain
  1493
+
  1494
+            if gold_dict['kind'] == 'self':
  1495
+                create_claimed_gold ("P" + txn_id, payer_email, paying_id,
  1496
+                                 pennies, days, None, buyer._id,
  1497
+                                 c.start_time, subscr_id)
  1498
+                admintools.engolden(buyer, days)
  1499
+
  1500
+                g.log.info("Just applied IPN for %s, %d days" %
  1501
+                           (buyer.name, days))
  1502
+
  1503
+#TODO: send a PM thanking them and showing them /r/lounge
  1504
+
  1505
+                g.hardcache.delete("gold_dict-" + custom)
  1506
+
  1507
+                return "Ok"
  1508
+            elif gold_dict['kind'] == 'gift':
  1509
+                recipient_name = gold_dict['recipient']
  1510
+                try:
  1511
+                    recipient = Account._by_name(recipient_name)
  1512
+                except NotFound:
  1513
+                    g.log.info("Just got IPN for unknown recipient %s"
  1514
+                               % recipient_name)
  1515
+                return "Ok" # nothing we can do until they complain
  1516
+                
  1517
+                create_claimed_gold ("P" + txn_id, payer_email, paying_id,
  1518
+                                 pennies, days, None, recipient._id,
  1519
+                                 c.start_time, subscr_id)
  1520
+                admintools.engolden(recipient, days)
  1521
+
  1522
+                g.log.info("Just applied IPN from %s to %s, %d days" %
  1523
+                           (buyer.name, recipient.name, days))
  1524
+
  1525
+#TODO: send PMs to buyer and recipient
  1526
+
  1527
+            else:
  1528
+                raise ValueError("Invalid gold_dict[kind] %r" %
  1529
+                                 gold_dict['kind'])
1418 1530
 
1419 1531
         gold_secret = secret_prefix + randstr(10)
1420 1532
 
1421 1533
         create_unclaimed_gold("P" + txn_id, payer_email, paying_id,
1422  
-                              pennies, gold_secret, c.start_time)
  1534
+                              pennies, days, gold_secret, c.start_time,
  1535
+                              subscr_id)
1423 1536
 
1424  
-        url = "http://www.reddit.com/thanks/" + gold_secret
  1537
+        notify_unclaimed_gold(txn_id, gold_secret, payer_email, "Paypal")
1425 1538
 
1426  
-        # No point in i18n, since we don't have access to the user's
1427  
-        # language info (or name) at this point
1428  
-        body = """
1429  
-Thanks for subscribing to reddit gold! We have received your PayPal
1430  
-transaction, number %s.
  1539
+        g.log.info("Just got IPN for %d days, secret=%s" % (days, gold_secret))
1431 1540
 
1432  
-Your secret subscription code is %s. You can use it to associate this
1433  
-subscription with your reddit account -- just visit
1434  
-%s
1435  
-        """ % (txn_id, gold_secret, url)
  1541
+        return "Ok"
1436 1542
 
1437  
-        emailer.gold_email(body, payer_email, "reddit gold subscriptions")
  1543
+    @textresponse(sn = VLength('serial-number', 100))
  1544
+    def POST_gcheckout(self, sn):
  1545
+        if sn:
  1546
+            g.log.error( "GOOGLE CHECKOUT: %s" % sn)
  1547
+            new_google_transaction(sn)
  1548
+            return '<notification-acknowledgment xmlns="http://checkout.google.com/schema/2" serial-number="%s" />' % sn
  1549
+        else:
  1550
+            g.log.error("GOOGLE CHCEKOUT: didn't work")
  1551
+            g.log.error(repr(list(request.POST.iteritems())))
1438 1552
 
1439  
-        g.log.info("Just got IPN for %d, secret=%s" % (pennies, gold_secret))
1440 1553
 
1441  
-        return "Ok"
1442 1554
 
1443 1555
     @noresponse(VUser(),
1444 1556
                 VModhash(),
@@ -1554,13 +1666,34 @@ def POST_moremessages(self, form, jquery, parent):
1554 1666
     @validatedForm(link = VByName('link_id'),
1555 1667
                    sort = VMenu('where', CommentSortMenu),
1556 1668
                    children = VCommentIDs('children'),
  1669
+                   pv_hex = VPrintable('pv_hex', 40),
1557 1670
                    mc_id = nop('id'))
1558  
-    def POST_morechildren(self, form, jquery,
1559  
-                          link, sort, children, mc_id):
  1671
+    def POST_morechildren(self, form, jquery, link, sort, children,
  1672
+                          pv_hex, mc_id):
1560 1673
         user = c.user if c.user_is_loggedin else None
  1674
+
  1675
+        mc_key = "morechildren-%s" % request.ip
  1676
+        try:
  1677
+            count = g.cache.incr(mc_key)
  1678
+        except:
  1679
+            g.cache.set(mc_key, 1, time=30)
  1680
+            count = 1
  1681
+
  1682
+        if count >= 10:
  1683
+            if user:
  1684
+                name = user.name
  1685
+            else:
  1686
+                name = "(unlogged user)"
  1687
+            g.log.warning("%s on %s hit morechildren %d times in 30 seconds"
  1688
+                          % (name, request.ip, count))
  1689
+            # TODO: redirect to rickroll or something
  1690
+
1561 1691
         if not link or not link.subreddit_slow.can_view(user):
1562 1692
             return abort(403,'forbidden')
1563 1693
 
  1694
+        if pv_hex:
  1695
+            c.previous_visits = g.cache.get(pv_hex)
  1696
+
1564 1697
         if children:
1565 1698
             builder = CommentBuilder(link, CommentSortMenu.operator(sort),
1566 1699
                                      children)
@@ -1592,6 +1725,9 @@ def _children(cur_items):
1592 1725
             jquery.things(str(mc_id)).remove()
1593 1726
             jquery.insert_things(a, append = True)
1594 1727
 
  1728
+            if pv_hex:
  1729
+                jquery.rehighlight_new_comments()
  1730
+
1595 1731
 
1596 1732
     @validate(uh = nop('uh'), # VModHash() will raise, check manually
1597 1733
               action = VOneOf('what', ('like', 'dislike', 'save')),
@@ -1642,45 +1778,35 @@ def POST_claimgold(self, form, jquery, code, postcard_okay):
1642 1778
             form.has_errors("code", errors.NO_TEXT)
1643 1779
             return
1644 1780
 
1645  
-        if code.startswith("pc_"):
1646  
-            gold_type = 'postcard'
1647  
-            if postcard_okay is None:
1648  
-                jquery(".postcard").show()
1649  
-                form.set_html(".status", _("just one more question"))
1650  
-                return
1651  
-            else:
1652  
-                d = dict(user=c.user.name, okay=postcard_okay)
1653  
-                g.hardcache.set("postcard-" + code, d, 86400 * 30)
1654  
-        elif code.startswith("ys_"):
1655  
-            gold_type = 'yearly special'
1656  
-        elif code.startswith("m_"):
1657  
-            gold_type = 'monthly'
1658  
-        else:
1659  
-            gold_type = 'old'
  1781
+        rv = claim_gold(code, c.user._id)
1660 1782
 
1661  
-        pennies = claim_gold(code, c.user._id)
1662  
-        if pennies is None:
  1783
+        if rv is None:
1663 1784
             c.errors.add(errors.INVALID_CODE, field = "code")
1664 1785
             log_text ("invalid gold claim",
1665 1786
                       "%s just tried to claim %s" % (c.user.name, code),
1666 1787
                       "info")
1667  
-        elif pennies == 0:
  1788
+        elif rv == "already claimed":
1668 1789
             c.errors.add(errors.CLAIMED_CODE, field = "code")
1669 1790
             log_text ("invalid gold reclaim",
1670 1791
                       "%s just tried to reclaim %s" % (c.user.name, code),
1671 1792
                       "info")
1672  
-        elif pennies > 0:
  1793
+        else:
  1794
+            days, subscr_id = rv
  1795
+            if days <= 0:
  1796
+                raise ValueError("days = %r?" % days)
  1797
+
1673 1798
             log_text ("valid gold claim",
1674 1799
                       "%s just claimed %s" % (c.user.name, code),
1675 1800
                       "info")
  1801
+
  1802
+            if subscr_id:
  1803
+                c.user.gold_subscr_id = subscr_id
  1804
+
  1805
+            admintools.engolden(c.user, days)
  1806
+
1676 1807
             g.cache.set("recent-gold-" + c.user.name, True, 600)
1677  
-            c.user.creddits += pennies
1678  
-            c.user.gold_type = gold_type
1679  
-            admintools.engolden(c.user, postcard_okay)
1680 1808
             form.set_html(".status", _("claimed!"))
1681 1809
             jquery(".lounge").show()
1682  
-        else:
1683  
-            raise ValueError("pennies = %r?" % pennies)
1684 1810
 
1685 1811
         # Activate any errors we just manually set
1686 1812
         form.has_errors("code", errors.INVALID_CODE, errors.CLAIMED_CODE,
11  r2/r2/controllers/awards.py
@@ -34,12 +34,17 @@ def GET_index(self):
34 34
         return res
35 35
 
36 36
     @validate(VAdmin(),
37  
-              award = VAwardByCodename('awardcn'))
38  
-    def GET_give(self, award):
  37
+              award = VAwardByCodename('awardcn'),
  38
+              recipient = nop('recipient'),
  39
+              desc = nop('desc'),
  40
+              url = nop('url'),
  41
+              hours = nop('hours'))
  42
+    def GET_give(self, award, recipient, desc, url, hours):
39 43
         if award is None:
40 44
             abort(404, 'page not found')
41 45
 
42  
-        res = AdminPage(content = AdminAwardGive(award),
  46
+        res = AdminPage(content = AdminAwardGive(award, recipient, desc,
  47
+                                                 url, hours),
43 48
                         title='give an award').render()
44 49
         return res
45 50
 
16  r2/r2/controllers/embed.py
@@ -31,6 +31,8 @@
31 31
 from urllib2 import HTTPError
32 32
 
33 33
 class EmbedController(RedditController):
  34
+    allow_stylesheets = True
  35
+
34 36
     def rendercontent(self, input, fp):
35 37
         soup = BeautifulSoup(input)
36 38
 
@@ -65,10 +67,14 @@ def rendercontent(self, input, fp):
65 67
                         content = Embed(content=output),
66 68
                         show_sidebar = None).render()
67 69
 
68  
-    def renderurl(self):
  70
+    def renderurl(self, override=None):
  71
+        if override:
  72
+            path = override
  73
+        else:
  74
+            path = request.path
69 75
 
70 76
         # Needed so http://reddit.com/help/ works
71  
-        fp = request.path.rstrip("/")
  77
+        fp = path.rstrip("/")
72 78
         u = "http://code.reddit.com/wiki" + fp + '?stripped=1'
73 79
 
74 80
         g.log.debug("Pulling %s for help" % u)
@@ -88,3 +94,9 @@ def GET_blog(self):
88 94
         return self.redirect("http://blog.%s/" %
89 95
                              get_domain(cname = False, subreddit = False,
90 96
                                         no_www = True))
  97
+
  98
+    def GET_faq(self):
  99
+        if c.default_sr:
  100
+            return self.redirect('/help/faq')
  101
+        else:
  102
+            return self.renderurl('/help/faqs/' + c.site.name)
13  r2/r2/controllers/error.py
@@ -33,7 +33,7 @@
33 33
     # place all r2 specific imports in here.  If there is a code error, it'll get caught and
34 34
     # the stack trace won't be presented to the user in production
35 35
     from reddit_base import RedditController, Cookies
36  
-    from r2.models.subreddit import Default, Subreddit
  36
+    from r2.models.subreddit import DefaultSR, Subreddit
37 37
     from r2.models.link import Link
38 38
     from r2.lib import pages
39 39
     from r2.lib.strings import strings, rand_strings
@@ -110,7 +110,7 @@ def __call__(self, environ, start_response):
110 110
 
111 111
     def send403(self):
112 112
         c.response.status_code = 403
113  
-        c.site = Default
  113
+        c.site = DefaultSR()
114 114
         res = pages.RedditError(_("forbidden (%(domain)s)") %
115 115
                                 dict(domain=g.domain))
116 116
         return res.render()
@@ -122,10 +122,11 @@ def send404(self):
122 122
         if c.site._spam and not c.user_is_admin:
123 123
             subject = ("the subreddit /r/%s has been incorrectly banned" %
124 124
                        c.site.name)
125  
-            message = (strings.banned_subreddit %
126  
-                       dict(link = '/message/compose?to=%s&subject=%s' %
127  
-                            (url_escape(g.admin_message_acct),
128  
-                             url_escape(subject))))
  125
+            lnk = ("/r/redditrequest/submit?url=%s&title=%s"
  126
+                   % (url_escape("http://%s/r/%s" % (g.domain, c.site.name)),
  127
+                      ("the subreddit /r/%s has been incorrectly banned" %
  128
+                       c.site.name)))
  129
+            message = strings.banned_subreddit % dict(link = lnk)
129 130
 
130 131
             res = pages.RedditError(_('this reddit has been banned'),
131 132
                                     unsafe(safemarkdown(message)))
225  r2/r2/controllers/front.py
@@ -37,7 +37,7 @@
37 37
 from r2.lib.db import queries
38 38
 from r2.lib.strings import strings
39 39
 from r2.lib.solrsearch import RelatedSearchQuery, SubredditSearchQuery
40  
-from r2.lib.indextank import IndextankQuery
  40
+from r2.lib.indextank import IndextankQuery, IndextankException
41 41
 from r2.lib.contrib.pysolr import SolrError
42 42
 from r2.lib import jsontemplates
43 43
 from r2.lib import sup
@@ -46,7 +46,7 @@
46 46
 from pylons import c, request, request, Response
47 47
 
48 48
 import random as rand
49  
-import re
  49
+import re, socket
50 50
 import time as time_module
51 51
 from urllib import quote_plus
52 52
 
@@ -114,8 +114,7 @@ def GET_details(self, article):
114 114
         return DetailsPage(link = article, expand_children=False).render()
115 115
 
116 116
 
117  
-    def GET_selfserviceoatmeal(self
118  
-):
  117
+    def GET_selfserviceoatmeal(self):
119 118
         return BoringPage(_("self service help"), 
120 119
                           show_sidebar = False,
121 120
                           content = SelfServiceOatmeal()).render()
@@ -130,15 +129,44 @@ def GET_shirt(self, article):
130 129
             return ShirtPage(link = article).render()
131 130
         return self.abort404()
132 131
 
  132
+    def _comment_visits(self, article, user, new_visit=None):
  133
+        hc_key = "comment_visits-%s-%s" % (user.name, article._id36)
  134
+        old_visits = g.hardcache.get(hc_key, [])
  135
+
  136
+        append = False
  137
+
  138
+        if new_visit is None:
  139
+            pass
  140
+        elif len(old_visits) == 0:
  141
+            append = True
  142
+        else:
  143
+            last_visit = max(old_visits)
  144
+            time_since_last = new_visit - last_visit
  145
+            if (time_since_last.days > 0
  146
+                or time_since_last.seconds > g.comment_visits_period):
  147
+                append = True
  148
+            else:
  149
+                # They were just here a few seconds ago; consider that
  150
+                # the same "visit" as right now
  151
+                old_visits.pop()
  152
+
  153
+        if append:
  154
+            copy = list(old_visits) # make a copy
  155
+            copy.append(new_visit)
  156
+            if len(copy) > 10:
  157
+                copy.pop(0)
  158
+            g.hardcache.set(hc_key, copy, 86400 * 2)
  159
+
  160
+        return old_visits
  161
+
  162
+
133 163
     @validate(article      = VLink('article'),
134 164
               comment      = VCommentID('comment'),
135 165
               context      = VInt('context', min = 0, max = 8),
136 166
               sort         = VMenu('controller', CommentSortMenu),
137  
-              num_comments = VMenu('controller', NumCommentsMenu),
138 167
               limit        = VInt('limit'),
139 168
               depth        = VInt('depth'))
140  
-    def GET_comments(self, article, comment, context, sort, num_comments,
141  
-                     limit, depth):
  169
+    def GET_comments(self, article, comment, context, sort, limit, depth):
142 170
         """Comment page for a given 'article'."""
143 171
         if comment and comment.link_id != article._id:
144 172
             return self.abort404()
@@ -158,10 +186,23 @@ def GET_comments(self, article, comment, context, sort, num_comments,
158 186
         #check for 304
159 187
         self.check_modified(article, 'comments')
160 188
 
161  
-        # if there is a focal comment, communicate down to
162  
-        # comment_skeleton.html who that will be
  189
+        # If there is a focal comment, communicate down to
  190
+        # comment_skeleton.html who that will be. Also, skip
  191
+        # comment_visits check
  192
+        previous_visits = None
163 193
         if comment:
164 194
             c.focal_comment = comment._id36
  195
+        elif (c.user_is_loggedin and c.user.gold and
  196
+              c.user.pref_highlight_new_comments):
  197
+            #TODO: remove this profiling if load seems okay
  198
+            from datetime import datetime
  199
+            before = datetime.now(g.tz)
  200
+            previous_visits = self._comment_visits(article, c.user, c.start_time)
  201
+            after = datetime.now(g.tz)
  202
+            delta = (after - before)
  203
+            msec = (delta.seconds * 1000 + delta.microseconds / 1000)
  204
+            if msec >= 100:
  205
+                g.log.warning("previous_visits code took %d msec" % msec)
165 206
 
166 207
         # check if we just came from the submit page
167 208
         infotext = None
@@ -170,11 +211,12 @@ def GET_comments(self, article, comment, context, sort, num_comments,
170 211
 
171 212
         check_cheating('comments')
172 213
 
173  
-        # figure out number to show based on the menu (when num_comments
174  
-        # is 'true', the user wants to temporarily override their
175  
-        # comments limit pref
176  
-        user_num = c.user.pref_num_comments or g.num_comments
177  
-        num = g.max_comments if num_comments == 'true' else user_num
  214
+        if not c.user.pref_num_comments:
  215
+            num = g.num_comments
  216
+        elif c.user.gold:
  217
+            num = min(c.user.pref_num_comments, g.max_comments_gold)
  218
+        else:
  219
+            num = min(c.user.pref_num_comments, g.max_comments)
178