Takedown tool

Conflicts:

	r2/r2/models/account.py

r2/example.ini

@@ -44,6 +44,18 @@ newsletter_api_key =
 # event-collector key and secret
 events_collector_key =
 events_collector_secret =
+# The API key for managing tickets on ZenDesk
+zendesk_api_key =
+# the user to use for managing tickets on ZenDesk
+zendesk_user = 
+# The API key for submitting takedown notices to ChillingEffects.org
+chillingeffects_org_api_key =
+# The API key for making calls to CloudFlare
+cloudflare_api_key =
+# The email address linked to the API key for CloudFlare
+cloudflare_email_address =
+# The API url to call to purge content from CloudFlare
+cloudflare_email_address =
 
 [DEFAULT]
 ############################################ SITE-SPECIFIC OPTIONS
@@ -763,6 +775,21 @@ create_sr_comment_karma = 0
 # Sample rate for event-collector processing
 events_collector_sample_rate = 0.0
 
+
+# Manage ZenDesk content from reddit
+ticket_provider = 
+# the domain of your support ticket provider
+ticket_base_url = 
+# the user ID to create the contact tickets under
+ticket_contact_user_id = 
+# the group name to ID for the groups defined in zendesk (comma separated list of label:numeric_ID)
+ticket_groups = 
+# the custom fields to ID for fields created in zendesk (comma separated list of label:numeric_ID)
+ticket_user_fields = 
+
+# Posting to ChillingEffects.org
+chillingeffects_org_api_base_url = https://chillingeffects.org/
+
 #### Features
 # Availability for the "force HTTPS" option
 feature_allow_force_https = {"employee": true}

r2/r2/controllers/api.py

@@ -1720,6 +1720,12 @@ def POST_editusertext(self, form, jquery, item, text):
 
         if isinstance(item, Link) and not item.is_self:
             return abort(403, "forbidden")
+
+        if getattr(item, 'admin_takedown', False):
+            # this item has been takendown by the admins,
+            # and not not be edited
+            # would love to use a 451 (legal) here, but pylons throws an error
+            return abort(403, "this content is locked and can not be edited")
 
         if isinstance(item, Comment):
             max_length = 10000

r2/r2/lib/app_globals.py

@@ -342,6 +342,10 @@ class Globals(object):
             'sidebar_message',
             'gold_sidebar_message',
         ],
+        ConfigValue.dict(ConfigValue.str, ConfigValue.int): [
+            'ticket_groups',
+            'ticket_user_fields', 
+        ],
         ConfigValue.dict(ConfigValue.str, ConfigValue.float): [
             'pennies_per_server_second',
         ],
@@ -455,6 +459,17 @@ def setup(self):
             "r2.provider.cdn",
             self.cdn_provider,
         )
+        self.ticket_provider = select_provider(
+            self.config,
+            self.pkg_resources_working_set,
+            "r2.provider.support",
+            # TODO: fix this later, it refuses to pick up 
+            # g.config['ticket_provider'] value, so hardcoding for now.
+            # really, the next uncommented line should be:
+            #self.ticket_provider,
+            # instead of:
+            "zendesk",
+        )
         self.startup_timer.intermediate("providers")
 
         ################# CONFIGURATION

r2/r2/lib/jsontemplates.py

@@ -194,6 +194,11 @@ def thing_attr(self, thing, attr):
                 elif attr == "approved_by":
                     return ban_info.get("unbanner") if not thing._spam else None
 
+        if attr == 'admin_takedown':
+            if thing.admin_takedown:
+                return 'legal'
+            return None
+
         return getattr(thing, attr, None)
 
     def data(self, thing):
@@ -531,6 +536,7 @@ class LinkJsonTemplate(ThingJsonTemplate):
         user_reports="user_reports",
         over_18="over_18",
         permalink="permalink",
+        removal_reason="admin_takedown",
         saved="saved",
         score="score",
         secure_media="secure_media_object",
@@ -645,6 +651,7 @@ class CommentJsonTemplate(ThingJsonTemplate):
         mod_reports="mod_reports",
         user_reports="user_reports",
         parent_id="parent_id",
+        removal_reason="admin_takedown",
         replies="child",
         saved="saved",
         score="score",

r2/r2/lib/pages/pages.py

@@ -4373,6 +4373,7 @@ def __init__(self,
                  expunged=False,
                  include_errors=True,
                  show_embed_help=False,
+                 admin_takedown=False,
                 ):
 
         css_class = "usertext"
@@ -4383,6 +4384,10 @@ def __init__(self,
 
         if text is None:
             text = ''
+
+        # set the attribute for admin takedowns
+        if getattr(item, 'admin_takedown', False):
+            admin_takedown = True
 
         fullname = ''
         # Do not pass fullname on deleted things, unless we're admin
@@ -4407,6 +4412,7 @@ def __init__(self,
                                 expunged=expunged,
                                 include_errors=include_errors,
                                 show_embed_help=show_embed_help,
+                                admin_takedown=admin_takedown,
                                )
 
 class MediaEmbedBody(CachedTemplate):

r2/r2/lib/pages/things.py

@@ -83,6 +83,11 @@ def __init__(self, thing, comments = True, delete = True, report = True):
         # do we show the report button?
         show_report = not is_author and report
 
+        # if they are the author, can they edit it?
+        thing_editable = getattr(thing, 'editable', True)
+        thing_takendown = getattr(thing, 'admin_takedown', False)
+        editable = is_author and thing_editable and not thing_takendown
+
         show_marknsfw = show_unmarknsfw = False
         show_rescrape = False
         if thing.can_ban or is_author or (thing.promoted and c.user_is_sponsor):
@@ -135,7 +140,7 @@ def __init__(self, thing, comments = True, delete = True, report = True):
                                   permalink  = thing.permalink,
                                   # button visibility
                                   saved = thing.saved,
-                                  editable = thing.editable, 
+                                  editable = editable, 
                                   hidden = thing.hidden, 
                                   ignore_reports = thing.ignore_reports,
                                   show_delete = show_delete,
@@ -158,6 +163,12 @@ class CommentButtons(PrintableButtons):
     def __init__(self, thing, delete = True, report = True):
         # is the current user the author?
         is_author = thing.is_author
+
+        # if they are the author, can they edit it?
+        thing_editable = getattr(thing, 'editable', True)
+        thing_takendown = getattr(thing, 'admin_takedown', False)
+        editable = is_author and thing_editable and not thing_takendown
+
         # do we show the report button?
         show_report = not is_author and report and thing.can_reply
         # do we show the delete button?
@@ -193,6 +204,7 @@ def __init__(self, thing, delete = True, report = True):
                                   profilepage = c.profilepage,
                                   permalink = thing.permalink,
                                   saved = thing.saved,
+                                  editable = editable,
                                   ignore_reports = thing.ignore_reports,
                                   full_comment_path = thing.full_comment_path,
                                   full_comment_count = thing.full_comment_count,

r2/r2/lib/providers/cdn/__init__.py

@@ -40,3 +40,7 @@ def get_client_ip(self, environ):
 
         """
         raise NotImplementedError
+
+    def purge_content(self, url):
+        """Purge content from the CDN by URL"""
+        raise NotImplementedError

r2/r2/lib/providers/cdn/cloudflare.py

@@ -21,17 +21,39 @@
 ###############################################################################
 
 import hashlib
+import json
+import requests
 
 from pylons import g
 
 from r2.lib.providers.cdn import CdnProvider
 from r2.lib.utils import constant_time_compare
 
-
 class CloudFlareCdnProvider(CdnProvider):
     """A provider for reddit's configuration of CloudFlare.
 
     """
+
+    def _do_content_purge(self, url):  
+        """Does the purge of the content from CloudFlare."""      
+        data = {
+            'files': [
+                url,
+            ]
+        }
+
+        timer = g.stats.get_timer("providers.cloudflare.content_purge")
+        timer.start()
+        response = requests.delete(
+            g.secrets['cloudflare_purge_key_url'],
+            headers={
+                'X-Auth-Email': g.secrets['cloudflare_email_address'],
+                'X-Auth-Key': g.secrets['cloudflare_api_key'],
+                'content-type': 'application/json',
+            },
+            data=json.dumps(data),
+        )
+        timer.stop()
 
     def get_client_ip(self, environ):
         try:
@@ -47,3 +69,32 @@ def get_client_ip(self, environ):
             return None
 
         return client_ip
+
+    def purge_content(self, url):
+        """Purges the content specified by url from the cache."""
+        # You'll notice purge is being called multiple times. Our sysadmins
+        # say that it doesn't always fully clear the first time, so they are
+        # now in the habit of always running the purge API call 3 times.
+        # Replicating that less than ideal behaviour here
+
+        self._do_content_purge(url)
+        self._do_content_purge(url)
+        self._do_content_purge(url)
+
+        # per the CloudFlare docs:
+        #    https://www.cloudflare.com/docs/client-api.html#s4.5
+        #    The full URL of the file that needs to be purged from 
+        #    CloudFlare's  cache. Keep in mind, that if an HTTP and 
+        #    an HTTPS version of the file exists, then both versions 
+        #    will need to be purged independently
+        # create the "alternate" URL for http or https
+        if 'https://' in url:
+            url_altered = url.replace('https://', 'http://')
+        else:
+            url_altered = url.replace('http://', 'https://')
+
+        self._do_content_purge(url_altered)
+        self._do_content_purge(url_altered)
+        self._do_content_purge(url_altered)
+
+        return True

r2/r2/lib/providers/media/__init__.py

@@ -28,7 +28,18 @@ class MediaProvider(object):
     users to be able to view those objects over HTTP.
 
     """
-    def put(self, name, contents):
+
+    def make_inaccessible(self, url):
+        """Make the content unavaiable, but do not remove. Content could
+        be recovered at a later time.
+
+        `url` must be a url linking to the content
+
+        The return value should be a 
+        """
+        raise NotImplementedError
+
+    def put(self, category, name, contents):
         """Put a media object on the media server and return its HTTP URL.
 
         `name` must be a local filename including an extension.
@@ -40,3 +51,7 @@ def put(self, name, contents):
 
         """
         raise NotImplementedError
+
+    def purge(self, url):
+        """Remove the content. Content can not be recovered."""
+        raise NotImplementedError

r2/r2/lib/providers/media/filesystem.py

@@ -50,9 +50,28 @@ class FileSystemMediaProvider(MediaProvider):
         ],
     }
 
-    def put(self, name, contents):
+    def make_inaccessible(self, url):
+        # When it comes to file system, there isn't really the concept of
+        # "making a file inaccessible" separate from deletion without
+        # losing track of it. For the sake of not creating orphaned files, 
+        # not implementing this method
+        g.log.warning(
+            'FileSystemMediaProvider.make_inaccessible is consciously '
+            'not implemented and does not raise an error.'
+        )
+        return True
+
+    def put(self, category, name, contents):
         assert os.path.dirname(name) == ""
         path = os.path.join(g.media_fs_root, name)
         with open(path, "w") as f:
             f.write(contents)
         return urlparse.urljoin(g.media_fs_base_url_http, name)
+
+    def purge(self, url):
+        """Remove the content from disk. Content can not be recovered."""
+
+        name = url.split('/')[-1]
+        path = os.path.join(g.media_fs_root, name)
+        os.remove(path)
+        return True