Remove UTM tags from url and guid

This helps preventing duplicate entries to appear -- some with utm tags
and others without.

feedhq/feeds/models.py

@@ -45,7 +45,7 @@
 from .tasks import (ensure_subscribed, store_entries, update_favicon,
                     update_feed)
 from .utils import (epoch_to_utc, FAVICON_FETCHER, get_job, is_feed,
-                    JobNotFound, USER_AGENT)
+                    JobNotFound, remove_utm_tags, USER_AGENT)
 from .. import es
 from ..storage import OverwritingStorage
 from ..tasks import enqueue
@@ -341,6 +341,7 @@ def entry_data(cls, entry, parsed):
             data['guid'] = entry.title
         if not data['guid']:
             return
+        data['guid'] = remove_utm_tags(data['guid'])
         if 'description' in entry:
             data['subtitle'] = entry.description
         if 'summary' in entry:

feedhq/feeds/utils.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import datetime
+from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
 
 from django.utils import timezone
 
@@ -36,3 +37,12 @@ def get_job(name):
     if not redis.exists(key):
         raise JobNotFound
     return job_details(name, connection=redis)
+
+
+def remove_utm_tags(guid):
+    parts = list(urlsplit(guid))
+    qs = parse_qs(parts[3])  # [3] is query component
+    filtered = sorted([(k, v) for k, v in qs.items()
+                       if not k.startswith('utm_')])
+    parts[3] = urlencode(filtered, doseq=True)
+    return urlunsplit(parts)