diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py
index 87e328e5037..05a5db62f00 100644
--- a/readthedocs/rtd_tests/tests/test_api.py
+++ b/readthedocs/rtd_tests/tests/test_api.py
@@ -12,6 +12,7 @@
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 from django.test import TestCase
+from django.utils import six
 from django_dynamic_fixture import get
 from rest_framework import status
 from rest_framework.test import APIClient
@@ -247,7 +248,8 @@ def test_project_features(self):
         resp = client.get('/api/v2/project/%s/' % (project.pk))
         self.assertEqual(resp.status_code, 200)
         self.assertIn('features', resp.data)
-        self.assertEqual(
+        six.assertCountEqual(
+            self,
             resp.data['features'],
             [feature1.feature_id, feature2.feature_id],
         )
diff --git a/readthedocs/rtd_tests/tests/test_notifications.py b/readthedocs/rtd_tests/tests/test_notifications.py
index 99f7f0940e5..cdc2fbc5c71 100644
--- a/readthedocs/rtd_tests/tests/test_notifications.py
+++ b/readthedocs/rtd_tests/tests/test_notifications.py
@@ -80,7 +80,7 @@ class TestNotification(Notification):
                 request=mock.ANY,
                 template='core/email/common.txt',
                 context={'content': 'Test'},
-                subject=u'This is 1',
+                subject=u'This is {}'.format(build.id),
                 template_html='core/email/common.html',
                 recipient=user.email,
             )