diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py
index e9bd048a18c..48fbc0a720c 100644
--- a/readthedocs/projects/tasks.py
+++ b/readthedocs/projects/tasks.py
@@ -24,6 +24,8 @@
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from slumber.exceptions import HttpClientError
+from sphinx.ext import intersphinx
+
 
 from readthedocs.builds.constants import (
     BUILD_STATE_BUILDING,
@@ -58,6 +60,7 @@
 )
 from readthedocs.doc_builder.loader import get_builder_class
 from readthedocs.doc_builder.python_environments import Conda, Virtualenv
+from readthedocs.sphinx_domains.models import SphinxDomain
 from readthedocs.projects.models import APIProject
 from readthedocs.restapi.client import api as api_v2
 from readthedocs.vcs_support import utils as vcs_support_utils
@@ -1232,6 +1235,7 @@ def fileify(version_pk, commit):
             ),
         )
         _manage_imported_files(version, path, commit)
+        _update_intersphinx_data(version, path, commit)
     else:
         log.info(
             LOG_TEMPLATE.format(
@@ -1242,6 +1246,59 @@ def fileify(version_pk, commit):
         )
 
 
+def _update_intersphinx_data(version, path, commit):
+    """
+    Update intersphinx data for this version
+
+    :param version: Version instance
+    :param path: Path to search
+    :param commit: Commit that updated path
+    """
+    object_file = os.path.join(path, 'objects.inv')
+
+    # These classes are copied from Sphinx
+    # https://git.io/fhFbI
+    class MockConfig:
+        intersphinx_timeout = None
+        tls_verify = False
+
+    class MockApp:
+        srcdir = ''
+        config = MockConfig()
+
+        def warn(self, msg):
+            log.warning('Sphinx MockApp: %s', msg)
+
+    invdata = intersphinx.fetch_inventory(MockApp(), '', object_file)
+    for key, value in sorted(invdata.items() or {}):
+        domain, _type = key.split(':')
+        for name, einfo in sorted(value.items()):
+            # project, version, url, display_name
+            # ('Sphinx', '1.7.9', 'faq.html#epub-faq', 'Epub info')
+            url = einfo[2]
+            if '#' in url:
+                doc_name, anchor = url.split('#')
+            else:
+                doc_name, anchor = url, ''
+            display_name = einfo[3]
+            obj, _ = SphinxDomain.objects.get_or_create(
+                project=version.project,
+                version=version,
+                domain=domain,
+                name=name,
+                display_name=display_name,
+                type=_type,
+                doc_name=doc_name,
+                anchor=anchor,
+            )
+            if obj.commit != commit:
+                obj.commit = commit
+                obj.save()
+    SphinxDomain.objects.filter(project=version.project,
+                                version=version
+                                ).exclude(commit=commit).delete()
+
+
 def _manage_imported_files(version, path, commit):
     """
     Update imported files for version.
diff --git a/readthedocs/restapi/urls.py b/readthedocs/restapi/urls.py
index cadf531c595..8642db0c431 100644
--- a/readthedocs/restapi/urls.py
+++ b/readthedocs/restapi/urls.py
@@ -25,6 +25,7 @@
     SocialAccountViewSet,
     VersionViewSet,
 )
+from readthedocs.sphinx_domains.api import SphinxDomainAPIView
 
 
 router = routers.DefaultRouter()
@@ -34,6 +35,7 @@
 router.register(r'project', ProjectViewSet, basename='project')
 router.register(r'notification', NotificationViewSet, basename='emailhook')
 router.register(r'domain', DomainViewSet, basename='domain')
+router.register(r'sphinx_domains', SphinxDomainAPIView, base_name='sphinxdomain')
 router.register(
     r'remote/org',
     RemoteOrganizationViewSet,
diff --git a/readthedocs/rtd_tests/tests/test_privacy_urls.py b/readthedocs/rtd_tests/tests/test_privacy_urls.py
index 9bffbb82558..1e5c8dfe329 100644
--- a/readthedocs/rtd_tests/tests/test_privacy_urls.py
+++ b/readthedocs/rtd_tests/tests/test_privacy_urls.py
@@ -364,6 +364,7 @@ def setUp(self):
             'api_webhook_gitlab': {'status_code': 405},
             'api_webhook_bitbucket': {'status_code': 405},
             'api_webhook_generic': {'status_code': 403},
+            'sphinxdomain-detail': {'status_code': 404},
             'remoteorganization-detail': {'status_code': 404},
             'remoterepository-detail': {'status_code': 404},
             'remoteaccount-detail': {'status_code': 404},
diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py
index a759450afeb..868fb9bb88d 100644
--- a/readthedocs/settings/base.py
+++ b/readthedocs/settings/base.py
@@ -103,6 +103,7 @@ def INSTALLED_APPS(self):  # noqa
             'readthedocs.notifications',
             'readthedocs.integrations',
             'readthedocs.analytics',
+            'readthedocs.sphinx_domains',
             'readthedocs.search',
 
 
diff --git a/readthedocs/sphinx_domains/__init__.py b/readthedocs/sphinx_domains/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/readthedocs/sphinx_domains/admin.py b/readthedocs/sphinx_domains/admin.py
new file mode 100644
index 00000000000..9182fb8c6b5
--- /dev/null
+++ b/readthedocs/sphinx_domains/admin.py
@@ -0,0 +1,13 @@
+"""Domain Admin classes."""
+from django.contrib import admin
+
+from .models import SphinxDomain
+
+
+class SphinxDomainAdmin(admin.ModelAdmin):
+    list_filter = ('type', 'project')
+    raw_id_fields = ('project', 'version')
+    search_fields = ('doc_name', 'name')
+
+
+admin.site.register(SphinxDomain, SphinxDomainAdmin)
diff --git a/readthedocs/sphinx_domains/api.py b/readthedocs/sphinx_domains/api.py
new file mode 100644
index 00000000000..8b3b48c63ad
--- /dev/null
+++ b/readthedocs/sphinx_domains/api.py
@@ -0,0 +1,43 @@
+"""Domain API classes."""
+
+from rest_framework import serializers
+
+from readthedocs.restapi.views.model_views import UserSelectViewSet
+
+from .models import SphinxDomain
+
+
+class SphinxDomainSerializer(serializers.ModelSerializer):
+    project = serializers.SlugRelatedField(slug_field='slug', read_only=True)
+    version = serializers.SlugRelatedField(slug_field='slug', read_only=True)
+
+    class Meta:
+        model = SphinxDomain
+        fields = (
+            'project',
+            'version',
+            'name',
+            'display_name',
+            'role_name',
+            'docs_url',
+        )
+
+
+class SphinxDomainAdminSerializer(SphinxDomainSerializer):
+
+    class Meta(SphinxDomainSerializer.Meta):
+        fields = '__all__'
+
+
+class SphinxDomainAPIView(UserSelectViewSet):  # pylint: disable=too-many-ancestors
+    model = SphinxDomain
+    serializer_class = SphinxDomainSerializer
+    admin_serializer_class = SphinxDomainAdminSerializer
+    filter_fields = (
+        'project__slug',
+        'version__slug',
+        'domain',
+        'type',
+        'doc_name',
+        'name',
+    )
diff --git a/readthedocs/sphinx_domains/models.py b/readthedocs/sphinx_domains/models.py
new file mode 100644
index 00000000000..5ca9e12213b
--- /dev/null
+++ b/readthedocs/sphinx_domains/models.py
@@ -0,0 +1,84 @@
+"""
+Sphinx Domain modeling.
+
+http://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html
+"""
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from django_extensions.db.models import TimeStampedModel
+
+from readthedocs.builds.models import Version
+from readthedocs.core.resolver import resolve
+from readthedocs.projects.models import Project
+from readthedocs.projects.querysets import RelatedProjectQuerySet
+
+
+class SphinxDomain(TimeStampedModel):
+
+    """
+    Information from a project about it's Sphinx domains.
+
+    This captures data about API objects that exist in that codebase.
+    """
+
+    project = models.ForeignKey(
+        Project,
+        related_name='sphinx_domains',
+    )
+    version = models.ForeignKey(
+        Version,
+        verbose_name=_('Version'),
+        related_name='sphinx_domains',
+    )
+    commit = models.CharField(_('Commit'), max_length=255, null=True)
+
+    domain = models.CharField(
+        _('Domain'),
+        max_length=255,
+    )
+    name = models.CharField(
+        _('Name'),
+        max_length=255,
+    )
+    display_name = models.CharField(
+        _('Display Name'),
+        max_length=255,
+    )
+    type = models.CharField(
+        _('Type'),
+        max_length=255,
+    )
+    doc_name = models.CharField(
+        _('Doc Name'),
+        max_length=255,
+    )
+    anchor = models.CharField(
+        _('Anchor'),
+        max_length=255,
+    )
+    objects = RelatedProjectQuerySet.as_manager()
+
+    def __str__(self):
+        return f'''
+            SphinxDomain [{self.project.slug}:{self.version.slug}]
+            [{self.domain}:{self.type}] {self.name} ->
+            {self.doc_name}#{self.anchor}
+        '''
+
+    @property
+    def role_name(self):
+        return f'{self.domain}:{self.type}'
+
+    @property
+    def docs_url(self):
+        path = self.doc_name
+        if self.anchor:
+            path += f'#{self.anchor}'
+        full_url = resolve(
+            project=self.project,
+            version_slug=self.version.slug,
+            filename=path,
+        )
+        return full_url
diff --git a/requirements/local-docs-build.txt b/requirements/local-docs-build.txt
index b0552556671..b38aeb006e6 100644
--- a/requirements/local-docs-build.txt
+++ b/requirements/local-docs-build.txt
@@ -2,7 +2,6 @@
 
 # Base packages
 docutils==0.14
-Sphinx==1.8.4
 sphinx_rtd_theme==0.4.3
 sphinx-tabs==1.1.10
 # Required to avoid Transifex error with reserved slug
diff --git a/requirements/pip.txt b/requirements/pip.txt
index 6d53a25da9c..0c4b30579ec 100644
--- a/requirements/pip.txt
+++ b/requirements/pip.txt
@@ -10,6 +10,9 @@ django-extensions==2.1.6
 
 djangorestframework==3.9.1
 
+# For intersphinx during builds
+Sphinx==1.8.4
+
 # Filtering for the REST API
 django-filter==2.1.0