Test for search functionality

.travis.yml

@@ -3,21 +3,23 @@ python:
   - 2.7
   - 3.6
 sudo: false
+env:
+  - ES_VERSION=1.3.9 ES_DOWNLOAD_URL=https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
 matrix:
   include:
     - python: 2.7
       env: TOXENV=docs
     - python: 2.7
       env: TOXENV=lint
+      script: tox
     - python: 2.7
       env: TOXENV=eslint
-script:
-  - tox
 cache:
   directories:
     - ~/.cache/pip
     - ~/.nvm/nvm.sh
 install:
+  - ./scripts/travis/install_elasticsearch.sh
   - pip install tox-travis
   - curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.2/install.sh | bash
   - source ~/.nvm/nvm.sh
@@ -27,7 +29,7 @@ install:
   - npm install
   - bower install
 script:
-  - tox
+  - ./scripts/travis/run_tests.sh
 notifications:
   slack:
     rooms:

conftest.py

@@ -0,0 +1,19 @@
+import logging
+
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption('--including-search', action='store_true', dest="searchtests",
+                     default=False, help="enable search tests")
+
+
+def pytest_configure(config):
+    if not config.option.searchtests:
+        # Include `not search` to parameters so that search test do not perform
+        setattr(config.option, 'markexpr', 'not search')
+
+
+@pytest.fixture(autouse=True)
+def settings_modification(settings):
+    settings.CELERY_ALWAYS_EAGER = True

docs/tests.rst

@@ -17,6 +17,11 @@ environments by running::
 
     tox
 
+In order to run all test including the search tests, include `"'--including-search'"`
+argument::
+
+    tox "'--including-search'"
+
 To target a specific environment::
 
     tox -e py27

readthedocs/core/management/commands/reindex_elasticsearch.py

@@ -52,5 +52,5 @@ def handle(self, *args, **options):
             try:
                 update_search(version.pk, commit,
                               delete_non_commit_files=False)
-            except Exception:
-                log.exception('Reindex failed for %s', version)
+            except Exception as e:
+                log.exception('Reindex failed for %s, %s', version, e)

readthedocs/search/indexes.py

@@ -107,6 +107,10 @@ def create_index(self, index=None):
         }
         self.es.indices.create(index=index, body=body)
 
+    def refresh_index(self, index=None):
+        index = index or self._index
+        self.es.indices.refresh(index=index)
+
     def put_mapping(self, index=None):
         index = index or self._index
         self.es.indices.put_mapping(self._type, self.get_mapping(), index)
@@ -155,6 +159,10 @@ def index_document(self, data, index=None, parent=None, routing=None):
             kwargs['routing'] = routing
         self.es.index(**kwargs)
 
+    def delete_index(self, index_name):
+
+        self.es.indices.delete(index=index_name)
+
     def delete_document(self, body, index=None, parent=None, routing=None):
         kwargs = {
             'index': index or self._index,

readthedocs/search/tests/conftest.py

@@ -0,0 +1,63 @@
+import random
+import string
+from random import shuffle
+
+import pytest
+from django_dynamic_fixture import G
+
+from readthedocs.projects.models import Project
+from readthedocs.search.indexes import Index, ProjectIndex, PageIndex, SectionIndex
+from .dummy_data import DUMMY_PAGE_JSON, ALL_PROJECTS
+
+
+@pytest.fixture(autouse=True)
+def mock_elastic_index(mocker):
+    index_name = ''.join([random.choice(string.ascii_letters) for _ in range(5)])
+    mocker.patch.object(Index, '_index', index_name.lower())
+
+
+@pytest.fixture(autouse=True)
+def es_index(mock_elastic_index):
+    # Create the index.
+    index = Index()
+    index_name = index.timestamped_index()
+    index.create_index(index_name)
+    index.update_aliases(index_name)
+    # Update mapping
+    proj = ProjectIndex()
+    proj.put_mapping()
+    page = PageIndex()
+    page.put_mapping()
+    sec = SectionIndex()
+    sec.put_mapping()
+
+    yield index
+    index.delete_index(index_name=index_name)
+
+
+@pytest.fixture
+def all_projects():
+    projects = [G(Project, slug=project_slug, name=project_slug) for project_slug in ALL_PROJECTS]
+    shuffle(projects)
+    return projects
+
+
+@pytest.fixture
+def project(all_projects):
+    # Return a single project
+    return all_projects[0]
+
+
+def get_dummy_page_json(version, *args, **kwargs):
+    dummy_page_json = DUMMY_PAGE_JSON
+    project_name = version.project.name
+    return dummy_page_json.get(project_name)
+
+
+@pytest.fixture(autouse=True)
+def mock_parse_json(mocker):
+
+    # patch the function from `projects.tasks` because it has been point to there
+    # http://www.voidspace.org.uk/python/mock/patch.html#where-to-patch
+    mocked_function = mocker.patch('readthedocs.projects.tasks.process_all_json_files')
+    mocked_function.side_effect = get_dummy_page_json

readthedocs/search/tests/data/docs/story.json

@@ -0,0 +1,32 @@
+{
+  "content": "Philosophy\nRead the Docs is Open Source software. We have licensed the code base as MIT, which provides almost no restrictions on the use of the code.\nHowever, as a project there are things that we care about more than others. We built Read the Docs to support  in the Open Source community. The code is open for people to contribute to, so that they may build features into https://readthedocs.org that they want. We also believe sharing the code openly is a valuable learning tool, especially for demonsrating how to collaborate and maintain an enormous website.\nOfficial Support\nThe time of the core developers of Read the Docs is limited. We provide official support for the following things:\nLocal development on the Python code base\nUsage of https://readthedocs.org for Open Source projects\nBug fixes in the code base, as it applies to running it on https://readthedocs.org\nUnsupported\nThere are use cases that we don\u2019t support, because it doesn\u2019t further our goal of promoting  in the Open Source Community.\nWe do not support:\nSpecific usage of Sphinx and Mkdocs, that don\u2019t affect our hosting\nCustom s of Read the Docs at your company\n of Read the Docs on other platforms\nAny  issues outside of the Read the Docs Python Code\nRationale\nRead the Docs was founded to improve  in the Open Source Community. We fully recognize and allow the code to be used for internal installs at companies, but we will not spend our time supporting it. Our time is limited, and we want to spend it on the mission that we set out to originally support.\nIf you feel strongly about installing Read the Docs internal to a company, we will happily link to third party resources on this topic. Please open an issue with a proposal if you want to take on this task.",
+  "headers": [
+    "Official Support",
+    "Unsupported",
+    "Rationale"
+  ],
+  "title": "Philosophy",
+  "sections": [
+    {
+      "content": "\nRead the Docs is Open Source software.\nWe have <a class=\"reference external\" href=\"https://github.com/rtfd/readthedocs.org/blob/master/LICENSE\">licensed</a> the code base as MIT,\nwhich provides almost no restrictions on the use of the code.\n\nHowever,\nas a project there are things that we care about more than others.\nWe built Read the Docs to support  in the Open Source community.\nThe code is open for people to contribute to,\nso that they may build features into <a class=\"reference external\" href=\"https://readthedocs.org\">https://readthedocs.org</a> that they want.\nWe also believe sharing the code openly is a valuable learning tool,\nespecially for demonsrating how to collaborate and maintain an enormous website.\n",
+      "id": "read-the-docs-open-source-philosophy",
+      "title": "Read the Docs Open Source Philosophy"
+    },
+    {
+      "content": "\n<h2>Official Support<a class=\"headerlink\" href=\"#official-support\" title=\"Permalink to this headline\">\u00b6</a></h2>\n<p>The time of the core developers of Read the Docs is limited.\nWe provide official support for the following things:</p>\n<ul class=\"simple\">\n<li>Local development on the Python code base</li>\n<li>Usage of <a class=\"reference external\" href=\"https://readthedocs.org\">https://readthedocs.org</a> for Open Source projects</li>\n<li>Bug fixes in the code base, as it applies to running it on <a class=\"reference external\" href=\"https://readthedocs.org\">https://readthedocs.org</a></li>\n</ul>\n",
+      "id": "official-support",
+      "title": "Official Support"
+    },
+    {
+      "content": "\n<h2>Unsupported<a class=\"headerlink\" href=\"#unsupported\" title=\"Permalink to this headline\">\u00b6</a></h2>\n<p>There are use cases that we don\u2019t support,\nbecause it doesn\u2019t further our goal of promoting  in the Open Source Community.</p>\n<p>We do not support:</p>\n<ul class=\"simple\">\n<li>Specific usage of Sphinx and Mkdocs, that don\u2019t affect our hosting</li>\n<li>Custom  of Read the Docs at your company</li>\n<li> of Read the Docs on other platforms</li>\n<li>Any  issues outside of the Read the Docs Python Code</li>\n</ul>\n",
+      "id": "unsupported",
+      "title": "Unsupported"
+    },
+    {
+      "content": "\n<h2>Rationale<a class=\"headerlink\" href=\"#rationale\" title=\"Permalink to this headline\">\u00b6</a></h2>\n<p>Read the Docs was founded to improve  in the Open Source Community.\nWe fully recognize and allow the code to be used for internal installs at companies,\nbut we will not spend our time supporting it.\nOur time is limited,\nand we want to spend it on the mission that we set out to originally support.</p>\n<p>If you feel strongly about installing Read the Docs internal to a company,\nwe will happily link to third party resources on this topic.\nPlease open an issue with a proposal if you want to take on this task.</p>\n",
+      "id": "rationale",
+      "title": "Rationale"
+    }
+  ],
+  "path": "open-source-philosophy"
+}

readthedocs/search/tests/data/docs/wiping.json

@@ -0,0 +1,15 @@
+{
+  "content": "Wiping a Build Environment\nSometimes it happen that your Builds start failing because the build environment where the  is created is stale or broken. This could happen for a couple of different reasons like pip not upgrading a package properly or a corrupted cached Python package.\nIn any of these cases (and many others), the solution could be just wiping out the existing build environment files and allow Read the Docs to create a new fresh one.\nFollow these steps to wipe the build environment:\nGo to Versions\nClick on the Edit button of the version you want to wipe on the right side of the page\nGo to the bottom of the page and click the wipe link, next to the \u201cSave\u201d button\nNote\nBy wiping the  build environment, all the rst, md, and code files associated with it will be removed but not the  already built (HTML and PDF files). Your  will still online after wiping the build environment.\nNow you can re-build the version with a fresh build environment!",
+  "headers": [
+    "Wiping a Build Environment"
+  ],
+  "title": "Wiping a Build Environment",
+  "sections": [
+    {
+      "content": "\nSometimes it happen that your Builds start failing because the build\nenvironment where the  is created is stale or\nbroken. This could happen for a couple of different reasons like <code class=\"xref py py-obj docutils literal notranslate\"><span class=\"pre\">pip</span></code>\nnot upgrading a package properly or a corrupted cached Python package.\n\nIn any of these cases (and many others), the solution could be just\nwiping out the existing build environment files and allow Read the\nDocs to create a new fresh one.\n\nFollow these steps to wipe the build environment:\n\n\n<li>Go to <strong>Versions</strong></li>\n<li>Click on the <strong>Edit</strong> button of the version you want to wipe on the\nright side of the page</li>\n<li>Go to the bottom of the page and click the <strong>wipe</strong> link, next to\nthe \u201cSave\u201d button</li>\n\n\n\n<p class=\"first admonition-title\">Note</p>\n<p class=\"last\">By wiping the  build environment, all the <code class=\"xref py py-obj docutils literal notranslate\"><span class=\"pre\">rst</span></code>, <code class=\"xref py py-obj docutils literal notranslate\"><span class=\"pre\">md</span></code>,\nand code files associated with it will be removed but not the\n already built (<code class=\"xref py py-obj docutils literal notranslate\"><span class=\"pre\">HTML</span></code> and <code class=\"xref py py-obj docutils literal notranslate\"><span class=\"pre\">PDF</span></code> files). Your\n will still online after wiping the build environment.</p>\n\n\nNow you can re-build the version with a fresh build environment!\n",
+      "id": "wiping-a-build-environment",
+      "title": "Wiping a Build Environment"
+    }
+  ],
+  "path": "guides/wipe-environment"
+}