fix solr 413 request too big for big batches

Allura/allura/lib/solr.py

@@ -20,8 +20,10 @@
 import logging
 
 from tg import config
+from webob.exc import HTTPRequestEntityTooLarge
 from paste.deploy.converters import asbool
 import pysolr
+from pysolr import SolrError
 import six
 
 from allura.lib.helpers import shlex_split
@@ -102,7 +104,13 @@ def add(self, *args, **kw):
             kw['commitWithin'] = self.commitWithin
         responses = []
         for solr in self.push_pool:
-            responses.append(solr.add(*args, **kw))
+            try:
+                responses.append(solr.add(*args, **kw))
+            except SolrError as e:
+                if '(HTTP 413)' in str(e):
+                    raise HTTPRequestEntityTooLarge() from e
+                else:
+                    raise
         return responses
 
     def delete(self, *args, **kw):

Allura/allura/tasks/index_tasks.py

@@ -14,20 +14,26 @@
 #       KIND, either express or implied.  See the License for the
 #       specific language governing permissions and limitations
 #       under the License.
+from __future__ import annotations
 
 import sys
 import logging
 from contextlib import contextmanager
+import typing
 
 from tg import app_globals as g
 from tg import tmpl_context as c
+from webob.exc import HTTPRequestEntityTooLarge
 
 from allura.lib import helpers as h
 from allura.lib.decorators import task
 from allura.lib.exceptions import CompoundError
 from allura.lib.solr import make_solr_from_config
 import six
 
+if typing.TYPE_CHECKING:
+    import pysolr
+
 
 log = logging.getLogger(__name__)
 
@@ -126,7 +132,20 @@ def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
             except Exception:
                 log.error('Error indexing artifact %s', ref._id)
                 exceptions.append(sys.exc_info())
-        __get_solr(solr_hosts).add(solr_updates)
+
+        def _add_artifact(solr: pysolr.Solr, artifacts: list):
+            try:
+                solr.add(artifacts)
+            except HTTPRequestEntityTooLarge:
+                if len(artifacts) > 1:
+                    log.warning(f"Solr.add raised HTTPRequestEntityTooLarge. Splitting {len(artifacts)} updates into two batches.")
+                    _add_artifact(solr, artifacts[:len(artifacts) // 2])
+                    _add_artifact(solr, artifacts[len(artifacts) // 2:])
+                else:
+                    log.info("Solr.add raised HTTPRequestEntityTooLarge but there is only one artifact. Raising exception.")
+                    raise
+
+        _add_artifact(__get_solr(solr_hosts), solr_updates)
 
     if len(exceptions) == 1:
         raise exceptions[0][1].with_traceback(exceptions[0][2])

Allura/allura/tests/unit/test_solr.py

@@ -19,6 +19,9 @@
 
 import mock
 from markupsafe import Markup
+from pysolr import SolrError
+import pytest
+from webob.exc import HTTPRequestEntityTooLarge
 
 from allura.lib import helpers as h
 from allura.tests import decorators as td
@@ -61,6 +64,15 @@ def test_add(self, pysolr):
                            commitWithin='10000', somekw='value')] * 2
         pysolr.Solr().add.assert_has_calls(calls)
 
+    @mock.patch('allura.lib.solr.pysolr')
+    def test_add_too_big(self, pysolr):
+        pysolr.Solr.return_value.add.side_effect = \
+            SolrError("Solr responded with an error (HTTP 413): [Reason: None]")
+        servers = ['server1', 'server2']
+        solr = Solr(servers, commit=False, commitWithin='10000')
+        with pytest.raises(HTTPRequestEntityTooLarge):
+            solr.add('foo', commit=True, commitWithin=None)
+
     @mock.patch('allura.lib.solr.pysolr')
     def test_delete(self, pysolr):
         servers = ['server1', 'server2']