From b921e6b833f573973d93293b49aa8d75fa6de4c7 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Fri, 8 Nov 2019 14:28:53 +0100 Subject: [PATCH 1/3] Fix issues with sync'ing being slow or timing out #5614 --- kalite/packages/bundled/fle_utils/internet/decorators.py | 6 +++++- kalite/packages/bundled/securesync/api_client.py | 3 ++- kalite/packages/bundled/securesync/engine/api_client.py | 8 +++++++- kalite/packages/bundled/securesync/engine/utils.py | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/kalite/packages/bundled/fle_utils/internet/decorators.py b/kalite/packages/bundled/fle_utils/internet/decorators.py index 19d168f491..90370c74d4 100644 --- a/kalite/packages/bundled/fle_utils/internet/decorators.py +++ b/kalite/packages/bundled/fle_utils/internet/decorators.py @@ -28,7 +28,11 @@ def api_handle_error_with_json_wrapper_fn(request, *args, **kwargs): except Http404: raise except Exception as e: - logger.error("Error in JSON view: {}".format(request.path)) + if hasattr(request, 'path'): + logger.error("Error in JSON view: {}".format(request.path)) + else: + logger.error("Error in JSON view. Request also has no attribute 'path', so not sure how this got called.") + logger.error(request) traceback.print_exc() return JsonResponseMessageError(_("Unexpected error: %(err)s") % {"err": e}, status=500) return api_handle_error_with_json_wrapper_fn diff --git a/kalite/packages/bundled/securesync/api_client.py b/kalite/packages/bundled/securesync/api_client.py index 9608f19bbf..554ff2ac80 100644 --- a/kalite/packages/bundled/securesync/api_client.py +++ b/kalite/packages/bundled/securesync/api_client.py @@ -20,8 +20,9 @@ def __init__(self, host=None, require_trusted=True, verbose=True): self.url = settings.CENTRAL_SERVER_URL else: self.url = "%s://%s/" % (settings.SECURESYNC_PROTOCOL, settings.CENTRAL_SERVER_HOST) + self.parsed_url = urllib2.urlparse.urlparse(self.url) else: - parsed_url = urllib2.urlparse.urlparse(host) + self.parsed_url = urllib2.urlparse.urlparse(host) self.url = "%s://%s" % (self.parsed_url.scheme, self.parsed_url.netloc) self.require_trusted = require_trusted diff --git a/kalite/packages/bundled/securesync/engine/api_client.py b/kalite/packages/bundled/securesync/engine/api_client.py index bc92b19ec3..ddd4e930a3 100644 --- a/kalite/packages/bundled/securesync/engine/api_client.py +++ b/kalite/packages/bundled/securesync/engine/api_client.py @@ -21,11 +21,14 @@ class SyncClient(BaseClient): session = None def post(self, path, payload={}, *args, **kwargs): + kwargs.setdefault("timeout", 120) if self.session and self.session.client_nonce: payload["client_nonce"] = self.session.client_nonce return super(SyncClient, self).post(path, payload, *args, **kwargs) def get(self, path, payload={}, *args, **kwargs): + # Set a high timeout since the server can be heavily loaded at times + kwargs.setdefault("timeout", 120) if self.session and self.session.client_nonce: payload["client_nonce"] = self.session.client_nonce # add a random parameter to ensure the request is not cached @@ -145,7 +148,10 @@ def close_session(self): def get_server_device_counters(self): r = self.get("device/counters") - data = json.loads(r.content or "{}") + try: + data = json.loads(r.content or "{}") + except ValueError: + raise Exception("Did not receive proper JSON data from server. URL: {}\n\nActual data:\n\n{}".format(r.url, r.content)) if "error" in data: raise Exception("Server error in retrieving counters: " + data["error"]) return data.get("device_counters", {}) diff --git a/kalite/packages/bundled/securesync/engine/utils.py b/kalite/packages/bundled/securesync/engine/utils.py index cd132d096c..cbcf6d6fb2 100644 --- a/kalite/packages/bundled/securesync/engine/utils.py +++ b/kalite/packages/bundled/securesync/engine/utils.py @@ -64,7 +64,7 @@ def get_device_counters(**kwargs): # The local device may have items that haven't incremented the device counter, # but instead have deferred until sync time. Include those! - if device.is_own_device(): + if not getattr(settings, 'CENTRAL_SERVER', False) and device.is_own_device(): cnt = 0 for Model in _syncing_models: cnt += Model.all_objects.filter(Q(counter__isnull=True) | Q(signature__isnull=True)).count() # include deleted records From 2871915abfd944807fa9528e4a3249d5fe7b0c0a Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Fri, 8 Nov 2019 14:29:38 +0100 Subject: [PATCH 2/3] Ensure an understandable log by logging django requests --- kalite/settings/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kalite/settings/base.py b/kalite/settings/base.py index 06d6325148..2ac89bfda5 100644 --- a/kalite/settings/base.py +++ b/kalite/settings/base.py @@ -106,8 +106,8 @@ 'level': 'INFO', }, 'django.request': { - 'handlers': ['null'], - 'level': 'DEBUG', + 'handlers': ['console', 'file'], + 'level': logging.ERROR, 'propagate': False, }, 'kalite': { @@ -132,7 +132,7 @@ }, 'cherrypy.access': { 'handlers': ['console', 'file'], - 'level': LOGGING_LEVEL, + 'level': logging.ERROR, 'propagate': False, }, 'cherrypy.error': { @@ -142,7 +142,7 @@ }, '': { 'handlers': ['console', 'file'], - 'level': 'INFO', + 'level': logging.INFO, 'propagate': False, }, } From e65b7ab33c7f8b98adeb90d0854297fe26256453 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Fri, 8 Nov 2019 14:33:25 +0100 Subject: [PATCH 3/3] Add release notes --- docs/installguide/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installguide/release_notes.rst b/docs/installguide/release_notes.rst index 8f167c3a86..b55cd643c1 100644 --- a/docs/installguide/release_notes.rst +++ b/docs/installguide/release_notes.rst @@ -15,6 +15,8 @@ Bug fixes ^^^^^^^^^ * Fix rare ``GEOSException`` on systems with libgeos :url-issue:`5592` +* Sync'ing timeouts fixed :url-issue:`5615` +* Platform-specific ``*.pyc`` artifacts found in distributed installers, likely harmless :url-issue:`5611` 0.17.5 ------