perf: reduce import overhead on background jobs (backport #25459) (#2…

…5460)

* perf: avoid importing posthog if not required

Avoids 7MB of overhead and cleanup costs for each background job 🎉

(cherry picked from commit 3226717)

* perf: preload modules in worker pool

(cherry picked from commit 0d6ec13)

* perf: import sentinal locally

(cherry picked from commit b7a5884)

---------

Co-authored-by: Ankush Menat <ankush@frappe.io>

frappe/hooks.py

@@ -446,7 +446,7 @@
 	"frappe.recorder.dump",
 	"frappe.monitor.stop",
 	"frappe.utils.file_lock.release_document_locks",
-	"frappe.utils.telemetry.flush",
+	"frappe.utils.background_jobs.flush_telemetry",
 ]
 
 extend_bootinfo = [

frappe/utils/background_jobs.py

@@ -4,6 +4,7 @@
 import time
 from collections import defaultdict
 from collections.abc import Callable
+from contextlib import suppress
 from functools import lru_cache
 from typing import Any, NoReturn
 from uuid import uuid4
@@ -307,6 +308,17 @@ def start_worker_pool(
 	"""
 
 	_start_sentry()
+
+	# If gc.freeze is done then importing modules before forking allows us to share the memory
+	import frappe.database.query  # sqlparse and indirect imports
+	import frappe.query_builder  # pypika
+	import frappe.utils.data  # common utils
+	import frappe.utils.safe_exec
+	import frappe.utils.typing_validations  # any whitelisted method uses this
+	import frappe.website.path_resolver  # all the page types and resolver
+
+	# end: module pre-loading
+
 	_freeze_gc()
 
 	with frappe.init_site():
@@ -573,6 +585,16 @@ def truncate_failed_registry(job, connection, type, value, traceback):
 				job_obj and fail_registry.remove(job_obj, delete_job=True)
 
 
+def flush_telemetry():
+	"""Forcefully flush pending events.
+
+	This is required in context of background jobs where process might die before posthog gets time
+	to push events."""
+	ph = getattr(frappe.local, "posthog", None)
+	with suppress(Exception):
+		ph and ph.flush()
+
+
 def _start_sentry():
 	sentry_dsn = os.getenv("FRAPPE_SENTRY_DSN")
 	if not sentry_dsn:

frappe/utils/redis_wrapper.py

@@ -5,7 +5,6 @@
 
 import redis
 from redis.commands.search import Search
-from redis.sentinel import Sentinel
 
 import frappe
 from frappe.utils import cstr
@@ -315,6 +314,8 @@ def get_sentinel_connection(
 	master_username=None,
 	master_password=None,
 ):
+	from redis.sentinel import Sentinel
+
 	sentinel_kwargs = {}
 	if sentinel_username:
 		sentinel_kwargs["username"] = sentinel_username

frappe/utils/telemetry.py

@@ -60,16 +60,6 @@ def capture(event, app, **kwargs):
 		ph and ph.capture(distinct_id=frappe.local.site, event=f"{app}_{event}", **kwargs)
 
 
-def flush():
-	"""Forcefully flush pending events.
-
-	This is required in context of background jobs where process might die before posthog gets time
-	to push events."""
-	ph: Posthog = getattr(frappe.local, "posthog", None)
-	with suppress(Exception):
-		ph and ph.flush()
-
-
 def capture_doc(doc, action):
 	with suppress(Exception):
 		age = site_age()