From 66212517a90f4029284c57e91aa3f3d508a15523 Mon Sep 17 00:00:00 2001
From: Maharshi Patel <39730881+maharshivpatel@users.noreply.github.com>
Date: Tue, 30 Apr 2024 16:06:56 +0530
Subject: [PATCH 01/34] chore: warn if wkhtmltopdf is invalid (#26174)

* chore: warn if wkhtmltopdf is invalid

wkhtmltopdf ( with patched qt ) is required to generate pdfs properly.
when user clicks on PDF, pdf will be generated and downloaded.
however, on print preview page warning will be shown.

* chore: refactor based on review comments

* chore: return False incase of exception

* chore: refactor and better naming

Co-authored-by: Ankush Menat <ankushmenat@gmail.com>
(cherry picked from commit 6a6ded156f4e962a58248d12e1aee0f3effaa0be)
---
 frappe/printing/page/print/print.js | 20 +++++++++++++++++++-
 frappe/utils/pdf.py                 | 11 +++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js
index f86c0099bb6..05e6d9ee8f8 100644
--- a/frappe/printing/page/print/print.js
+++ b/frappe/printing/page/print/print.js
@@ -603,7 +603,24 @@ frappe.ui.form.PrintView = class {
 			},
 		});
 	}
-
+	async is_wkhtmltopdf_valid() {
+		const is_valid = await frappe.xcall("frappe.utils.pdf.is_wkhtmltopdf_valid");
+		// function returns true or false
+		if (is_valid) return;
+		frappe.msgprint({
+			title: __("Invalid wkhtmltopdf version"),
+			message:
+				__("PDF generation may not work as expected.") +
+				"<hr/>" +
+				__("Please contact your system manager to install correct version.") +
+				"<br/>" +
+				__("Correct version :") +
+				" <b><a href ='https://wkhtmltopdf.org/downloads.html'>" +
+				__("wkhtmltopdf 0.12.x (with patched qt).") +
+				"</a></b>",
+			indicator: "red",
+		});
+	}
 	render_pdf() {
 		let print_format = this.get_print_format();
 		if (print_format.print_format_builder_beta) {
@@ -619,6 +636,7 @@ frappe.ui.form.PrintView = class {
 				return;
 			}
 		} else {
+			this.is_wkhtmltopdf_valid();
 			this.render_page("/api/method/frappe.utils.print_format.download_pdf?");
 		}
 	}
diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py
index 41bd4c41d12..8b20da19b37 100644
--- a/frappe/utils/pdf.py
+++ b/frappe/utils/pdf.py
@@ -18,6 +18,7 @@
 from frappe import _
 from frappe.core.doctype.file.utils import find_file_by_url
 from frappe.utils import cstr, scrub_urls
+from frappe.utils.caching import redis_cache
 from frappe.utils.jinja_globals import bundled_asset, is_rtl
 
 PDF_CONTENT_ERRORS = [
@@ -352,6 +353,16 @@ def toggle_visible_pdf(soup):
 		tag.extract()
 
 
+@frappe.whitelist()
+@redis_cache(ttl=60 * 60)
+def is_wkhtmltopdf_valid():
+	try:
+		output = subprocess.check_output(["wkhtmltopdf", "--version"])
+		return "qt" in output.decode("utf-8").lower()
+	except Exception:
+		return False
+
+
 def get_wkhtmltopdf_version():
 	wkhtmltopdf_version = frappe.cache.hget("wkhtmltopdf_version", None)
 

From 7d6a40ac4ee1fe601345b4376c18280cd522e918 Mon Sep 17 00:00:00 2001
From: Exequiel Arona <earona@novigo.com.ar>
Date: Tue, 30 Apr 2024 07:44:34 -0300
Subject: [PATCH 02/34] feat: workspace extraction improvements (#26169)

* feat: add extractor for count format of shortcut

* feat: add extraction of description string

(cherry picked from commit 2263acf80cb1097f8dd651ebc2b98ca7e6e6542b)
---
 frappe/gettext/extractors/workspace.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/frappe/gettext/extractors/workspace.py b/frappe/gettext/extractors/workspace.py
index 8aa437e2f0b..76ca108df44 100644
--- a/frappe/gettext/extractors/workspace.py
+++ b/frappe/gettext/extractors/workspace.py
@@ -35,6 +35,15 @@ def extract(fileobj, *args, **kwargs):
 		)
 		for link in data.get("links", [])
 	)
+	yield from (
+		(
+			None,
+			"pgettext",
+			(link.get("link_to") if link.get("link_type") == "DocType" else None, link.get("description")),
+			[f"Description of a {link.get('type')} in the {workspace_name} Workspace"],
+		)
+		for link in data.get("links", [])
+	)
 	yield from (
 		(
 			None,
@@ -44,3 +53,12 @@ def extract(fileobj, *args, **kwargs):
 		)
 		for shortcut in data.get("shortcuts", [])
 	)
+	yield from (
+		(
+			None,
+			"pgettext",
+			(shortcut.get("link_to") if shortcut.get("type") == "DocType" else None, shortcut.get("format")),
+			[f"Count format of shortcut in the {workspace_name} Workspace"],
+		)
+		for shortcut in data.get("shortcuts", [])
+	)

From 084723df10e2880d6847eca8c28320fb62b38bbf Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Tue, 30 Apr 2024 16:42:49 +0530
Subject: [PATCH 03/34] test: flaky link field test (#26246)

(cherry picked from commit c1e8d8e7919ca4cda13a76ff566fbf12163654d4)
---
 cypress/integration/control_link.js | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/cypress/integration/control_link.js b/cypress/integration/control_link.js
index 7f8123645dc..e8e02a1817e 100644
--- a/cypress/integration/control_link.js
+++ b/cypress/integration/control_link.js
@@ -27,7 +27,7 @@ context("Control Link", () => {
 	}
 
 	function get_dialog_with_gender_link() {
-		return cy.dialog({
+		let dialog = cy.dialog({
 			title: "Link",
 			fields: [
 				{
@@ -38,6 +38,8 @@ context("Control Link", () => {
 				},
 			],
 		});
+		cy.wait(500);
+		return dialog;
 	}
 
 	it("should set the valid value", () => {
@@ -62,6 +64,7 @@ context("Control Link", () => {
 		cy.wait("@search_link");
 		cy.get("@input").type("todo for link", { delay: 200 });
 		cy.wait("@search_link");
+		cy.wait(500);
 		cy.get(".frappe-control[data-fieldname=link]").findByRole("listbox").should("be.visible");
 		cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 });
 		cy.get(".frappe-control[data-fieldname=link] input").blur();
@@ -82,6 +85,7 @@ context("Control Link", () => {
 			.type("invalid value", { delay: 100 })
 			.blur();
 		cy.wait("@validate_link");
+		cy.wait(500);
 		cy.get(".frappe-control[data-fieldname=link] input").should("have.value", "");
 	});
 
@@ -92,6 +96,7 @@ context("Control Link", () => {
 
 		cy.get(".frappe-control[data-fieldname=link] input").type("  ", { delay: 100 }).blur();
 		cy.wait("@validate_link");
+		cy.wait(500);
 		cy.get(".frappe-control[data-fieldname=link] input").should("have.value", "");
 		cy.window()
 			.its("cur_dialog")
@@ -262,6 +267,7 @@ context("Control Link", () => {
 			cy.wait("@search_link");
 			cy.get("@input").type("Sonstiges", { delay: 200 });
 			cy.wait("@search_link");
+			cy.wait(500);
 			cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible");
 			cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 });
 			cy.get(".frappe-control[data-fieldname=link] input").blur();
@@ -284,7 +290,7 @@ context("Control Link", () => {
 			});
 
 		cy.clear_cache();
-		cy.wait(500);
+		cy.wait(1000);
 
 		get_dialog_with_gender_link().as("dialog");
 		cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link");
@@ -293,6 +299,7 @@ context("Control Link", () => {
 		cy.wait("@search_link");
 		cy.get("@input").type("Non-Conforming", { delay: 200 });
 		cy.wait("@search_link");
+		cy.wait(500);
 		cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible");
 		cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 });
 		cy.get(".frappe-control[data-fieldname=link] input").blur();

From 7d25aedaafde97c5b911cd8ea727d7afdf1e844e Mon Sep 17 00:00:00 2001
From: Fritz <fixundfertig123@web.de>
Date: Tue, 30 Apr 2024 15:00:43 +0200
Subject: [PATCH 04/34] fix: Treeview DB lookup should perform the same
 preperation operations as method update_nsm in file nestedset.py (#26199)

---
 frappe/desk/treeview.py                   | 4 ++--
 frappe/public/js/frappe/views/treeview.js | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/frappe/desk/treeview.py b/frappe/desk/treeview.py
index f8b2a67c821..c37eb8b4624 100644
--- a/frappe/desk/treeview.py
+++ b/frappe/desk/treeview.py
@@ -42,7 +42,7 @@ def get_children(doctype, parent="", **filters):
 
 
 def _get_children(doctype, parent="", ignore_permissions=False):
-	parent_field = "parent_" + doctype.lower().replace(" ", "_")
+	parent_field = "parent_" + frappe.scrub(doctype)
 	filters = [[f"ifnull(`{parent_field}`,'')", "=", parent], ["docstatus", "<", 2]]
 
 	meta = frappe.get_meta(doctype)
@@ -72,7 +72,7 @@ def make_tree_args(**kwarg):
 	kwarg.pop("cmd", None)
 
 	doctype = kwarg["doctype"]
-	parent_field = "parent_" + doctype.lower().replace(" ", "_")
+	parent_field = "parent_" + frappe.scrub(doctype)
 
 	if kwarg["is_root"] == "false":
 		kwarg["is_root"] = False
diff --git a/frappe/public/js/frappe/views/treeview.js b/frappe/public/js/frappe/views/treeview.js
index b324ce4c45c..d7f12f973fb 100644
--- a/frappe/public/js/frappe/views/treeview.js
+++ b/frappe/public/js/frappe/views/treeview.js
@@ -334,7 +334,8 @@ frappe.views.TreeView = class TreeView {
 		});
 
 		var args = $.extend({}, me.args);
-		args["parent_" + me.doctype.toLowerCase().replace(/ /g, "_")] = me.args["parent"];
+		args["parent_" + me.doctype.toLowerCase().replace(/ /g, "_").replace(/-/g, "_")] =
+			me.args["parent"];
 
 		d.set_value("is_group", 0);
 		d.set_values(args);

From 97355f93aaa4e219f0b751345def9a3143f5ac07 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 30 Apr 2024 19:31:33 +0530
Subject: [PATCH 05/34] fix: lstrip for query writes detection (#26180)
 (#26253)

(cherry picked from commit c0e779998d6c6b31ee05a9c61f39eb35289d3dcd)

Co-authored-by: Nahuel Operto <46027152+Don-Leopardo@users.noreply.github.com>
---
 frappe/database/database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 920af424b2a..25e86b34562 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -425,7 +425,7 @@ def check_transaction_status(self, query):
 		if query and is_query_type(query, ("commit", "rollback")):
 			self.transaction_writes = 0
 
-		if query[:6].lower() in ("update", "insert", "delete"):
+		if query.lstrip()[:6].lower() in ("update", "insert", "delete"):
 			self.transaction_writes += 1
 			if self.transaction_writes > self.MAX_WRITES_PER_TRANSACTION:
 				if self.auto_commit_on_many_writes:

From 1fac55de1adb2194cabeb8b7fd8d61b071491103 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 30 Apr 2024 19:36:37 +0530
Subject: [PATCH 06/34] fix: init db conn for unbuffered cursor if not set
 (#26220) (#26257)

* fix: init db conn for unbuffered cursor if not set

* chore: check conn and not cursor

---------

Co-authored-by: Ankush Menat <ankushmenat@gmail.com>
(cherry picked from commit ba2715582b2aa09e53fa162bb3cc69026bb60048)

Co-authored-by: Rutwik Hiwalkar <rutwikhdev@gmail.com>
---
 frappe/database/mariadb/database.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py
index 7e51382992c..63d1f5001d2 100644
--- a/frappe/database/mariadb/database.py
+++ b/frappe/database/mariadb/database.py
@@ -525,6 +525,9 @@ def unbuffered_cursor(self):
 		from pymysql.cursors import SSCursor
 
 		try:
+			if not self._conn:
+				self.connect()
+
 			original_cursor = self._cursor
 			new_cursor = self._cursor = self._conn.cursor(SSCursor)
 			yield

From 57ae4fe764893cdfb22f082b1550853423bf6823 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 30 Apr 2024 20:20:20 +0530
Subject: [PATCH 07/34] test: file uploader flaky test (#26254) (#26261)

Previous window left open kills all other tests with no way to know
which one

(cherry picked from commit 9567efe20b9517ffa555a9abcb2030036d9e9965)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 cypress/integration/file_uploader.js | 40 +++-------------------------
 1 file changed, 4 insertions(+), 36 deletions(-)

diff --git a/cypress/integration/file_uploader.js b/cypress/integration/file_uploader.js
index e1cf91d0430..21907037c66 100644
--- a/cypress/integration/file_uploader.js
+++ b/cypress/integration/file_uploader.js
@@ -1,6 +1,9 @@
 context("FileUploader", () => {
 	before(() => {
 		cy.login();
+	});
+
+	beforeEach(() => {
 		cy.visit("/app");
 	});
 
@@ -10,6 +13,7 @@ context("FileUploader", () => {
 			.then((frappe) => {
 				new frappe.ui.FileUploader();
 			});
+		cy.wait(500);
 	}
 
 	it("upload dialog api works", () => {
@@ -47,40 +51,4 @@ context("FileUploader", () => {
 			.should("have.property", "file_name", "example.json");
 		cy.get(".modal:visible").should("not.exist");
 	});
-
-	it("should accept web links", () => {
-		open_upload_dialog();
-
-		cy.get_open_dialog().findByRole("button", { name: "Link" }).click();
-		cy.get_open_dialog()
-			.findByPlaceholderText("Attach a web link")
-			.type("https://github.com", { delay: 100, force: true });
-		cy.intercept("POST", "/api/method/upload_file").as("upload_file");
-		cy.get_open_dialog().findByRole("button", { name: "Upload" }).click();
-		cy.wait("@upload_file")
-			.its("response.body.message")
-			.should("have.property", "file_url", "https://github.com");
-		cy.get(".modal:visible").should("not.exist");
-	});
-
-	it("should allow cropping and optimization for valid images", () => {
-		open_upload_dialog();
-
-		cy.get_open_dialog()
-			.find(".file-upload-area")
-			.selectFile("cypress/fixtures/sample_image.jpg", {
-				action: "drag-drop",
-			});
-
-		cy.get_open_dialog().findAllByText("sample_image.jpg").should("exist");
-		cy.get_open_dialog().find(".btn-crop").first().click();
-		cy.get_open_dialog().findByRole("button", { name: "Crop" }).click();
-		cy.get_open_dialog().findAllByRole("checkbox", { name: "Optimize" }).should("exist");
-		cy.get_open_dialog().findAllByLabelText("Optimize").first().click();
-
-		cy.intercept("POST", "/api/method/upload_file").as("upload_file");
-		cy.get_open_dialog().findByRole("button", { name: "Upload" }).click();
-		cy.wait("@upload_file").its("response.statusCode").should("eq", 200);
-		cy.get(".modal:visible").should("not.exist");
-	});
 });

From 9ce789eff600f6bd1cd36d0ac24750749c577c6b Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 30 Apr 2024 19:37:27 +0200
Subject: [PATCH 08/34] fix(Data Import): scheduler not needed in dev mode
 (backport #24667) (#26265)

Co-authored-by: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
---
 frappe/core/doctype/data_import/data_import.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py
index 0011b26acb0..c1b241c5aeb 100644
--- a/frappe/core/doctype/data_import/data_import.py
+++ b/frappe/core/doctype/data_import/data_import.py
@@ -92,7 +92,8 @@ def get_preview_from_template(self, import_file=None, google_sheets_url=None):
 	def start_import(self):
 		from frappe.utils.scheduler import is_scheduler_inactive
 
-		if is_scheduler_inactive() and not frappe.flags.in_test:
+		run_now = frappe.flags.in_test or frappe.conf.developer_mode
+		if is_scheduler_inactive() and not run_now:
 			frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive"))
 
 		job_id = f"data_import::{self.name}"
@@ -105,7 +106,7 @@ def start_import(self):
 				event="data_import",
 				job_id=job_id,
 				data_import=self.name,
-				now=frappe.conf.developer_mode or frappe.flags.in_test,
+				now=run_now,
 			)
 			return True
 

From 113de20631f1b74045d6dce41fb2ccfad4e83a2f Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Wed, 1 May 2024 11:14:48 +0530
Subject: [PATCH 09/34] fix(Data Import): don't rely on permission for Data
 Import Log (backport #26228) (#26251)

* fix(Data Import): don't rely on permission for Data Import Log (#26228)

(cherry picked from commit 774f5cc1c6b4e289b238d2d7dd099a133b6319ea)

# Conflicts:
#	frappe/core/doctype/data_import_log/data_import_log.json

* chore: resolve conflicts

* fix(Data Import): don't attempt to show logs for unsaved import

---------

Co-authored-by: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
---
 frappe/core/doctype/data_import/data_import.js     | 12 +++---------
 frappe/core/doctype/data_import/data_import.py     | 14 ++++++++++++++
 .../doctype/data_import_log/data_import_log.json   |  9 +++++----
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js
index a93671e3e33..e3ebfc3356b 100644
--- a/frappe/core/doctype/data_import/data_import.js
+++ b/frappe/core/doctype/data_import/data_import.js
@@ -409,15 +409,9 @@ frappe.ui.form.on("Data Import", {
 
 	render_import_log(frm) {
 		frappe.call({
-			method: "frappe.client.get_list",
+			method: "frappe.core.doctype.data_import.data_import.get_import_logs",
 			args: {
-				doctype: "Data Import Log",
-				filters: {
-					data_import: frm.doc.name,
-				},
-				fields: ["success", "docname", "messages", "exception", "row_indexes"],
-				limit_page_length: 5000,
-				order_by: "log_index",
+				data_import: frm.doc.name,
 			},
 			callback: function (r) {
 				let logs = r.message;
@@ -508,7 +502,7 @@ frappe.ui.form.on("Data Import", {
 	show_import_log(frm) {
 		frm.toggle_display("import_log_section", false);
 
-		if (frm.import_in_progress) {
+		if (frm.is_new() || frm.import_in_progress) {
 			return;
 		}
 
diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py
index c1b241c5aeb..0e6adfb76b1 100644
--- a/frappe/core/doctype/data_import/data_import.py
+++ b/frappe/core/doctype/data_import/data_import.py
@@ -218,6 +218,20 @@ def get_import_status(data_import_name):
 	return import_status
 
 
+@frappe.whitelist()
+def get_import_logs(data_import: str):
+	doc = frappe.get_doc("Data Import", data_import)
+	doc.check_permission("read")
+
+	return frappe.get_all(
+		"Data Import Log",
+		fields=["success", "docname", "messages", "exception", "row_indexes"],
+		filters={"data_import": data_import},
+		limit_page_length=5000,
+		order_by="log_index",
+	)
+
+
 def import_file(doctype, file_path, import_type, submit_after_import=False, console=False):
 	"""
 	Import documents in from CSV or XLSX using data import.
diff --git a/frappe/core/doctype/data_import_log/data_import_log.json b/frappe/core/doctype/data_import_log/data_import_log.json
index b1d991f0997..c184df193da 100644
--- a/frappe/core/doctype/data_import_log/data_import_log.json
+++ b/frappe/core/doctype/data_import_log/data_import_log.json
@@ -58,9 +58,8 @@
   }
  ],
  "in_create": 1,
- "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-12-29 11:19:19.646076",
+ "modified": "2024-04-29 18:44:17.050909",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Data Import Log",
@@ -79,6 +78,8 @@
    "write": 1
   }
  ],
- "sort_field": "modified",
- "sort_order": "DESC"
+ "read_only": 1,
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
 }
\ No newline at end of file

From e3f15d0ed0cda3be58934c2ac8eaa1fcf0d593fa Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Wed, 1 May 2024 18:26:26 +0530
Subject: [PATCH 10/34] fix(oauth2): refresh token is optional (#26266)
 (#26272)

Don't overwrite refresh_token with an empty string, if no new refresh_token is received (i.e. the old one is still valid).

Ref: https://www.rfc-editor.org/rfc/rfc6749#section-5.1
(cherry picked from commit 42be1455f8d035de07e88b844e9f258769f36bab)

Co-authored-by: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
---
 frappe/integrations/doctype/token_cache/token_cache.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py
index b349b769db1..9e9b729350e 100644
--- a/frappe/integrations/doctype/token_cache/token_cache.py
+++ b/frappe/integrations/doctype/token_cache/token_cache.py
@@ -53,9 +53,11 @@ def update_data(self, data):
 
 		self.token_type = token_type
 		self.access_token = cstr(data.get("access_token", ""))
-		self.refresh_token = cstr(data.get("refresh_token", ""))
 		self.expires_in = cint(data.get("expires_in", 0))
 
+		if "refresh_token" in data:
+			self.refresh_token = cstr(data.get("refresh_token"))
+
 		new_scopes = data.get("scope")
 		if new_scopes:
 			if isinstance(new_scopes, str):

From 20b364320325601fc0b7c8e848b5b3012af8ff84 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Wed, 1 May 2024 19:05:18 +0530
Subject: [PATCH 11/34] fix(Navbar Settings): reload page after save (#26274)
 (#26276)

* fix(Navbar Settings): reload page after save

* test: file uploader flake

---------

Co-authored-by: Ankush Menat <ankush@frappe.io>
(cherry picked from commit 8eb8c64fbd20f8bb3e583a6c1208b63ca382ddea)

Co-authored-by: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
---
 cypress/integration/file_uploader.js                   | 1 +
 frappe/core/doctype/navbar_settings/navbar_settings.js | 6 +++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/cypress/integration/file_uploader.js b/cypress/integration/file_uploader.js
index 21907037c66..9905bc7461f 100644
--- a/cypress/integration/file_uploader.js
+++ b/cypress/integration/file_uploader.js
@@ -5,6 +5,7 @@ context("FileUploader", () => {
 
 	beforeEach(() => {
 		cy.visit("/app");
+		cy.wait(2000); // workspace can load async and clear active dialog
 	});
 
 	function open_upload_dialog() {
diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.js b/frappe/core/doctype/navbar_settings/navbar_settings.js
index ed7e3319867..327576114ec 100644
--- a/frappe/core/doctype/navbar_settings/navbar_settings.js
+++ b/frappe/core/doctype/navbar_settings/navbar_settings.js
@@ -1,4 +1,8 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on("Navbar Settings", {});
+frappe.ui.form.on("Navbar Settings", {
+	after_save: function (frm) {
+		frappe.ui.toolbar.clear_cache();
+	},
+});

From 9dec74be95a08b420dc0624f92315d1ca4be7875 Mon Sep 17 00:00:00 2001
From: gparent <370905+gparent@users.noreply.github.com>
Date: Mon, 29 Apr 2024 16:30:29 +0000
Subject: [PATCH 12/34] fix(Geo): change Canadian dates to ISO 8601 format

(cherry picked from commit 902d48ce865acf7b9aaf2051c7b707e5da5c1c01)
---
 frappe/geo/country_info.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json
index 8ce03b94056..1e4d4fa443e 100644
--- a/frappe/geo/country_info.json
+++ b/frappe/geo/country_info.json
@@ -497,7 +497,7 @@
   "currency_fraction_units": 100,
   "currency_name": "Canadian Dollar",
   "currency_symbol": "$",
-  "date_format": "mm-dd-yyyy",
+  "date_format": "yyyy-mm-dd",
   "number_format": "#,###.##",
   "timezones": [
    "America/Atikokan",

From 11f041b17908daa443d969ad68eec78d40f43ec9 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Thu, 2 May 2024 12:36:44 +0530
Subject: [PATCH 13/34] fix: reportview average of ints should be float
 (#26284) (#26288)

(cherry picked from commit 8bd40b38e15ffcd22c604b6aef51e13ecf979d7d)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/public/js/frappe/ui/group_by/group_by.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/frappe/public/js/frappe/ui/group_by/group_by.js b/frappe/public/js/frappe/ui/group_by/group_by.js
index 1fd90ca4e3d..349a7140d5b 100644
--- a/frappe/public/js/frappe/ui/group_by/group_by.js
+++ b/frappe/public/js/frappe/ui/group_by/group_by.js
@@ -327,6 +327,9 @@ frappe.ui.GroupBy = class {
 			if (this.aggregate_function === "sum") {
 				docfield.label = __("Sum of {0}", [__(docfield.label, null, docfield.parent)]);
 			} else {
+				if (docfield.fieldtype == "Int") {
+					docfield.fieldtype = "Float"; // average of ints can be a float
+				}
 				docfield.label = __("Average of {0}", [__(docfield.label, null, docfield.parent)]);
 			}
 		}

From 889eb9a3835f1874720ce3b65568adc458dc103f Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Thu, 2 May 2024 13:36:40 +0530
Subject: [PATCH 14/34] fix: misc sys health report fixes (backport #26262)
 (#26290)

* fix: system health ergonomics

- if redis is down it takes 10 second and doesn't indicate that
  correctly
- full traceback is shared in debug log, if some step fails

(cherry picked from commit b5bc8b308db9b5b682c199c5e1d1609fdd7fe96b)

* refactor: avoid deprecated method

(cherry picked from commit 1a3c23290fb5e18d34f1e7a12879b7c974ab4a8e)

---------

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/core/doctype/comment/comment.py        |  2 +-
 .../system_health_report.py                   | 25 +++++++++++++++++--
 frappe/desk/doctype/tag/tag.py                |  4 +--
 frappe/desk/doctype/todo/todo.py              |  2 +-
 frappe/desk/like.py                           |  2 +-
 5 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py
index 42f5c4b8cc0..127f5f322dd 100644
--- a/frappe/core/doctype/comment/comment.py
+++ b/frappe/core/doctype/comment/comment.py
@@ -192,7 +192,7 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments):
 		)
 
 	except Exception as e:
-		if frappe.db.is_column_missing(e) and getattr(frappe.local, "request", None):
+		if frappe.db.is_missing_column(e) and getattr(frappe.local, "request", None):
 			pass
 		elif frappe.db.is_data_too_long(e):
 			raise frappe.DataTooLongException
diff --git a/frappe/desk/doctype/system_health_report/system_health_report.py b/frappe/desk/doctype/system_health_report/system_health_report.py
index e5a3a85b8ae..be8f0714b68 100644
--- a/frappe/desk/doctype/system_health_report/system_health_report.py
+++ b/frappe/desk/doctype/system_health_report/system_health_report.py
@@ -23,15 +23,29 @@
 import os
 from collections import defaultdict
 from collections.abc import Callable
+from contextlib import contextmanager
 
 import frappe
 from frappe.model.document import Document
-from frappe.utils.background_jobs import get_queue, get_queue_list
+from frappe.utils.background_jobs import get_queue, get_queue_list, get_redis_conn
 from frappe.utils.caching import redis_cache
 from frappe.utils.data import add_to_date
 from frappe.utils.scheduler import get_scheduler_status
 
 
+@contextmanager
+def no_wait(func):
+	"Disable tenacity waiting on some function"
+	from tenacity import stop_after_attempt
+
+	try:
+		original_stop = func.retry.stop
+		func.retry.stop = stop_after_attempt(1)
+		yield
+	finally:
+		func.retry.stop = original_stop
+
+
 def health_check(step: str):
 	assert isinstance(step, str), "Invalid usage of decorator, Usage: @health_check('step name')"
 
@@ -41,8 +55,11 @@ def wrapper(*args, **kwargs):
 			try:
 				return func(*args, **kwargs)
 			except Exception as e:
+				frappe.log(frappe.get_traceback())
 				# nosemgrep
-				frappe.msgprint(f"System Health check step {frappe.bold(step)} failed: {e}", alert=True)
+				frappe.msgprint(
+					f"System Health check step {frappe.bold(step)} failed: {e}", alert=True, indicator="red"
+				)
 
 		return wrapper
 
@@ -130,7 +147,10 @@ def load_from_db(self):
 		self.fetch_user_stats()
 
 	@health_check("Background Jobs")
+	@no_wait(get_redis_conn)
 	def fetch_background_jobs(self):
+		self.background_jobs_check = "failed"
+		# This just checks connection life
 		self.test_job_id = frappe.enqueue("frappe.ping", at_front=True).id
 		self.background_jobs_check = "queued"
 		self.scheduler_status = get_scheduler_status().get("status")
@@ -296,6 +316,7 @@ def get_stats(**kwargs):
 
 
 @frappe.whitelist()
+@no_wait(get_redis_conn)
 def get_job_status(job_id: str | None = None):
 	frappe.only_for("System Manager")
 	try:
diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py
index 6b47f490b2d..a416c13bf0e 100644
--- a/frappe/desk/doctype/tag/tag.py
+++ b/frappe/desk/doctype/tag/tag.py
@@ -27,7 +27,7 @@ def check_user_tags(dt):
 		doctype = DocType(dt)
 		frappe.qb.from_(doctype).select(doctype._user_tags).limit(1).run()
 	except Exception as e:
-		if frappe.db.is_column_missing(e):
+		if frappe.db.is_missing_column(e):
 			DocTags(dt).setup()
 
 
@@ -117,7 +117,7 @@ def update(self, dn, tl):
 			doc = frappe.get_doc(self.dt, dn)
 			update_tags(doc, tags)
 		except Exception as e:
-			if frappe.db.is_column_missing(e):
+			if frappe.db.is_missing_column(e):
 				if not tags:
 					# no tags, nothing to do
 					return
diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py
index 83d9332f76a..b82384b732d 100644
--- a/frappe/desk/doctype/todo/todo.py
+++ b/frappe/desk/doctype/todo/todo.py
@@ -122,7 +122,7 @@ def update_in_reference(self):
 				# no table
 				return
 
-			elif frappe.db.is_column_missing(e):
+			elif frappe.db.is_missing_column(e):
 				from frappe.database.schema import add_column
 
 				add_column(self.reference_type, "_assign", "Text")
diff --git a/frappe/desk/like.py b/frappe/desk/like.py
index 2be2362b2de..42211087f45 100644
--- a/frappe/desk/like.py
+++ b/frappe/desk/like.py
@@ -58,7 +58,7 @@ def _toggle_like(doctype, name, add, user=None):
 			frappe.db.set_value(doctype, name, "_liked_by", json.dumps(liked_by), update_modified=False)
 
 	except frappe.db.ProgrammingError as e:
-		if frappe.db.is_column_missing(e):
+		if frappe.db.is_missing_column(e):
 			add_column(doctype, "_liked_by", "Text")
 			_toggle_like(doctype, name, add, user)
 		else:

From f183a3110f812624c751cd343b88f30f3e2a1100 Mon Sep 17 00:00:00 2001
From: Ankush Menat <ankush@frappe.io>
Date: Thu, 2 May 2024 14:23:19 +0530
Subject: [PATCH 15/34] fix: don't add creation index if one exists (#26295)

---
 frappe/database/mariadb/schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py
index c16c1496ff2..1af18caeb47 100644
--- a/frappe/database/mariadb/schema.py
+++ b/frappe/database/mariadb/schema.py
@@ -81,7 +81,7 @@ def alter(self):
 			if not frappe.db.get_column_index(self.table_name, col.fieldname, unique=False)
 		]
 
-		if self.meta.sort_field == "creation" and frappe.db.get_column_index(
+		if self.meta.sort_field == "creation" and not frappe.db.get_column_index(
 			self.table_name, "creation", unique=False
 		):
 			add_index_query.append("ADD INDEX `creation`(`creation`)")

From d72c6eccfeb35c02c5933d014946f577ebc3317d Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Thu, 2 May 2024 09:11:06 +0000
Subject: [PATCH 16/34] fix: changes for scheduler reliability (backport
 #26292) (#26294)

* fix: Dont let one invalid cron fail scheduler

Scenario:
- One bad cron job exists
- When it fails nothing after that job is enqueued.

After this fix, that failure is skipped and rest of the jobs are enqueued.

(cherry picked from commit b0aaeb50969ff26954b275f81a7e2beca204dfea)

# Conflicts:
#	frappe/utils/scheduler.py

* fix: update last_execution for no-log unconditionally

Cron jobs can now also disable logging.

(cherry picked from commit 86b1e9ec31581ea8722c5dc732d7070bf12a2b55)

* feat: show oldest unscheduled job in health report

(cherry picked from commit b2ef2cd506c9645e637928b1d353f8f4f01f1c66)

* Update frappe/utils/scheduler.py

---------

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 .../scheduled_job_type/scheduled_job_type.py  |  7 +++----
 .../system_health_report.js                   |  6 +++++-
 .../system_health_report.json                 | 20 ++++++++++++++++++-
 .../system_health_report.py                   | 16 ++++++++++++++-
 frappe/utils/scheduler.py                     | 17 +++++++++++++---
 5 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py
index 3fb251362ef..8072b58222b 100644
--- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py
+++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py
@@ -117,7 +117,7 @@ def get_next_execution(self):
 		}
 
 		if not self.cron_format:
-			self.cron_format = CRON_MAP[self.frequency]
+			self.cron_format = CRON_MAP.get(self.frequency)
 
 		# If this is a cold start then last_execution will not be set.
 		# Creation is set as fallback because if very old fallback is set job might trigger
@@ -155,9 +155,8 @@ def log_status(self, status):
 	def update_scheduler_log(self, status):
 		if not self.create_log:
 			# self.get_next_execution will work properly iff self.last_execution is properly set
-			if self.frequency == "All" and status == "Start":
-				self.db_set("last_execution", now_datetime(), update_modified=False)
-				frappe.db.commit()
+			self.db_set("last_execution", now_datetime(), update_modified=False)
+			frappe.db.commit()
 			return
 		if not self.scheduler_log:
 			self.scheduler_log = frappe.get_doc(
diff --git a/frappe/desk/doctype/system_health_report/system_health_report.js b/frappe/desk/doctype/system_health_report/system_health_report.js
index a57c9076a55..fcf5d522893 100644
--- a/frappe/desk/doctype/system_health_report/system_health_report.js
+++ b/frappe/desk/doctype/system_health_report/system_health_report.js
@@ -56,6 +56,7 @@ frappe.ui.form.on("System Health Report", {
 				val > 3 &&
 				frm.doc.total_outgoing_emails > 3 &&
 				val / frm.doc.total_outgoing_emails > 0.1,
+			oldest_unscheduled_job: (val) => !!val,
 			"queue_status.pending_jobs": (val) => val > 50,
 			"background_workers.utilization": (val) => val > 70,
 			"background_workers.failed_jobs": (val) => val > 50,
@@ -72,6 +73,9 @@ frappe.ui.form.on("System Health Report", {
 		document.head.appendChild(style);
 
 		const update_fields = () => {
+			if (!frappe.get_route().includes(frm.doc.name)) {
+				clearInterval(interval);
+			}
 			Object.entries(conditions).forEach(([field, condition]) => {
 				try {
 					if (field.includes(".")) {
@@ -93,6 +97,6 @@ frappe.ui.form.on("System Health Report", {
 		};
 
 		update_fields();
-		setInterval(update_fields, 1000);
+		const interval = setInterval(update_fields, 1000);
 	},
 });
diff --git a/frappe/desk/doctype/system_health_report/system_health_report.json b/frappe/desk/doctype/system_health_report/system_health_report.json
index 038f7369467..4a7352ba889 100644
--- a/frappe/desk/doctype/system_health_report/system_health_report.json
+++ b/frappe/desk/doctype/system_health_report/system_health_report.json
@@ -17,6 +17,9 @@
   "background_workers",
   "scheduler_section",
   "scheduler_status",
+  "column_break_bxog",
+  "oldest_unscheduled_job",
+  "section_break_vpuw",
   "failing_scheduled_jobs",
   "database_section",
   "database",
@@ -368,6 +371,7 @@
   {
    "fieldname": "scheduler_section",
    "fieldtype": "Section Break",
+   "hide_border": 1,
    "label": "Scheduler"
   },
   {
@@ -375,6 +379,20 @@
    "fieldtype": "Table",
    "label": "Failing Scheduled Jobs (last 7 days)",
    "options": "System Health Report Failing Jobs"
+  },
+  {
+   "fieldname": "column_break_bxog",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "oldest_unscheduled_job",
+   "fieldtype": "Link",
+   "label": "Oldest Unscheduled Job",
+   "options": "Scheduled Job Type"
+  },
+  {
+   "fieldname": "section_break_vpuw",
+   "fieldtype": "Section Break"
   }
  ],
  "hide_toolbar": 1,
@@ -382,7 +400,7 @@
  "is_virtual": 1,
  "issingle": 1,
  "links": [],
- "modified": "2024-04-22 11:47:52.194784",
+ "modified": "2024-05-02 13:32:16.495750",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "System Health Report",
diff --git a/frappe/desk/doctype/system_health_report/system_health_report.py b/frappe/desk/doctype/system_health_report/system_health_report.py
index be8f0714b68..c3e52afa3d5 100644
--- a/frappe/desk/doctype/system_health_report/system_health_report.py
+++ b/frappe/desk/doctype/system_health_report/system_health_report.py
@@ -26,11 +26,12 @@
 from contextlib import contextmanager
 
 import frappe
+from frappe.core.doctype.scheduled_job_type.scheduled_job_type import ScheduledJobType
 from frappe.model.document import Document
 from frappe.utils.background_jobs import get_queue, get_queue_list, get_redis_conn
 from frappe.utils.caching import redis_cache
 from frappe.utils.data import add_to_date
-from frappe.utils.scheduler import get_scheduler_status
+from frappe.utils.scheduler import get_scheduler_status, get_scheduler_tick
 
 
 @contextmanager
@@ -107,6 +108,7 @@ class SystemHealthReport(Document):
 		handled_emails: DF.Int
 		last_10_active_users: DF.Code | None
 		new_users: DF.Int
+		oldest_unscheduled_job: DF.Link | None
 		onsite_backups: DF.Int
 		pending_emails: DF.Int
 		private_files_size: DF.Float
@@ -208,6 +210,18 @@ def fetch_scheduler(self):
 		for job in failing_jobs:
 			self.append("failing_scheduled_jobs", job)
 
+		threshold = add_to_date(None, seconds=-30 * get_scheduler_tick(), as_datetime=True)
+		for job_type in frappe.get_all(
+			"Scheduled Job Type",
+			filters={"stopped": 0, "last_execution": ("<", threshold)},
+			fields="*",
+			order_by="last_execution asc",
+		):
+			job_type: ScheduledJobType = frappe.get_doc(doctype="Scheduled Job Type", **job_type)
+			if job_type.is_event_due():
+				self.oldest_unscheduled_job = job_type.name
+				break
+
 	@health_check("Emails")
 	def fetch_email_stats(self):
 		threshold = add_to_date(None, days=-7, as_datetime=True)
diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py
index 1ab0128f97d..a3b3dae1dfd 100755
--- a/frappe/utils/scheduler.py
+++ b/frappe/utils/scheduler.py
@@ -14,6 +14,8 @@
 import time
 from typing import NoReturn
 
+from croniter import CroniterBadCronError
+
 # imports - module imports
 import frappe
 from frappe.utils import cint, get_datetime, get_sites, now_datetime
@@ -35,7 +37,7 @@ def start_scheduler() -> NoReturn:
 	"""Run enqueue_events_for_all_sites based on scheduler tick.
 	Specify scheduler_interval in seconds in common_site_config.json"""
 
-	tick = cint(frappe.get_conf().scheduler_tick_interval) or 60
+	tick = get_scheduler_tick()
 	set_niceness()
 
 	while True:
@@ -90,8 +92,13 @@ def enqueue_events(site: str) -> list[str] | None:
 		enqueued_jobs = []
 		for job_type in frappe.get_all("Scheduled Job Type", filters={"stopped": 0}, fields="*"):
 			job_type = frappe.get_doc(doctype="Scheduled Job Type", **job_type)
-			if job_type.enqueue():
-				enqueued_jobs.append(job_type.method)
+			try:
+				if job_type.enqueue():
+					enqueued_jobs.append(job_type.method)
+			except CroniterBadCronError:
+				frappe.logger("scheduler").error(
+					f"Invalid Job on {frappe.local.site} - {job_type.name}", exc_info=True
+				)
 
 		return enqueued_jobs
 
@@ -196,3 +203,7 @@ def get_scheduler_status():
 	if is_scheduler_inactive():
 		return {"status": "inactive"}
 	return {"status": "active"}
+
+
+def get_scheduler_tick() -> int:
+	return cint(frappe.get_conf().scheduler_tick_interval) or 60

From b2950d252b7b132de7f8d9fec6e7ff97482b869f Mon Sep 17 00:00:00 2001
From: Akhil Narang <me@akhilnarang.dev>
Date: Wed, 10 Apr 2024 16:00:38 +0530
Subject: [PATCH 17/34] fix: allow accessing reports without roles

Signed-off-by: Akhil Narang <me@akhilnarang.dev>
(cherry picked from commit 26012aceb579797830ca8007104ccb26b1a6f94e)
Signed-off-by: Akhil Narang <me@akhilnarang.dev>
---
 frappe/boot.py | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/frappe/boot.py b/frappe/boot.py
index 2befb0937b4..da13a110f98 100644
--- a/frappe/boot.py
+++ b/frappe/boot.py
@@ -164,7 +164,9 @@ def get_user_pages_or_reports(parent, cache=False):
 	page = DocType("Page")
 	report = DocType("Report")
 
-	if parent == "Report":
+	is_report = parent == "Report"
+
+	if is_report:
 		columns = (report.name.as_("title"), report.ref_doctype, report.report_type)
 	else:
 		columns = (page.title.as_("title"),)
@@ -206,7 +208,7 @@ def get_user_pages_or_reports(parent, cache=False):
 		.distinct()
 	)
 
-	if parent == "Report":
+	if is_report:
 		pages_with_standard_roles = pages_with_standard_roles.where(report.disabled == 0)
 
 	pages_with_standard_roles = pages_with_standard_roles.run(as_dict=True)
@@ -221,19 +223,20 @@ def get_user_pages_or_reports(parent, cache=False):
 		frappe.qb.from_(hasRole).select(Count("*")).where(hasRole.parent == parentTable.name)
 	)
 
-	# pages with no role are allowed
-	if parent == "Page":
-		pages_with_no_roles = (
-			frappe.qb.from_(parentTable)
-			.select(parentTable.name, parentTable.modified, *columns)
-			.where(no_of_roles == 0)
-		).run(as_dict=True)
+	# pages and reports with no role are allowed
+	rows_with_no_roles = (
+		frappe.qb.from_(parentTable)
+		.select(parentTable.name, parentTable.modified, *columns)
+		.where(no_of_roles == 0)
+	).run(as_dict=True)
 
-		for p in pages_with_no_roles:
-			if p.name not in has_role:
-				has_role[p.name] = {"modified": p.modified, "title": p.title}
+	for r in rows_with_no_roles:
+		if r.name not in has_role:
+			has_role[r.name] = {"modified": r.modified, "title": r.title}
+			if is_report:
+				has_role[r.name] |= {"ref_doctype": r.ref_doctype}
 
-	elif parent == "Report":
+	if is_report:
 		if not has_permission("Report", raise_exception=False):
 			return {}
 

From 475189c7dee66136023e519819200ffb9158186d Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Thu, 2 May 2024 13:11:42 +0000
Subject: [PATCH 18/34] fix: only redirect to same domain (#26304) (#26306)

This limits post login redirects to same domain to avoid social engineering attempts.

(cherry picked from commit 65b3c42635038cdff17d3109be6c373bac004829)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/tests/test_api.py | 15 ++++++++++++++-
 frappe/www/login.py      | 25 +++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py
index 165ee8123f4..ad4e757e1dd 100644
--- a/frappe/tests/test_api.py
+++ b/frappe/tests/test_api.py
@@ -7,7 +7,7 @@
 from threading import Thread
 from time import time
 from unittest.mock import patch
-from urllib.parse import urljoin
+from urllib.parse import urlencode, urljoin
 
 import requests
 from filetype import guess_mime
@@ -454,6 +454,19 @@ def test_download_private_file_with_unique_url(self):
 		self.assertEqual(self.get(file.unique_url, {"sid": self.sid}).text, test_content)
 		self.assertEqual(self.get(file.file_url, {"sid": self.sid}).text, test_content)
 
+	def test_login_redirects(self):
+		expected_redirects = {
+			"/app/user": "/app/user",
+			"/app/user?enabled=1": "/app/user?enabled=1",
+			"http://example.com": "/app",  # No external redirect
+			"https://google.com": "/app",
+			"http://localhost:8000": "/app",
+			"http://localhost/app": "http://localhost/app",
+		}
+		for redirect, expected_redirect in expected_redirects.items():
+			response = self.get(f"/login?{urlencode({'redirect-to':redirect})}", {"sid": self.sid})
+			self.assertEqual(response.location, expected_redirect)
+
 
 def generate_admin_keys():
 	from frappe.core.doctype.user.user import generate_keys
diff --git a/frappe/www/login.py b/frappe/www/login.py
index c988efd2f46..4182496c0cc 100644
--- a/frappe/www/login.py
+++ b/frappe/www/login.py
@@ -1,6 +1,9 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+
+from urllib.parse import urlparse
+
 import frappe
 import frappe.utils
 from frappe import _
@@ -19,6 +22,7 @@
 
 def get_context(context):
 	redirect_to = frappe.local.request.args.get("redirect-to")
+	redirect_to = sanitize_redirect(redirect_to)
 
 	if frappe.session.user != "Guest":
 		if not redirect_to:
@@ -180,3 +184,24 @@ def login_via_key(key: str):
 			http_status_code=403,
 			indicator_color="red",
 		)
+
+
+def sanitize_redirect(redirect: str | None) -> str | None:
+	"""Only allow redirect on same domain.
+
+	Allowed redirects:
+	- Same host e.g. https://frappe.localhost/path
+	- Just path e.g. /app
+	"""
+	if not redirect:
+		return redirect
+
+	parsed_redirect = urlparse(redirect)
+	if not parsed_redirect.netloc:
+		return redirect
+
+	parsed_request_host = urlparse(frappe.local.request.url)
+	if parsed_request_host.netloc == parsed_redirect.netloc:
+		return redirect
+
+	return None

From 5f25ae6d63912c9d77bd5e7ed0cc4eaba0cc7838 Mon Sep 17 00:00:00 2001
From: Akhil Narang <me@akhilnarang.dev>
Date: Fri, 3 May 2024 17:21:16 +0530
Subject: [PATCH 19/34] fix: args is a stringified JSON

Signed-off-by: Akhil Narang <me@akhilnarang.dev>
---
 frappe/model/document.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frappe/model/document.py b/frappe/model/document.py
index 3ee08327d3e..152cead59e9 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -1737,8 +1737,9 @@ def _document_values_generator(
 
 @frappe.whitelist()
 def unlock_document(doctype: str | None = None, name: str | None = None, args=None):
+	# Backward compatibility
 	if not doctype and not name and args:
-		# Backward compatibility
+		args = json.loads(args)
 		doctype = str(args["doctype"])
 		name = str(args["name"])
 	frappe.get_doc(doctype, name).unlock()

From 9a151914652a843576f0edd38b300a67d78bd4b7 Mon Sep 17 00:00:00 2001
From: RitvikSardana <65544983+RitvikSardana@users.noreply.github.com>
Date: Fri, 3 May 2024 18:34:54 +0530
Subject: [PATCH 20/34] feat: link field filter backport v15 (#25966)

* feat: Apply Filters to Link Fields Via Form Builder (#22844)

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>

* fix: link field shown only for Link fieldtype

* fix: add back if condition

* feat: add eval support for link field filters

* fix: support for adding filters for child table

* fix: eval support for child table

* chore: code cleanup

* chore: code cleanup

* fix: convert to string when operator is of type 'like'

* style: wspace

---------

Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com>
Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 cypress/fixtures/form_builder_doctype.js      |   6 +
 cypress/integration/form_builder.js           |  34 +++++
 frappe/core/doctype/docfield/docfield.json    |   6 +
 frappe/core/doctype/docfield/docfield.py      |   1 +
 frappe/core/doctype/doctype/doctype.py        |   1 +
 .../doctype/custom_field/custom_field.json    |   7 +
 .../doctype/custom_field/custom_field.py      |   1 +
 .../customize_form/customize_form.json        |   7 +
 .../doctype/customize_form/customize_form.py  |  13 ++
 .../customize_form_field.json                 |   6 +
 .../customize_form_field.py                   |   1 +
 .../js/form_builder/components/Field.vue      | 134 ++++++++++++++++--
 .../components/FieldProperties.vue            |   6 +-
 frappe/public/js/form_builder/store.js        |  16 +++
 frappe/public/js/frappe/form/controls/link.js |  40 ++++++
 .../js/frappe/ui/filters/filter_list.js       |   3 +-
 16 files changed, 268 insertions(+), 14 deletions(-)

diff --git a/cypress/fixtures/form_builder_doctype.js b/cypress/fixtures/form_builder_doctype.js
index 08b598f82a3..995971bed47 100644
--- a/cypress/fixtures/form_builder_doctype.js
+++ b/cypress/fixtures/form_builder_doctype.js
@@ -10,6 +10,12 @@ export default {
 			fieldtype: "Data",
 			label: "Data 3",
 		},
+		{
+			fieldname: "gender",
+			fieldtype: "Link",
+			label: "Gender",
+			options: "Gender",
+		},
 		{
 			fieldname: "tab",
 			fieldtype: "Tab Break",
diff --git a/cypress/integration/form_builder.js b/cypress/integration/form_builder.js
index 817a677f82b..68ceaf34f55 100644
--- a/cypress/integration/form_builder.js
+++ b/cypress/integration/form_builder.js
@@ -35,6 +35,40 @@ context("Form Builder", () => {
 		cy.get(".title-area .indicator-pill.orange").should("have.text", "Not Saved");
 	});
 
+	it("Check if Filters are applied to the link field", () => {
+		// Visit the Form Builder
+		cy.visit(`/app/doctype/${doctype_name}`);
+		cy.findByRole("tab", { name: "Form" }).click();
+
+		cy.get("[data-fieldname='gender']").click();
+
+		// click on filter action button
+		cy.get('[data-fieldname="gender"] .field-actions button:first').click();
+
+		// add filter
+		cy.get(".modal-body .clear-filters").click();
+		cy.get(".modal-body .filter-action-buttons .add-filter").click();
+		cy.wait(100);
+		cy.get(".modal-body .filter-box .list_filter .filter-field .link-field input").type(
+			"Male"
+		);
+		cy.get(".btn-modal-primary").click();
+
+		// Save the document
+		cy.click_doc_primary_button("Save");
+
+		// Open a new Form
+		cy.new_form(doctype_name);
+		// Click on the "salutation" field
+		cy.get_field("gender").clear().click();
+
+		cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link");
+		cy.wait("@search_link").then((data) => {
+			expect(data.response.body.message.length).to.eq(1);
+			expect(data.response.body.message[0].value).to.eq("Male");
+		});
+	});
+
 	it("Add empty section and save", () => {
 		cy.visit(`/app/doctype/${doctype_name}`);
 		cy.findByRole("tab", { name: "Form" }).click();
diff --git a/frappe/core/doctype/docfield/docfield.json b/frappe/core/doctype/docfield/docfield.json
index f94a51a0d49..049a56d3d85 100644
--- a/frappe/core/doctype/docfield/docfield.json
+++ b/frappe/core/doctype/docfield/docfield.json
@@ -23,6 +23,7 @@
   "options",
   "sort_options",
   "show_dashboard",
+  "link_filters",
   "defaults_section",
   "default",
   "column_break_6",
@@ -561,6 +562,11 @@
    "fieldname": "sort_options",
    "fieldtype": "Check",
    "label": "Sort Options"
+  },
+  {
+   "fieldname": "link_filters",
+   "fieldtype": "JSON",
+   "label": "Link Filters"
   }
  ],
  "idx": 1,
diff --git a/frappe/core/doctype/docfield/docfield.py b/frappe/core/doctype/docfield/docfield.py
index 327395661e9..6fe75ade8ff 100644
--- a/frappe/core/doctype/docfield/docfield.py
+++ b/frappe/core/doctype/docfield/docfield.py
@@ -87,6 +87,7 @@ class DocField(Document):
 		is_virtual: DF.Check
 		label: DF.Data | None
 		length: DF.Int
+		link_filters: DF.JSON | None
 		mandatory_depends_on: DF.Code | None
 		max_height: DF.Data | None
 		no_copy: DF.Check
diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py
index 9075d20a0f8..d17f110ade1 100644
--- a/frappe/core/doctype/doctype/doctype.py
+++ b/frappe/core/doctype/doctype/doctype.py
@@ -134,6 +134,7 @@ class DocType(Document):
 		is_virtual: DF.Check
 		issingle: DF.Check
 		istable: DF.Check
+		link_filters: DF.JSON
 		links: DF.Table[DocTypeLink]
 		make_attachments_public: DF.Check
 		max_attachments: DF.Int
diff --git a/frappe/custom/doctype/custom_field/custom_field.json b/frappe/custom/doctype/custom_field/custom_field.json
index 32c4552f19b..b0971a13810 100644
--- a/frappe/custom/doctype/custom_field/custom_field.json
+++ b/frappe/custom/doctype/custom_field/custom_field.json
@@ -15,6 +15,7 @@
   "fieldname",
   "insert_after",
   "length",
+  "link_filters",
   "column_break_6",
   "fieldtype",
   "precision",
@@ -446,6 +447,12 @@
    "fieldtype": "Check",
    "label": "Sort Options"
   },
+  {
+   "fieldname": "link_filters",
+   "fieldtype": "JSON",
+   "hidden": 1,
+   "label": "Link Filters"
+  },
   {
    "default": "0",
    "fieldname": "show_dashboard",
diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py
index 4617bdc96a1..4b7c962d2e1 100644
--- a/frappe/custom/doctype/custom_field/custom_field.py
+++ b/frappe/custom/doctype/custom_field/custom_field.py
@@ -94,6 +94,7 @@ class CustomField(Document):
 		is_virtual: DF.Check
 		label: DF.Data | None
 		length: DF.Int
+		link_filters: DF.JSON | None
 		mandatory_depends_on: DF.Code | None
 		module: DF.Link | None
 		no_copy: DF.Check
diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json
index b3bb3eeb6c5..b025ad33c67 100644
--- a/frappe/custom/doctype/customize_form/customize_form.json
+++ b/frappe/custom/doctype/customize_form/customize_form.json
@@ -12,6 +12,7 @@
   "properties",
   "label",
   "search_fields",
+  "link_filters",
   "column_break_5",
   "istable",
   "is_calendar_and_gantt",
@@ -382,6 +383,12 @@
    "fieldtype": "Tab Break",
    "label": "Form"
   },
+  {
+   "fieldname": "link_filters",
+   "fieldtype": "JSON",
+   "hidden": 1,
+   "label": "Link Filters"
+  },
   {
    "fieldname": "details_tab",
    "fieldtype": "Tab Break",
diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py
index 74461506d80..43b3d10c340 100644
--- a/frappe/custom/doctype/customize_form/customize_form.py
+++ b/frappe/custom/doctype/customize_form/customize_form.py
@@ -55,6 +55,7 @@ class CustomizeForm(Document):
 		is_calendar_and_gantt: DF.Check
 		istable: DF.Check
 		label: DF.Data | None
+		link_filters: DF.JSON | None
 		links: DF.Table[DocTypeLink]
 		make_attachments_public: DF.Check
 		max_attachments: DF.Int
@@ -700,6 +701,17 @@ def is_standard_or_system_generated_field(df):
 	return not df.get("is_custom_field") or df.get("is_system_generated")
 
 
+@frappe.whitelist()
+def get_link_filters_from_doc_without_customisations(doctype, fieldname):
+	"""Get the filters of a link field from a doc without customisations
+	In backend the customisations are not applied.
+	Customisations are applied in the client side.
+	"""
+	doc = frappe.get_doc("DocType", doctype)
+	field = list(filter(lambda x: x.fieldname == fieldname, doc.fields))
+	return field[0].link_filters
+
+
 doctype_properties = {
 	"search_fields": "Data",
 	"title_field": "Data",
@@ -780,6 +792,7 @@ def is_standard_or_system_generated_field(df):
 	"hide_days": "Check",
 	"hide_seconds": "Check",
 	"is_virtual": "Check",
+	"link_filters": "JSON",
 }
 
 doctype_link_properties = {
diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.json b/frappe/custom/doctype/customize_form_field/customize_form_field.json
index fd559636d3f..0dfd7a6c458 100644
--- a/frappe/custom/doctype/customize_form_field/customize_form_field.json
+++ b/frappe/custom/doctype/customize_form_field/customize_form_field.json
@@ -24,6 +24,7 @@
   "no_copy",
   "allow_in_quick_entry",
   "translatable",
+  "link_filters",
   "column_break_7",
   "default",
   "precision",
@@ -472,6 +473,11 @@
    "fieldname": "sort_options",
    "fieldtype": "Check",
    "label": "Sort Options"
+  },
+  {
+   "fieldname": "link_filters",
+   "fieldtype": "JSON",
+   "label": "Link Filters"
   }
  ],
  "idx": 1,
diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.py b/frappe/custom/doctype/customize_form_field/customize_form_field.py
index ae8d6ff39df..76ab6535e37 100644
--- a/frappe/custom/doctype/customize_form_field/customize_form_field.py
+++ b/frappe/custom/doctype/customize_form_field/customize_form_field.py
@@ -86,6 +86,7 @@ class CustomizeFormField(Document):
 		is_virtual: DF.Check
 		label: DF.Data | None
 		length: DF.Int
+		link_filters: DF.JSON | None
 		mandatory_depends_on: DF.Code | None
 		no_copy: DF.Check
 		non_negative: DF.Check
diff --git a/frappe/public/js/form_builder/components/Field.vue b/frappe/public/js/form_builder/components/Field.vue
index d853f5533e3..d565aecda71 100644
--- a/frappe/public/js/form_builder/components/Field.vue
+++ b/frappe/public/js/form_builder/components/Field.vue
@@ -75,6 +75,108 @@ function duplicate_field() {
 	store.form.selected_field = duplicate_field.df;
 }
 
+function make_dialog(frm) {
+	frm.dialog = new frappe.ui.Dialog({
+		title: __("Set Filters"),
+		fields: [
+			{
+				fieldtype: "HTML",
+				fieldname: "filter_area",
+			},
+		],
+		primary_action: () => {
+			let fieldname = props.field.df.fieldname;
+			let field_option = props.field.df.options;
+			let filters = frm.filter_group.get_filters().map((filter) => {
+				// last element is a boolean which hides the filter hence not required to store in meta
+				filter.pop();
+
+				// filter_group component requires options and frm.set_query requires fieldname so storing both
+				filter[0] = field_option;
+				return filter;
+			});
+
+			props.field.df.link_filters = JSON.stringify(filters);
+			store.form.selected_field = props.field.df;
+			frm.dialog.hide();
+		},
+		primary_action_label: __("Apply"),
+	});
+
+	if (frm.doctype === "Customize Form") {
+		let current_doctype = frm.doc.doc_type;
+		let fieldname = props.field.df.fieldname;
+		let property = "link_filters";
+		let property_setter_id = current_doctype + "-" + fieldname + "-" + property;
+
+		frappe.db.exists("Property Setter", property_setter_id).then((exits) => {
+			if (exits) {
+				frm.dialog.set_secondary_action_label(__("Reset To Default"));
+				frm.dialog.set_secondary_action(() => {
+					frappe.call({
+						method: "frappe.custom.doctype.customize_form.customize_form.get_link_filters_from_doc_without_customisations",
+						args: {
+							doctype: current_doctype,
+							fieldname: fieldname,
+						},
+						callback: function (r) {
+							if (r.message) {
+								props.field.df.link_filters = r.message;
+
+								frm.filter_group.clear_filters();
+								add_existing_filter(frm, props.field.df);
+								// hide the secondary action button
+								frm.dialog.get_secondary_btn().addClass("hidden");
+							}
+						},
+					});
+				});
+			}
+		});
+	}
+}
+
+function make_filter_area(frm, doctype) {
+	frm.filter_group = new frappe.ui.FilterGroup({
+		parent: frm.dialog.get_field("filter_area").$wrapper,
+		doctype: doctype,
+		on_change: () => {},
+	});
+}
+
+function add_existing_filter(frm, df) {
+	if (df.link_filters) {
+		let filters = JSON.parse(df.link_filters);
+		if (filters) {
+			frm.filter_group.add_filters_to_filter_group(filters);
+		}
+	}
+}
+
+function edit_filters() {
+	let field_doctype = props.field.df.options;
+	const { frm } = store;
+
+	make_dialog(frm);
+	make_filter_area(frm, field_doctype);
+	frappe.model.with_doctype(field_doctype, () => {
+		frm.dialog.show();
+		add_existing_filter(frm, props.field.df);
+	});
+}
+
+function is_filter_applied() {
+	if (props.field.df.link_filters) {
+		try {
+			if (JSON.parse(props.field.df.link_filters).length > 0) {
+				return "btn-filter-applied";
+			}
+		} catch (error) {
+			return "";
+		}
+	}
+}
+
 onMounted(() => selected.value && label_input.value.focus_on_label());
 </script>
 
@@ -111,22 +213,17 @@ onMounted(() => selected.value && label_input.value.focus_on_label());
 			</template>
 			<template #actions>
 				<div class="field-actions" :hidden="store.read_only">
-					<AddFieldButton
-						v-if="column.fields.indexOf(field) != column.fields.length - 1"
-						ref="add_field_ref"
-						:field="field"
-						:column="column"
-						:tooltip="__('Add field below')"
-					>
-						<div v-html="frappe.utils.icon('add', 'sm')" />
-					</AddFieldButton>
 					<button
+						v-if="field.df.fieldtype === 'Link'"
 						class="btn btn-xs btn-icon"
-						:title="__('Duplicate field')"
-						@click.stop="duplicate_field"
+						:class="is_filter_applied()"
+						@click="edit_filters"
 					>
-						<div v-html="frappe.utils.icon('duplicate', 'sm')"></div>
+						<div v-html="frappe.utils.icon('filter', 'sm')"></div>
 					</button>
+					<AddFieldButton ref="add_field_ref" :column="column" :field="field">
+						<div v-html="frappe.utils.icon('add', 'sm')" />
+					</AddFieldButton>
 					<button
 						v-if="column.fields.indexOf(field)"
 						class="btn btn-xs btn-icon"
@@ -137,6 +234,13 @@ onMounted(() => selected.value && label_input.value.focus_on_label());
 					>
 						<div v-html="frappe.utils.icon('move', 'sm')"></div>
 					</button>
+					<button
+						class="btn btn-xs btn-icon"
+						:title="__('Duplicate field')"
+						@click.stop="duplicate_field"
+					>
+						<div v-html="frappe.utils.icon('duplicate', 'sm')"></div>
+					</button>
 					<button
 						class="btn btn-xs btn-icon"
 						:title="__('Remove field')"
@@ -211,4 +315,10 @@ onMounted(() => selected.value && label_input.value.focus_on_label());
 		}
 	}
 }
+.btn-filter-applied {
+	background-color: var(--gray-300) !important;
+	&:hover {
+		background-color: var(--gray-400) !important;
+	}
+}
 </style>
diff --git a/frappe/public/js/form_builder/components/FieldProperties.vue b/frappe/public/js/form_builder/components/FieldProperties.vue
index a1c95b0a66a..159d8eb43fc 100644
--- a/frappe/public/js/form_builder/components/FieldProperties.vue
+++ b/frappe/public/js/form_builder/components/FieldProperties.vue
@@ -51,6 +51,11 @@ let docfield_df = computed(() => {
 			}
 		}
 
+		// show link_filters docfield only when link field is selected
+		if (df.fieldname === "link_filters" && store.form.selected_field.fieldtype !== "Link") {
+			return false;
+		}
+
 		if (search_text.value) {
 			if (
 				df.label.toLowerCase().includes(search_text.value.toLowerCase()) ||
@@ -62,7 +67,6 @@ let docfield_df = computed(() => {
 		}
 		return true;
 	});
-
 	return [...fields];
 });
 </script>
diff --git a/frappe/public/js/form_builder/store.js b/frappe/public/js/form_builder/store.js
index 473f5be98be..7c0328362ad 100644
--- a/frappe/public/js/form_builder/store.js
+++ b/frappe/public/js/form_builder/store.js
@@ -206,6 +206,22 @@ export const useStore = defineStore("form-builder-store", () => {
 					get_field_data(df)
 				);
 			}
+
+			if (df.link_filters === "") {
+				delete df.link_filters;
+			}
+
+			// check if link_filters format is correct or not
+			if (df.link_filters) {
+				try {
+					let link_filters = JSON.parse(df.link_filters);
+				} catch (e) {
+					error_message = __(
+						"Invalid Filter Format for field {0} of type {1}. Try using filter icon on the field to set it correctly",
+						get_field_data(df)
+					);
+				}
+			}
 		});
 
 		return error_message;
diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js
index 7534c7f9516..80720db8a2c 100644
--- a/frappe/public/js/frappe/form/controls/link.js
+++ b/frappe/public/js/frappe/form/controls/link.js
@@ -512,6 +512,12 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
 			});
 			return obj;
 		};
+
+		// apply link field filters
+		if (this.df.link_filters && !!this.df.link_filters.length) {
+			this.apply_link_field_filters();
+		}
+
 		if (this.get_query || this.df.get_query) {
 			var get_query = this.get_query || this.df.get_query;
 			if ($.isPlainObject(get_query)) {
@@ -574,6 +580,40 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
 			$.extend(args.filters, this.df.filters);
 		}
 	}
+
+	apply_link_field_filters() {
+		let link_filters = JSON.parse(this.df.link_filters);
+		let filters = this.parse_filters(link_filters);
+		// take filters from the link field and add to the query
+		this.get_query = function () {
+			return {
+				filters,
+			};
+		};
+	}
+
+	parse_filters(link_filters) {
+		let filters = {};
+		link_filters.forEach((filter) => {
+			let [_, fieldname, operator, value] = filter;
+			if (operator === "like") {
+				value = String(value).replace(/%/g, "");
+			}
+
+			if (value?.startsWith?.("eval:")) {
+				value = value.split("eval:")[1];
+				let context = {
+					doc: this.doc,
+					parent: this.doc.parenttype ? this.frm.doc : null,
+					frappe,
+				};
+				value = frappe.utils.eval(value, context);
+			}
+			filters[fieldname] = [operator, value];
+		});
+		return filters;
+	}
+
 	validate(value) {
 		// validate the value just entered
 		if (this._validated || this.df.options == "[Select]" || this.df.ignore_link_validation) {
diff --git a/frappe/public/js/frappe/ui/filters/filter_list.js b/frappe/public/js/frappe/ui/filters/filter_list.js
index 329db36a781..100ceecc8ee 100644
--- a/frappe/public/js/frappe/ui/filters/filter_list.js
+++ b/frappe/public/js/frappe/ui/filters/filter_list.js
@@ -1,7 +1,7 @@
 frappe.ui.FilterGroup = class {
 	constructor(opts) {
 		$.extend(this, opts);
-		this.filters = [];
+		this.filters = this.filters || [];
 		window.fltr = this;
 		if (!this.filter_button) {
 			this.wrapper = this.parent;
@@ -241,6 +241,7 @@ frappe.ui.FilterGroup = class {
 			},
 			filter_list: this.base_list || this,
 		};
+
 		let filter = new frappe.ui.Filter(args);
 		this.filters.push(filter);
 		return filter;

From 13535a688114d8f054c7a3a0a67a2cc18fe0cab7 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Fri, 3 May 2024 13:28:22 +0000
Subject: [PATCH 21/34] fix: like filter from meta filters (#26311) (#26313)

(cherry picked from commit 8e76a94a62ea762e118c6803bc5774a4d69aae99)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/public/js/frappe/form/controls/link.js | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js
index 80720db8a2c..470eb053f24 100644
--- a/frappe/public/js/frappe/form/controls/link.js
+++ b/frappe/public/js/frappe/form/controls/link.js
@@ -596,10 +596,6 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
 		let filters = {};
 		link_filters.forEach((filter) => {
 			let [_, fieldname, operator, value] = filter;
-			if (operator === "like") {
-				value = String(value).replace(/%/g, "");
-			}
-
 			if (value?.startsWith?.("eval:")) {
 				value = value.split("eval:")[1];
 				let context = {

From 3ac1edabc2ef2bbbaf9e7379b4dc348d8a30bec4 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Fri, 3 May 2024 13:38:48 +0000
Subject: [PATCH 22/34] fix: disabled user login from login via link feature
 (#26134) (#26141)

* fix(ui): button color for login via link template

- blue > black

* fix: restrict login via link for disabled users

* fix: validate user on session.resume

(cherry picked from commit a73e65581dfc8b589311fe9d2d592467f30a5046)

Co-authored-by: Rutwik Hiwalkar <rutwikhdev@gmail.com>
---
 frappe/sessions.py                                 |  9 +++++++++
 frappe/templates/emails/login_with_email_link.html | 10 ++++------
 frappe/www/login.py                                |  1 -
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/frappe/sessions.py b/frappe/sessions.py
index f0463eab65e..37c1fc97ea3 100644
--- a/frappe/sessions.py
+++ b/frappe/sessions.py
@@ -210,8 +210,16 @@ def __init__(self, user, resume=False, full_name=None, user_type=None):
 
 		else:
 			if self.user:
+				self.validate_user()
 				self.start()
 
+	def validate_user(self):
+		if not frappe.get_cached_value("User", self.user, "enabled"):
+			frappe.throw(
+				_("User {0} is disabled. Please contact your System Manager.").format(self.user),
+				frappe.ValidationError,
+			)
+
 	def start(self):
 		"""start a new session"""
 		# generate sid
@@ -274,6 +282,7 @@ def resume(self):
 		if data:
 			self.data.update({"data": data, "user": data.user, "sid": self.sid})
 			self.user = data.user
+			self.validate_user()
 			validate_ip_address(self.user)
 		else:
 			self.start_as_guest()
diff --git a/frappe/templates/emails/login_with_email_link.html b/frappe/templates/emails/login_with_email_link.html
index 144869e2e67..6badd5c89d0 100644
--- a/frappe/templates/emails/login_with_email_link.html
+++ b/frappe/templates/emails/login_with_email_link.html
@@ -23,11 +23,9 @@
 		</tr>
 		<tr>
 			<td align="center">
-				<div class="btn btn-primary" style="margin-top: 30px;">
-					<a href="{{ link or '#'}}" style="color: #fff; text-decoration: none;">
-						{{ _('Log In To {0}').format(app_name) }}
-					</a>
-				</div>
+				<a href="{{ link or '#'}}" class="btn btn-primary" style="background-color: #171717; text-decoration: none; margin-top: 30px;">
+					{{ _('Log In To {0}').format(app_name) }}
+				</a>
 			</td>
 		</tr>
 	</tbody>
@@ -38,4 +36,4 @@
 	<div class="body-content">
 		{{ table(table(body(), 'email-body'), 'email-container') }}
 	</div>
-</div>
\ No newline at end of file
+</div>
diff --git a/frappe/www/login.py b/frappe/www/login.py
index 4182496c0cc..e4b873b25b8 100644
--- a/frappe/www/login.py
+++ b/frappe/www/login.py
@@ -171,7 +171,6 @@ def login_via_key(key: str):
 
 	if email:
 		frappe.cache.delete_value(cache_key)
-
 		frappe.local.login_manager.login_as(email)
 
 		redirect_post_login(

From f60be3c56c06659e6dd683cba537e581734a0445 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Fri, 3 May 2024 14:24:56 +0000
Subject: [PATCH 23/34] fix: automatic logo width (#26273) (#26278)

(cherry picked from commit 02ea846cd448256ed4b6af9a5e8e41fbcb12022f)

# Conflicts:
#	frappe/core/doctype/navbar_settings/navbar_settings.json

Co-authored-by: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
---
 .../doctype/navbar_settings/navbar_settings.json  | 15 ++-------------
 .../doctype/navbar_settings/navbar_settings.py    |  1 -
 frappe/public/js/frappe/ui/toolbar/navbar.html    |  1 -
 3 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.json b/frappe/core/doctype/navbar_settings/navbar_settings.json
index b8142066c98..09833e6164b 100644
--- a/frappe/core/doctype/navbar_settings/navbar_settings.json
+++ b/frappe/core/doctype/navbar_settings/navbar_settings.json
@@ -7,8 +7,6 @@
  "field_order": [
   "logo_section",
   "app_logo",
-  "column_break_3",
-  "logo_width",
   "section_break_2",
   "settings_dropdown",
   "help_dropdown",
@@ -43,15 +41,6 @@
    "fieldtype": "Section Break",
    "label": "Application Logo"
   },
-  {
-   "fieldname": "column_break_3",
-   "fieldtype": "Column Break"
-  },
-  {
-   "fieldname": "logo_width",
-   "fieldtype": "Int",
-   "label": "Logo Width"
-  },
   {
    "fieldname": "announcements_section",
    "fieldtype": "Section Break",
@@ -67,7 +56,7 @@
  ],
  "issingle": 1,
  "links": [],
- "modified": "2024-03-18 13:54:44.498352",
+ "modified": "2024-05-01 14:09:54.587137",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Navbar Settings",
@@ -89,4 +78,4 @@
  "sort_order": "DESC",
  "states": [],
  "track_changes": 1
-}
\ No newline at end of file
+}
diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.py b/frappe/core/doctype/navbar_settings/navbar_settings.py
index 1fee71212f0..694df34373b 100644
--- a/frappe/core/doctype/navbar_settings/navbar_settings.py
+++ b/frappe/core/doctype/navbar_settings/navbar_settings.py
@@ -19,7 +19,6 @@ class NavbarSettings(Document):
 		announcement_widget: DF.TextEditor | None
 		app_logo: DF.AttachImage | None
 		help_dropdown: DF.Table[NavbarItem]
-		logo_width: DF.Int
 		settings_dropdown: DF.Table[NavbarItem]
 
 	# end: auto-generated types
diff --git a/frappe/public/js/frappe/ui/toolbar/navbar.html b/frappe/public/js/frappe/ui/toolbar/navbar.html
index 081b2435048..ea94b37417d 100644
--- a/frappe/public/js/frappe/ui/toolbar/navbar.html
+++ b/frappe/public/js/frappe/ui/toolbar/navbar.html
@@ -4,7 +4,6 @@
 			<a class="navbar-brand navbar-home" href="/app">
 				<img
 					class="app-logo"
-					style="width: {{ navbar_settings.logo_width || 28 }}px"
 					src="{{ frappe.boot.app_logo_url }}"
 					alt="{{ __("App Logo") }}"
 				>

From 173fd2a08a043b5e007173d28fc9699ae7344ed0 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Sat, 4 May 2024 17:55:25 +0530
Subject: [PATCH 24/34] style: whitespace in social_login_key.json (#26317)
 (#26321)

(cherry picked from commit 076e25658b4ec6b0e2e2a85c4fb89aef8c78424b)

Co-authored-by: mahsem <137205921+mahsem@users.noreply.github.com>
---
 .../doctype/social_login_key/social_login_key.json            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frappe/integrations/doctype/social_login_key/social_login_key.json b/frappe/integrations/doctype/social_login_key/social_login_key.json
index 12514c3fddc..dcc27c45e68 100644
--- a/frappe/integrations/doctype/social_login_key/social_login_key.json
+++ b/frappe/integrations/doctype/social_login_key/social_login_key.json
@@ -167,7 +167,7 @@
    "label": "Configuration"
   },
   {
-   "description": "Controls whether new users can sign up using this Social Login Key. If unset, Website Settings is respected. ",
+   "description": "Controls whether new users can sign up using this Social Login Key. If unset, Website Settings is respected.",
    "fieldname": "sign_ups",
    "fieldtype": "Select",
    "label": "Sign ups",
@@ -200,4 +200,4 @@
  "states": [],
  "title_field": "provider_name",
  "track_changes": 1
-}
\ No newline at end of file
+}

From 24be120ab8db243f8f3bdb263353a086e465f0a8 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Sat, 4 May 2024 12:48:10 +0000
Subject: [PATCH 25/34] feat: improved update notifications (backport #26314)
 (#26320)

* fix: update notifier never running

(cherry picked from commit 330a1b20444a609c95bce1fc82c0f402a70fdc35)

# Conflicts:
#	frappe/utils/change_log.py

* chore: dead hooks

(cherry picked from commit 56233f3182ae0b9a718e7bc2b803d30c179eaf03)

* feat: FC specific update notifications

(cherry picked from commit 5ca14bb171aa41892dc6f5b71f25c993814f11f1)

# Conflicts:
#	frappe/tests/test_utils.py
#	frappe/utils/change_log.py
#	pyproject.toml

* feat: show security issues count

chore: conflicts
(cherry picked from commit e0c171b23bc8a104188dbf218994206b40fd4a5b)

* refactor: misc changes

- Move sys setting check to server side
- tomli import handling

(cherry picked from commit a234e79790976dc0e27e682f44452670bf475a9f)

* feat: give higher priority to minor versions

Major versions will be shown when minor updates are exhausted only.

(cherry picked from commit 45026aed376a777f483bd04d0a097439d5d3b61d)

# Conflicts:
#	frappe/utils/change_log.py

* feat: Keep certain keys persistent in cache

(cherry picked from commit 8ab308838bd77068242b4e78ba6fcdfcee52cd56)

* chore: conflicts

* chore: drop git url parse dependency

---------

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/__init__.py                            |   6 +-
 frappe/boot.py                                |   3 +-
 .../core/doctype/translation/translation.py   |   2 +-
 frappe/hooks.py                               |  15 +-
 frappe/public/js/frappe/desk.js               |   7 +-
 frappe/sessions.py                            |   2 +
 frappe/tests/test_deferred_insert.py          |   6 +
 frappe/tests/test_utils.py                    |  37 +++-
 frappe/translate.py                           |  43 ----
 frappe/utils/change_log.py                    | 189 +++++++++++++-----
 frappe/utils/frappecloud.py                   |  11 +
 pyproject.toml                                |   7 +-
 12 files changed, 211 insertions(+), 117 deletions(-)
 create mode 100644 frappe/utils/frappecloud.py

diff --git a/frappe/__init__.py b/frappe/__init__.py
index b47908acab5..de7de2d897d 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1013,7 +1013,11 @@ def clear_cache(user: str | None = None, doctype: str | None = None):
 		frappe.cache_manager.clear_user_cache(user)
 	else:  # everything
 		# Delete ALL keys associated with this site.
-		frappe.cache.delete_keys("")
+		keys_to_delete = set(frappe.cache.get_keys(""))
+		for key in frappe.get_hooks("persistent_cache_keys"):
+			keys_to_delete.difference_update(frappe.cache.get_keys(key))
+		frappe.cache.delete_value(list(keys_to_delete), make_keys=False)
+
 		reset_metadata_version()
 		local.cache = {}
 		local.new_doc_templates = {}
diff --git a/frappe/boot.py b/frappe/boot.py
index da13a110f98..571f0d52fa9 100644
--- a/frappe/boot.py
+++ b/frappe/boot.py
@@ -24,6 +24,7 @@
 )
 from frappe.utils import add_user_info, cstr, get_system_timezone
 from frappe.utils.change_log import get_versions
+from frappe.utils.frappecloud import on_frappecloud
 from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled
 
 
@@ -448,7 +449,7 @@ def get_marketplace_apps():
 	apps = []
 	cache_key = "frappe_marketplace_apps"
 
-	if frappe.conf.developer_mode:
+	if frappe.conf.developer_mode or not on_frappecloud():
 		return apps
 
 	def get_apps_from_fc():
diff --git a/frappe/core/doctype/translation/translation.py b/frappe/core/doctype/translation/translation.py
index e54f4ce352c..7bb20689ddd 100644
--- a/frappe/core/doctype/translation/translation.py
+++ b/frappe/core/doctype/translation/translation.py
@@ -5,7 +5,7 @@
 
 import frappe
 from frappe.model.document import Document
-from frappe.translate import MERGED_TRANSLATION_KEY, USER_TRANSLATION_KEY, get_translator_url
+from frappe.translate import MERGED_TRANSLATION_KEY, USER_TRANSLATION_KEY
 from frappe.utils import is_html, strip_html_tags
 
 
diff --git a/frappe/hooks.py b/frappe/hooks.py
index 0aa2f833075..5d4c11f0f30 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -6,18 +6,12 @@
 app_title = "Frappe Framework"
 app_publisher = "Frappe Technologies"
 app_description = "Full stack web framework with Python, Javascript, MariaDB, Redis, Node"
-source_link = "https://github.com/frappe/frappe"
 app_license = "MIT"
 app_logo_url = "/assets/frappe/images/frappe-framework-logo.svg"
-
 develop_version = "15.x.x-develop"
 
 app_email = "developers@frappe.io"
 
-docs_app = "frappe_docs"
-
-translator_url = "https://translate.erpnext.com"
-
 before_install = "frappe.utils.install.before_install"
 after_install = "frappe.utils.install.after_install"
 
@@ -248,7 +242,6 @@
 	],
 	"daily_long": [
 		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",
-		"frappe.utils.change_log.check_for_update",
 		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_daily",
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
 		"frappe.integrations.doctype.google_drive.google_drive.daily_backup",
@@ -257,6 +250,7 @@
 		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly",
 		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_weekly",
 		"frappe.desk.form.document_follow.send_weekly_updates",
+		"frappe.utils.change_log.check_for_update",
 		"frappe.social.doctype.energy_point_log.energy_point_log.send_weekly_summary",
 		"frappe.integrations.doctype.google_drive.google_drive.weekly_backup",
 		"frappe.desk.doctype.changelog_feed.changelog_feed.fetch_changelog_feed",
@@ -546,3 +540,10 @@
 	"Activity Log": 90,
 	"Route History": 90,
 }
+
+# These keys will not be erased when doing frappe.clear_cache()
+persistent_cache_keys = [
+	"update-user-set",
+	"update-info",
+	"insert_queue_for_*",  # Deferred Insert
+]
diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js
index 475c5f70ca3..a472507b2bd 100644
--- a/frappe/public/js/frappe/desk.js
+++ b/frappe/public/js/frappe/desk.js
@@ -430,11 +430,8 @@ frappe.Application = class Application {
 	}
 
 	show_update_available() {
-		if (frappe.boot.sysdefaults.disable_system_update_notification) return;
-
-		frappe.call({
-			method: "frappe.utils.change_log.show_update_popup",
-		});
+		if (!frappe.boot.has_app_updates) return;
+		frappe.xcall("frappe.utils.change_log.show_update_popup");
 	}
 
 	add_browser_class() {
diff --git a/frappe/sessions.py b/frappe/sessions.py
index 37c1fc97ea3..e91777be2eb 100644
--- a/frappe/sessions.py
+++ b/frappe/sessions.py
@@ -20,6 +20,7 @@
 from frappe.cache_manager import clear_user_cache
 from frappe.query_builder import Order
 from frappe.utils import cint, cstr, get_assets_json
+from frappe.utils.change_log import has_app_update_notifications
 from frappe.utils.data import add_to_date
 
 
@@ -169,6 +170,7 @@ def get():
 	bootinfo["desk_theme"] = frappe.db.get_value("User", frappe.session.user, "desk_theme") or "Light"
 	bootinfo["user"]["impersonated_by"] = frappe.session.data.get("impersonated_by")
 	bootinfo["navbar_settings"] = frappe.get_cached_doc("Navbar Settings")
+	bootinfo.has_app_updates = has_app_update_notifications()
 
 	return bootinfo
 
diff --git a/frappe/tests/test_deferred_insert.py b/frappe/tests/test_deferred_insert.py
index 4f27bef4f0b..8c44b875911 100644
--- a/frappe/tests/test_deferred_insert.py
+++ b/frappe/tests/test_deferred_insert.py
@@ -10,3 +10,9 @@ def test_deferred_insert(self):
 
 		save_to_db()
 		self.assertTrue(frappe.db.exists("Route History", route_history))
+
+		route_history = {"route": frappe.generate_hash(), "user": "Administrator"}
+		deferred_insert("Route History", [route_history])
+		frappe.clear_cache()  # deferred_insert cache keys are supposed to be persistent
+		save_to_db()
+		self.assertTrue(frappe.db.exists("Route History", route_history))
diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py
index 41e5ed6c18e..d34d5c55391 100644
--- a/frappe/tests/test_utils.py
+++ b/frappe/tests/test_utils.py
@@ -46,6 +46,10 @@
 	validate_phone_number_with_country_code,
 	validate_url,
 )
+from frappe.utils.change_log import (
+	get_source_url,
+	parse_github_url,
+)
 from frappe.utils.data import (
 	add_to_date,
 	add_years,
@@ -1032,13 +1036,13 @@ def test_validate_whitelisted_doc_method(self):
 class TestTBSanitization(FrappeTestCase):
 	def test_traceback_sanitzation(self):
 		try:
-			password = "42"  # noqa: F841
-			args = {"password": "42", "pwd": "42", "safe": "safe_value"}
-			args = frappe._dict({"password": "42", "pwd": "42", "safe": "safe_value"})  # noqa: F841
+			password = frappe.generate_hash()
+			args = {"password": "42", "pwd": password, "safe": "safe_value"}
+			args = frappe._dict({"password": "42", "pwd": password, "safe": "safe_value"})  # noqa: F841
 			raise Exception
 		except Exception:
 			traceback = frappe.get_traceback(with_context=True)
-			self.assertNotIn("42", traceback)
+			self.assertNotIn(password, traceback)
 			self.assertIn("********", traceback)
 			self.assertIn("password =", traceback)
 			self.assertIn("safe_value", traceback)
@@ -1246,6 +1250,31 @@ def test_doctypes(a: DocType | dict):
 			test_doctypes("a")
 
 
+class TestChangeLog(FrappeTestCase):
+	def test_get_remote_url(self):
+		self.assertIsInstance(get_source_url("frappe"), str)
+
+	def test_parse_github_url(self):
+		# using erpnext as repo in order to be different from the owner
+		owner, repo = parse_github_url("https://github.com/frappe/erpnext.git")
+		self.assertEqual(owner, "frappe")
+		self.assertEqual(repo, "erpnext")
+
+		owner, repo = parse_github_url("https://github.com/frappe/erpnext")
+		self.assertEqual(owner, "frappe")
+		self.assertEqual(repo, "erpnext")
+
+		owner, repo = parse_github_url("git@github.com:frappe/erpnext.git")
+		self.assertEqual(owner, "frappe")
+		self.assertEqual(repo, "erpnext")
+
+		owner, repo = parse_github_url("https://gitlab.com/gitlab-org/gitlab")
+		self.assertIsNone(owner)
+		self.assertIsNone(repo)
+
+		self.assertRaises(ValueError, parse_github_url, remote_url=None)
+
+
 class TestCrypto(FrappeTestCase):
 	def test_hashing(self):
 		self.assertEqual(sha256_hash(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
diff --git a/frappe/translate.py b/frappe/translate.py
index a293bb7937b..48240c60298 100644
--- a/frappe/translate.py
+++ b/frappe/translate.py
@@ -929,49 +929,6 @@ def get_translations(source_text):
 	)
 
 
-@frappe.whitelist()
-def get_messages(language, start=0, page_length=100, search_text=""):
-	from frappe.frappeclient import FrappeClient
-
-	translator = FrappeClient(get_translator_url())
-	return translator.post_api("translator.api.get_strings_for_translation", params=locals())
-
-
-@frappe.whitelist()
-def get_source_additional_info(source, language=""):
-	from frappe.frappeclient import FrappeClient
-
-	translator = FrappeClient(get_translator_url())
-	return translator.post_api("translator.api.get_source_additional_info", params=locals())
-
-
-@frappe.whitelist()
-def get_contributions(language):
-	return frappe.get_all(
-		"Translation",
-		fields=["*"],
-		filters={
-			"contributed": 1,
-		},
-	)
-
-
-@frappe.whitelist()
-def get_contribution_status(message_id):
-	from frappe.frappeclient import FrappeClient
-
-	doc = frappe.get_doc("Translation", message_id)
-	translator = FrappeClient(get_translator_url())
-	return translator.get_api(
-		"translator.api.get_contribution_status",
-		params={"translation_id": doc.contribution_docname},
-	)
-
-
-def get_translator_url():
-	return frappe.get_hooks()["translator_url"][0]
-
-
 @frappe.whitelist(allow_guest=True)
 def get_all_languages(with_language_name: bool = False) -> list:
 	"""Return all enabled language codes ar, ch etc."""
diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py
index 93f35d25cf7..d20ead24896 100644
--- a/frappe/utils/change_log.py
+++ b/frappe/utils/change_log.py
@@ -4,12 +4,14 @@
 import json
 import os
 import subprocess  # nosec
+from contextlib import suppress
 
-from semantic_version import Version
+from semantic_version import SimpleSpec, Version
 
 import frappe
 from frappe import _, safe_decode
 from frappe.utils import cstr
+from frappe.utils.frappecloud import on_frappecloud
 
 
 def get_change_log(user=None):
@@ -129,7 +131,7 @@ def get_versions():
 
 
 def get_app_branch(app):
-	"""Returns branch of an app"""
+	"""Return branch of an app."""
 	try:
 		with open(os.devnull, "wb") as null_stream:
 			result = subprocess.check_output(
@@ -162,21 +164,31 @@ def get_app_last_commit_ref(app):
 
 
 def check_for_update():
+	if frappe.get_system_settings("disable_system_update_notification"):
+		return
+
 	updates = frappe._dict(major=[], minor=[], patch=[])
 	apps = get_versions()
 
 	for app in apps:
-		app_details = check_release_on_github(app)
-		if not app_details:
+		remote_url = get_source_url(app)
+		if not remote_url:
 			continue
 
-		github_version, org_name = app_details
-		# Get local instance's current version or the app
+		owner, repo = parse_github_url(remote_url)
+		if not owner or not repo:
+			continue
 
+		# Get local instance's current version or the app
 		branch_version = (
 			apps[app]["branch_version"].split(" ", 1)[0] if apps[app].get("branch_version", "") else ""
 		)
 		instance_version = Version(branch_version or apps[app].get("version"))
+
+		github_version, org_name = check_release_on_github(owner, repo, instance_version)
+		if not github_version or not org_name:
+			continue
+
 		# Compare and popup update message
 		for update_type in updates:
 			if github_version.__dict__[update_type] > instance_version.__dict__[update_type]:
@@ -187,6 +199,7 @@ def check_for_update():
 						org_name=org_name,
 						app_name=app,
 						title=apps[app]["title"],
+						security_issues=security_issues_count(owner, repo, instance_version, github_version),
 					)
 				)
 				break
@@ -194,72 +207,102 @@ def check_for_update():
 				break
 
 	add_message_to_redis(updates)
+	return updates
 
 
-def parse_latest_non_beta_release(response):
-	"""
-	Parses the response JSON for all the releases and returns the latest non prerelease
+def has_app_update_notifications() -> bool:
+	return bool(frappe.cache.sismember("update-user-set", frappe.session.user))
+
+
+def parse_latest_non_beta_release(response: list, current_version: Version) -> list | None:
+	"""Parse the response JSON for all the releases and return the latest non prerelease.
+
+	Args:
 
-	Parameters
 	response (list): response object returned by github
 
-	Returns
-	json   : json object pertaining to the latest non-beta release
+	Return a json object pertaining to the latest non-beta release
 	"""
 	version_list = [
 		release.get("tag_name").strip("v") for release in response if not release.get("prerelease")
 	]
 
+	def prioritize_minor_update(v: str) -> Version:
+		target = Version(v)
+		return (current_version.major == target.major, target)
+
 	if version_list:
-		return sorted(version_list, key=Version, reverse=True)[0]
+		return sorted(version_list, key=prioritize_minor_update, reverse=True)[0]
 
 	return None
 
 
-def check_release_on_github(app: str):
-	"""
-	Check the latest release for a given Frappe application hosted on Github.
+def check_release_on_github(
+	owner: str, repo: str, current_version: Version
+) -> tuple[Version, str] | tuple[None, None]:
+	"""Check the latest release for a repo URL on GitHub."""
+	import requests
 
-	Args:
-	        app (str): The name of the Frappe application.
+	if not owner:
+		raise ValueError("Owner cannot be empty")
+
+	if not repo:
+		raise ValueError("Repo cannot be empty")
+
+	# Get latest version from GitHub
+	r = requests.get(f"https://api.github.com/repos/{owner}/{repo}/releases")
+	if r.ok:
+		latest_non_beta_release = parse_latest_non_beta_release(r.json(), current_version)
+		if latest_non_beta_release:
+			return Version(latest_non_beta_release), owner
+
+	return None, None
 
-	Returns:
-	        tuple(Version, str): The semantic version object of the latest release and the
-	                organization name, if the application exists, otherwise None.
-	"""
 
+def security_issues_count(owner: str, repo: str, current_version: Version, target_version: Version) -> int:
 	import requests
-	from giturlparse import parse
-	from giturlparse.parser import ParserError
 
-	try:
-		# Check if repo remote is on github
-		remote_url = subprocess.check_output(f"cd ../apps/{app} && git ls-remote --get-url", shell=True)
-	except subprocess.CalledProcessError:
-		# Passing this since some apps may not have git initialized in them
-		return
+	r = requests.get(f"https://api.github.com/repos/{owner}/{repo}/security-advisories")
+	if not r.ok:
+		return 0
+	advisories = r.json()
 
-	if isinstance(remote_url, bytes):
-		remote_url = remote_url.decode()
+	def applicable(advisory) -> bool:
+		# Current version is in vulnerable range
+		# Target version is not in vulnerabe range
+		for vuln in advisory["vulnerabilities"]:
+			with suppress(Exception):
+				vulnerable_range = SimpleSpec(vuln["vulnerable_version_range"].replace(" ", ""))
+				patch_version = Version(vuln["patched_versions"].replace(" ", ""))
+				if (
+					current_version in vulnerable_range
+					and target_version not in vulnerable_range
+					# XXX: this is not 100% correct, but works for frappe
+					and current_version.major == patch_version.major
+				):
+					return True
 
-	try:
-		parsed_url = parse(remote_url)
-	except ParserError:
-		# Invalid URL
-		return
+	return len([sa for sa in advisories if applicable(sa)])
 
-	if parsed_url.resource != "github.com":
-		return
 
-	owner = parsed_url.owner
-	repo = parsed_url.name
+def parse_github_url(remote_url: str) -> tuple[str, str] | tuple[None, None]:
+	"""Parse the remote URL to get the owner and repo name."""
+	import re
+
+	if not remote_url:
+		raise ValueError("Remote URL cannot be empty")
+
+	pattern = r"github\.com[:/](.+)\/([^\.]+)"
+	match = re.search(pattern, remote_url)
+
+	return (match[1], match[2]) if match else (None, None)
 
-	# Get latest version from GitHub
-	r = requests.get(f"https://api.github.com/repos/{owner}/{repo}/releases")
-	if r.ok:
-		latest_non_beta_release = parse_latest_non_beta_release(r.json())
-		if latest_non_beta_release:
-			return Version(latest_non_beta_release), owner
+
+def get_source_url(app: str) -> str | None:
+	"""Get the remote URL of the app."""
+	pyproject = get_pyproject(app)
+	if remote_url := pyproject.get("project", {}).get("urls", {}).get("Repository"):
+		return remote_url.rstrip("/")
 
 
 def add_message_to_redis(update_json):
@@ -273,6 +316,8 @@ def add_message_to_redis(update_json):
 
 @frappe.whitelist()
 def show_update_popup():
+	if frappe.get_system_settings("disable_system_update_notification"):
+		return
 	user = frappe.session.user
 
 	update_info = frappe.cache.get_value("update-info")
@@ -288,12 +333,23 @@ def show_update_popup():
 			release_links = ""
 			for app in updates[update_type]:
 				app = frappe._dict(app)
-				release_links += "<b>{title}</b>: <a href='https://github.com/{org_name}/{app_name}/releases/tag/v{available_version}'>v{available_version}</a><br>".format(
-					available_version=app.available_version,
-					org_name=app.org_name,
-					app_name=app.app_name,
-					title=app.title,
-				)
+				security_msg = ""
+				if app.security_issues:
+					security_msg = (
+						_("Contains {0} security fixes")
+						if app.security_issues > 1
+						else _("Contains {0} security fix")
+					)
+					security_msg = security_msg.format(frappe.bold(app.security_issues))
+					security_msg = f"""( <a href='https://github.com/{app.org_name}/{app.app_name}/security/advisories'
+						 target='_blank'>{security_msg}</a> )"""
+				release_links += f"""
+					<b>{app.title}</b>:
+						<a href='https://github.com/{app.org_name}/{app.app_name}/releases/tag/v{app.available_version}'
+							target="_blank">
+							v{app.available_version}
+						</a> {security_msg}<br>
+					"""
 			if release_links:
 				message = _("New {} releases for the following apps are available").format(_(update_type))
 				update_message += (
@@ -302,6 +358,31 @@ def show_update_popup():
 					)
 				)
 
+	primary_action = None
+	if on_frappecloud():
+		primary_action = {
+			"label": _("Update from Frappe Cloud"),
+			"client_action": "window.open",
+			"args": f"https://frappecloud.com/dashboard/sites/{frappe.local.site}",
+		}
+
 	if update_message:
-		frappe.msgprint(update_message, title=_("New updates are available"), indicator="green")
+		frappe.msgprint(
+			update_message,
+			title=_("New updates are available"),
+			indicator="green",
+			primary_action=primary_action,
+		)
 		frappe.cache.srem("update-user-set", user)
+
+
+def get_pyproject(app: str) -> dict | None:
+	from tomli import load
+
+	pyproject_path = frappe.get_app_path(app, "..", "pyproject.toml")
+
+	if not os.path.exists(pyproject_path):
+		return None
+
+	with open(pyproject_path, "rb") as f:
+		return load(f)
diff --git a/frappe/utils/frappecloud.py b/frappe/utils/frappecloud.py
new file mode 100644
index 00000000000..5244429f18f
--- /dev/null
+++ b/frappe/utils/frappecloud.py
@@ -0,0 +1,11 @@
+import frappe
+
+FRAPPE_CLOUD_DOMAINS = ("frappe.cloud", "erpnext.com", "frappehr.com")
+
+
+def on_frappecloud() -> bool:
+	"""Returns true if running on Frappe Cloud.
+
+
+	Useful for modifying few features for better UX."""
+	return frappe.local.site.endswith(FRAPPE_CLOUD_DOMAINS)
diff --git a/pyproject.toml b/pyproject.toml
index 85d82defeba..844fd8921a5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,7 +37,6 @@ dependencies = [
     "cryptography~=42.0.0",
     "cssutils~=2.9.0",
     "email-reply-parser~=0.5.12",
-    "git-url-parse~=1.2.2",
     "gunicorn~=22.0.0",
     "html5lib~=1.1",
     "ipython~=8.15.0",
@@ -74,6 +73,7 @@ dependencies = [
     "terminaltables~=3.1.10",
     "traceback-with-variables~=2.0.4",
     "typing_extensions>=4.6.1,<5",
+    "tomli~=2.0.1",
     "xlrd~=2.0.1",
     "zxcvbn~=4.4.28",
     "markdownify~=0.11.6",
@@ -87,6 +87,11 @@ dependencies = [
     "posthog~=3.0.1",
 ]
 
+[project.urls]
+Homepage = "https://frappeframework.com/"
+Repository = "https://github.com/frappe/frappe.git"
+"Bug Reports" = "https://github.com/frappe/frappe/issues"
+
 [build-system]
 requires = ["flit_core >=3.4,<4"]
 build-backend = "flit_core.buildapi"

From f6a07d465093f6bd1c3c05d1888d420cb6970210 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Mon, 6 May 2024 11:13:51 +0000
Subject: [PATCH 26/34] test: smtp4dev API changes (#26330) (#26333)

(cherry picked from commit aa48d7d84aaa45fcf5e009390884fc20467193ca)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/tests/test_email.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py
index f9d0adda39d..a481ef18c50 100644
--- a/frappe/tests/test_email.py
+++ b/frappe/tests/test_email.py
@@ -348,9 +348,7 @@ def tearDown(self) -> None:
 
 	@classmethod
 	def get_last_sent_emails(cls):
-		return requests.get(
-			f"{cls.SMTP4DEV_WEB}/api/Messages?sortColumn=receivedDate&sortIsDescending=true"
-		).json()
+		return requests.get(f"{cls.SMTP4DEV_WEB}/api/Messages").json().get("results")
 
 	@classmethod
 	def get_message(cls, message_id):
@@ -376,7 +374,7 @@ def test_send_email(self):
 		for sent_mail in sent_mails:
 			self.assertEqual(sent_mail["from"], sender)
 			self.assertEqual(sent_mail["subject"], subject)
-		self.assertSetEqual(set(recipients.split(",")), {m["to"] for m in sent_mails})
+		self.assertSetEqual(set(recipients.split(",")), {m["to"][0] for m in sent_mails})
 
 	@run_only_if(db_type_is.MARIADB)
 	@change_settings("System Settings", store_attached_pdf_document=1)

From e2c843688b0de4d3f3cc96f73f9646c604b97ba6 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Mon, 6 May 2024 17:59:29 +0530
Subject: [PATCH 27/34] fix: Apply configured perms on address list (#26334)
 (#26336)

(cherry picked from commit 1defbf5be829af92a5df5f658f62a9b843abcb68)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/contacts/doctype/address/address.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py
index 811d90bd74a..6b40763d1ed 100644
--- a/frappe/contacts/doctype/address/address.py
+++ b/frappe/contacts/doctype/address/address.py
@@ -214,15 +214,12 @@ def get_address_list(doctype, txt, filters, limit_start, limit_page_length=20, o
 	from frappe.www.list import get_list
 
 	user = frappe.session.user
-	ignore_permissions = True
 
 	if not filters:
 		filters = []
 	filters.append(("Address", "owner", "=", user))
 
-	return get_list(
-		doctype, txt, filters, limit_start, limit_page_length, ignore_permissions=ignore_permissions
-	)
+	return get_list(doctype, txt, filters, limit_start, limit_page_length)
 
 
 def has_website_permission(doc, ptype, user, verbose=False):

From 6b63f460c20b71c2b81e89a4789ac81c4e85716b Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Mon, 6 May 2024 12:51:50 +0000
Subject: [PATCH 28/34] test: increase threshold for perf test (#26316)
 (#26329)

We are >2x faster since the time this test was written :smile:

(cherry picked from commit 2b95aa66a77ba2f621450bec831cb5a893362e1e)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/tests/test_perf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/tests/test_perf.py b/frappe/tests/test_perf.py
index 91e8c5c890b..e0a9e4bf927 100644
--- a/frappe/tests/test_perf.py
+++ b/frappe/tests/test_perf.py
@@ -128,7 +128,7 @@ def test_req_per_seconds_basic(self):
 		"""Ideally should be ran against gunicorn worker, though I have not seen any difference
 		when using werkzeug's run_simple for synchronous requests."""
 
-		EXPECTED_RPS = 50  # measured on GHA
+		EXPECTED_RPS = 120  # measured on GHA
 		FAILURE_THREASHOLD = 0.1
 
 		req_count = 1000

From 32dbc5ab9d73acad618a10cdce90659d8f40efee Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Mon, 6 May 2024 13:13:11 +0000
Subject: [PATCH 29/34] perf: Reduce 1 redis call while dumping monitor logs
 (#26337) (#26339)

(cherry picked from commit 724d886f889f8ddb972bc4a83c52db0278c3c0b5)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/monitor.py             | 7 ++++---
 frappe/utils/redis_wrapper.py | 4 ++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/frappe/monitor.py b/frappe/monitor.py
index 9b8f5003582..ed150a18ca5 100644
--- a/frappe/monitor.py
+++ b/frappe/monitor.py
@@ -10,6 +10,7 @@
 import rq
 
 import frappe
+from frappe.utils.data import cint
 
 MONITOR_REDIS_KEY = "monitor-transactions"
 MONITOR_MAX_ENTRIES = 1000000
@@ -114,10 +115,10 @@ def dump(self, response=None):
 			traceback.print_exc()
 
 	def store(self):
-		if frappe.cache.llen(MONITOR_REDIS_KEY) > MONITOR_MAX_ENTRIES:
-			frappe.cache.ltrim(MONITOR_REDIS_KEY, 1, -1)
 		serialized = json.dumps(self.data, sort_keys=True, default=str, separators=(",", ":"))
-		frappe.cache.rpush(MONITOR_REDIS_KEY, serialized)
+		length = frappe.cache.rpush(MONITOR_REDIS_KEY, serialized)
+		if cint(length) > MONITOR_MAX_ENTRIES:
+			frappe.cache.ltrim(MONITOR_REDIS_KEY, 1, -1)
 
 
 def flush():
diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py
index f8ee1cd8db0..44bfbd1e9ca 100644
--- a/frappe/utils/redis_wrapper.py
+++ b/frappe/utils/redis_wrapper.py
@@ -149,10 +149,10 @@ def delete_value(self, keys, user=None, make_keys=True, shared=False):
 			pass
 
 	def lpush(self, key, value):
-		super().lpush(self.make_key(key), value)
+		return super().lpush(self.make_key(key), value)
 
 	def rpush(self, key, value):
-		super().rpush(self.make_key(key), value)
+		return super().rpush(self.make_key(key), value)
 
 	def lpop(self, key):
 		return super().lpop(self.make_key(key))

From d802eb67fd10ca05b10491093a41cf4969e342ad Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Mon, 6 May 2024 13:45:49 +0000
Subject: [PATCH 30/34] perf: avoid two layer cache of timezone (backport
 #26340) (#26342)

* perf: avoid two layer cache of timezone (#26340)

It's same thing, system settings are always fetched, no idea what's point of additional layer of caching on top of it.

(cherry picked from commit f8abd09ab91d0634badc1197626789169839c365)

# Conflicts:
#	frappe/utils/data.py

* Update data.py

---------

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/utils/data.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/frappe/utils/data.py b/frappe/utils/data.py
index 143ebbaf5f2..3205cfd8495 100644
--- a/frappe/utils/data.py
+++ b/frappe/utils/data.py
@@ -312,17 +312,11 @@ def get_eta(from_time, percent_complete):
 	return str(datetime.timedelta(seconds=(100 - percent_complete) / percent_complete * diff))
 
 
-def _get_system_timezone():
+def get_system_timezone() -> str:
+	"""Return the system timezone."""
 	return frappe.get_system_settings("time_zone") or "Asia/Kolkata"  # Default to India ?!
 
 
-def get_system_timezone():
-	if frappe.local.flags.in_test:
-		return _get_system_timezone()
-
-	return frappe.cache.get_value("time_zone", _get_system_timezone)
-
-
 def convert_utc_to_timezone(utc_timestamp, time_zone):
 	from pytz import UnknownTimeZoneError, timezone
 

From 6e8de8188b34cf41d4802345ce2d612d4e787bb7 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 7 May 2024 06:12:42 +0000
Subject: [PATCH 31/34] fix: QB name in error message (#26345) (#26348)

"Query Engine" wording made the edited error message ambiguous, since the docs all refer to this feature as Query Builder

(cherry picked from commit cd50b025d5344d0e8a949d71fe8807d730edc788)

Co-authored-by: Kevin Shenk <kevin@avu.nu>
---
 frappe/database/database.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frappe/database/database.py b/frappe/database/database.py
index 25e86b34562..ff4e802b2f3 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -194,7 +194,7 @@ def sql(
 
 		"""
 		if isinstance(query, MySQLQueryBuilder | PostgreSQLQueryBuilder):
-			frappe.log("Use run method to execute SQL queries generated by Query Engine")
+			frappe.log("Use run method to execute SQL queries generated by Query Builder")
 
 		debug = debug or getattr(self, "debug", False)
 		query = str(query)

From e1048a8c40d4e553326bb5a4d5485974541b60bd Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 7 May 2024 13:23:47 +0530
Subject: [PATCH 32/34] perf: Reuse cached user for `get_user_lang` (#26350)
 (#26355)

Saves ~1% of overhead.

User doc is almost always cached to create session so we can reuse it in
same request without additional redis call.

(cherry picked from commit c17eb87c70aee9a39588691d994c30ab4dfd7423)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/translate.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/frappe/translate.py b/frappe/translate.py
index 48240c60298..609ec5168e4 100644
--- a/frappe/translate.py
+++ b/frappe/translate.py
@@ -97,20 +97,14 @@ def get_parent_language(lang: str) -> str:
 def get_user_lang(user: str | None = None) -> str:
 	"""Set frappe.local.lang from user preferences on session beginning or resumption"""
 	user = user or frappe.session.user
-	lang = frappe.cache.hget("lang", user)
 
-	if not lang:
-		# User.language => Session Defaults => frappe.local.lang => 'en'
-		lang = (
-			frappe.db.get_value("User", user, "language")
-			or frappe.db.get_default("lang")
-			or frappe.local.lang
-			or "en"
-		)
-
-		frappe.cache.hset("lang", user, lang)
-
-	return lang
+	# User.language => Session Defaults => frappe.local.lang => 'en'
+	return (
+		frappe.get_cached_value("User", user, "language")
+		or frappe.db.get_default("lang")
+		or frappe.local.lang
+		or "en"
+	)
 
 
 def get_lang_code(lang: str) -> str | None:

From 8961a96af47d8538128eae76a855e015bf12859a Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 7 May 2024 14:32:11 +0530
Subject: [PATCH 33/34] perf: Avoid caching `module_app` (#26349) (#26351)

module_app is just reverse of app_module.

This saves ~1.5% of overhead.

(cherry picked from commit e240c6bdf804bcbc4ac8b0fdb7b91eaed7b2cf1a)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/__init__.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/frappe/__init__.py b/frappe/__init__.py
index de7de2d897d..bc93e5c7aa7 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1677,34 +1677,37 @@ def setup_module_map(include_all_apps: bool = True) -> None:
 	"""
 	if include_all_apps:
 		local.app_modules = cache.get_value("app_modules")
-		local.module_app = cache.get_value("module_app")
 	else:
 		local.app_modules = cache.get_value("installed_app_modules")
-		local.module_app = cache.get_value("module_installed_app")
 
-	if not (local.app_modules and local.module_app):
-		local.module_app, local.app_modules = {}, {}
+	if not local.app_modules:
+		local.app_modules = {}
 		if include_all_apps:
 			apps = get_all_apps(with_internal_apps=True)
 		else:
 			apps = get_installed_apps(_ensure_on_bench=True)
+
 		for app in apps:
 			local.app_modules.setdefault(app, [])
 			for module in get_module_list(app):
 				module = scrub(module)
-				if module in local.module_app:
-					print(
-						f"WARNING: module `{module}` found in apps `{local.module_app[module]}` and `{app}`"
-					)
-				local.module_app[module] = app
 				local.app_modules[app].append(module)
 
 		if include_all_apps:
 			cache.set_value("app_modules", local.app_modules)
-			cache.set_value("module_app", local.module_app)
 		else:
 			cache.set_value("installed_app_modules", local.app_modules)
-			cache.set_value("module_installed_app", local.module_app)
+
+	# Init module_app (reverse mapping)
+	local.module_app = {}
+	for app, modules in local.app_modules.items():
+		for module in modules:
+			if module in local.module_app:
+				warnings.warn(
+					f"WARNING: module `{module}` found in apps `{local.module_app[module]}` and `{app}`",
+					stacklevel=1,
+				)
+			local.module_app[module] = app
 
 
 def get_file_items(path, raise_not_found=False, ignore_empty_lines=True):

From e8195e9224f5e2059c4833d0265fb59f083c410e Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 7 May 2024 15:59:41 +0530
Subject: [PATCH 34/34] test: check redis calls and overhead (#26356) (#26359)

(cherry picked from commit f07ea138d501f33e55f48f5d3cafc0dfb6adba7d)

Co-authored-by: Ankush Menat <ankush@frappe.io>
---
 frappe/tests/test_perf.py | 38 ++++++++++++++++++++++++++++++++++++++
 frappe/tests/utils.py     | 32 +++++++++++++++++++++++++++-----
 2 files changed, 65 insertions(+), 5 deletions(-)

diff --git a/frappe/tests/test_perf.py b/frappe/tests/test_perf.py
index e0a9e4bf927..98ab0453f62 100644
--- a/frappe/tests/test_perf.py
+++ b/frappe/tests/test_perf.py
@@ -27,6 +27,7 @@
 from frappe.frappeclient import FrappeClient
 from frappe.model.base_document import get_controller
 from frappe.query_builder.utils import db_type_is
+from frappe.tests.test_api import FrappeAPITestCase
 from frappe.tests.test_query_builder import run_only_if
 from frappe.tests.utils import FrappeTestCase
 from frappe.utils import cint
@@ -196,3 +197,40 @@ def test_no_stale_ref_sql(self):
 	def test_no_cyclic_references(self):
 		doc = frappe.get_doc("User", "Administrator")
 		self.assertEqual(sys.getrefcount(doc), 2)  # Note: This always returns +1
+
+	def test_get_doc_cache_calls(self):
+		frappe.get_doc("User", "Administrator")
+		with self.assertRedisCallCounts(1):
+			frappe.get_doc("User", "Administrator")
+
+
+@run_only_if(db_type_is.MARIADB)
+class TestOverheadCalls(FrappeAPITestCase):
+	"""Test that typical redis and db calls remain same overtime.
+
+	If this tests fail on your PR, make sure you're not introducing something in hot-path of these
+	endpoints. Only update values if you're really sure that's the right call.
+	Every call increase here is an actual increase in cost!
+	"""
+
+	BASE_SQL_CALLS = 2  # rollback + begin
+
+	def test_ping_overheads(self):
+		self.get(self.method("ping"), {"sid": "Guest"})
+		with self.assertRedisCallCounts(12), self.assertQueryCount(self.BASE_SQL_CALLS):
+			self.get(self.method("ping"), {"sid": "Guest"})
+
+	def test_list_view_overheads(self):
+		sid = self.sid
+		self.get(self.resource("ToDo"), {"sid": sid})
+		self.get(self.resource("ToDo"), {"sid": sid})
+		with self.assertRedisCallCounts(24), self.assertQueryCount(self.BASE_SQL_CALLS + 1):
+			self.get(self.resource("ToDo"), {"sid": sid})
+
+	def test_get_doc_overheads(self):
+		sid = self.sid
+		tables = len(frappe.get_meta("User").get_table_fields())
+		self.get(self.resource("User", "Administrator"), {"sid": sid})
+		self.get(self.resource("User", "Administrator"), {"sid": sid})
+		with self.assertRedisCallCounts(19), self.assertQueryCount(self.BASE_SQL_CALLS + 1 + tables):
+			self.get(self.resource("User", "Administrator"), {"sid": sid})
diff --git a/frappe/tests/utils.py b/frappe/tests/utils.py
index 92af12eb4df..633b0419757 100644
--- a/frappe/tests/utils.py
+++ b/frappe/tests/utils.py
@@ -134,16 +134,38 @@ def assertQueryCount(self, count):
 
 		def _sql_with_count(*args, **kwargs):
 			ret = orig_sql(*args, **kwargs)
-			queries.append(frappe.db.last_query)
+			queries.append(args[0].last_query)
 			return ret
 
 		try:
-			orig_sql = frappe.db.sql
-			frappe.db.sql = _sql_with_count
+			orig_sql = frappe.db.__class__.sql
+			frappe.db.__class__.sql = _sql_with_count
 			yield
-			self.assertLessEqual(len(queries), count, msg="Queries executed: " + "\n\n".join(queries))
+			self.assertLessEqual(len(queries), count, msg="Queries executed: \n" + "\n\n".join(queries))
 		finally:
-			frappe.db.sql = orig_sql
+			frappe.db.__class__.sql = orig_sql
+
+	@contextmanager
+	def assertRedisCallCounts(self, count):
+		commands = []
+
+		def execute_command_and_count(*args, **kwargs):
+			ret = orig_execute(*args, **kwargs)
+			key_len = 2
+			if "H" in args[0]:
+				key_len = 3
+			commands.append((args)[:key_len])
+			return ret
+
+		try:
+			orig_execute = frappe.cache.execute_command
+			frappe.cache.execute_command = execute_command_and_count
+			yield
+			self.assertLessEqual(
+				len(commands), count, msg="commands executed: \n" + "\n".join(str(c) for c in commands)
+			)
+		finally:
+			frappe.cache.execute_command = orig_execute
 
 	@contextmanager
 	def assertRowsRead(self, count):