feat: improved update notifications (backport #26314) (#26320)

* fix: update notifier never running

(cherry picked from commit 330a1b2)

# Conflicts:
#	frappe/utils/change_log.py

* chore: dead hooks

(cherry picked from commit 56233f3)

* feat: FC specific update notifications

(cherry picked from commit 5ca14bb)

# Conflicts:
#	frappe/tests/test_utils.py
#	frappe/utils/change_log.py
#	pyproject.toml

* feat: show security issues count

chore: conflicts
(cherry picked from commit e0c171b)

* refactor: misc changes

- Move sys setting check to server side
- tomli import handling

(cherry picked from commit a234e79)

* feat: give higher priority to minor versions

Major versions will be shown when minor updates are exhausted only.

(cherry picked from commit 45026ae)

# Conflicts:
#	frappe/utils/change_log.py

* feat: Keep certain keys persistent in cache

(cherry picked from commit 8ab3088)

* chore: conflicts

* chore: drop git url parse dependency

---------

Co-authored-by: Ankush Menat <ankush@frappe.io>

frappe/__init__.py

@@ -1013,7 +1013,11 @@ def clear_cache(user: str | None = None, doctype: str | None = None):
 		frappe.cache_manager.clear_user_cache(user)
 	else:  # everything
 		# Delete ALL keys associated with this site.
-		frappe.cache.delete_keys("")
+		keys_to_delete = set(frappe.cache.get_keys(""))
+		for key in frappe.get_hooks("persistent_cache_keys"):
+			keys_to_delete.difference_update(frappe.cache.get_keys(key))
+		frappe.cache.delete_value(list(keys_to_delete), make_keys=False)
+
 		reset_metadata_version()
 		local.cache = {}
 		local.new_doc_templates = {}

frappe/boot.py

@@ -24,6 +24,7 @@
 )
 from frappe.utils import add_user_info, cstr, get_system_timezone
 from frappe.utils.change_log import get_versions
+from frappe.utils.frappecloud import on_frappecloud
 from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled
 
 
@@ -448,7 +449,7 @@ def get_marketplace_apps():
 	apps = []
 	cache_key = "frappe_marketplace_apps"
 
-	if frappe.conf.developer_mode:
+	if frappe.conf.developer_mode or not on_frappecloud():
 		return apps
 
 	def get_apps_from_fc():

frappe/core/doctype/translation/translation.py

@@ -5,7 +5,7 @@
 
 import frappe
 from frappe.model.document import Document
-from frappe.translate import MERGED_TRANSLATION_KEY, USER_TRANSLATION_KEY, get_translator_url
+from frappe.translate import MERGED_TRANSLATION_KEY, USER_TRANSLATION_KEY
 from frappe.utils import is_html, strip_html_tags
 
 

frappe/hooks.py

@@ -6,18 +6,12 @@
 app_title = "Frappe Framework"
 app_publisher = "Frappe Technologies"
 app_description = "Full stack web framework with Python, Javascript, MariaDB, Redis, Node"
-source_link = "https://github.com/frappe/frappe"
 app_license = "MIT"
 app_logo_url = "/assets/frappe/images/frappe-framework-logo.svg"
-
 develop_version = "15.x.x-develop"
 
 app_email = "developers@frappe.io"
 
-docs_app = "frappe_docs"
-
-translator_url = "https://translate.erpnext.com"
-
 before_install = "frappe.utils.install.before_install"
 after_install = "frappe.utils.install.after_install"
 
@@ -248,7 +242,6 @@
 	],
 	"daily_long": [
 		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",
-		"frappe.utils.change_log.check_for_update",
 		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_daily",
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
 		"frappe.integrations.doctype.google_drive.google_drive.daily_backup",
@@ -257,6 +250,7 @@
 		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly",
 		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_weekly",
 		"frappe.desk.form.document_follow.send_weekly_updates",
+		"frappe.utils.change_log.check_for_update",
 		"frappe.social.doctype.energy_point_log.energy_point_log.send_weekly_summary",
 		"frappe.integrations.doctype.google_drive.google_drive.weekly_backup",
 		"frappe.desk.doctype.changelog_feed.changelog_feed.fetch_changelog_feed",
@@ -546,3 +540,10 @@
 	"Activity Log": 90,
 	"Route History": 90,
 }
+
+# These keys will not be erased when doing frappe.clear_cache()
+persistent_cache_keys = [
+	"update-user-set",
+	"update-info",
+	"insert_queue_for_*",  # Deferred Insert
+]

frappe/public/js/frappe/desk.js

@@ -430,11 +430,8 @@ frappe.Application = class Application {
 	}
 
 	show_update_available() {
-		if (frappe.boot.sysdefaults.disable_system_update_notification) return;
-
-		frappe.call({
-			method: "frappe.utils.change_log.show_update_popup",
-		});
+		if (!frappe.boot.has_app_updates) return;
+		frappe.xcall("frappe.utils.change_log.show_update_popup");
 	}
 
 	add_browser_class() {

frappe/sessions.py

@@ -20,6 +20,7 @@
 from frappe.cache_manager import clear_user_cache
 from frappe.query_builder import Order
 from frappe.utils import cint, cstr, get_assets_json
+from frappe.utils.change_log import has_app_update_notifications
 from frappe.utils.data import add_to_date
 
 
@@ -169,6 +170,7 @@ def get():
 	bootinfo["desk_theme"] = frappe.db.get_value("User", frappe.session.user, "desk_theme") or "Light"
 	bootinfo["user"]["impersonated_by"] = frappe.session.data.get("impersonated_by")
 	bootinfo["navbar_settings"] = frappe.get_cached_doc("Navbar Settings")
+	bootinfo.has_app_updates = has_app_update_notifications()
 
 	return bootinfo
 

frappe/tests/test_deferred_insert.py

@@ -10,3 +10,9 @@ def test_deferred_insert(self):
 
 		save_to_db()
 		self.assertTrue(frappe.db.exists("Route History", route_history))
+
+		route_history = {"route": frappe.generate_hash(), "user": "Administrator"}
+		deferred_insert("Route History", [route_history])
+		frappe.clear_cache()  # deferred_insert cache keys are supposed to be persistent
+		save_to_db()
+		self.assertTrue(frappe.db.exists("Route History", route_history))

frappe/tests/test_utils.py

@@ -46,6 +46,10 @@
 	validate_phone_number_with_country_code,
 	validate_url,
 )
+from frappe.utils.change_log import (
+	get_source_url,
+	parse_github_url,
+)
 from frappe.utils.data import (
 	add_to_date,
 	add_years,
@@ -1032,13 +1036,13 @@ def test_validate_whitelisted_doc_method(self):
 class TestTBSanitization(FrappeTestCase):
 	def test_traceback_sanitzation(self):
 		try:
-			password = "42"  # noqa: F841
-			args = {"password": "42", "pwd": "42", "safe": "safe_value"}
-			args = frappe._dict({"password": "42", "pwd": "42", "safe": "safe_value"})  # noqa: F841
+			password = frappe.generate_hash()
+			args = {"password": "42", "pwd": password, "safe": "safe_value"}
+			args = frappe._dict({"password": "42", "pwd": password, "safe": "safe_value"})  # noqa: F841
 			raise Exception
 		except Exception:
 			traceback = frappe.get_traceback(with_context=True)
-			self.assertNotIn("42", traceback)
+			self.assertNotIn(password, traceback)
 			self.assertIn("********", traceback)
 			self.assertIn("password =", traceback)
 			self.assertIn("safe_value", traceback)
@@ -1246,6 +1250,31 @@ def test_doctypes(a: DocType | dict):
 			test_doctypes("a")
 
 
+class TestChangeLog(FrappeTestCase):
+	def test_get_remote_url(self):
+		self.assertIsInstance(get_source_url("frappe"), str)
+
+	def test_parse_github_url(self):
+		# using erpnext as repo in order to be different from the owner
+		owner, repo = parse_github_url("https://github.com/frappe/erpnext.git")
+		self.assertEqual(owner, "frappe")
+		self.assertEqual(repo, "erpnext")
+
+		owner, repo = parse_github_url("https://github.com/frappe/erpnext")
+		self.assertEqual(owner, "frappe")
+		self.assertEqual(repo, "erpnext")
+
+		owner, repo = parse_github_url("git@github.com:frappe/erpnext.git")
+		self.assertEqual(owner, "frappe")
+		self.assertEqual(repo, "erpnext")
+
+		owner, repo = parse_github_url("https://gitlab.com/gitlab-org/gitlab")
+		self.assertIsNone(owner)
+		self.assertIsNone(repo)
+
+		self.assertRaises(ValueError, parse_github_url, remote_url=None)
+
+
 class TestCrypto(FrappeTestCase):
 	def test_hashing(self):
 		self.assertEqual(sha256_hash(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")

frappe/translate.py

@@ -929,49 +929,6 @@ def get_translations(source_text):
 	)
 
 
-@frappe.whitelist()
-def get_messages(language, start=0, page_length=100, search_text=""):
-	from frappe.frappeclient import FrappeClient
-
-	translator = FrappeClient(get_translator_url())
-	return translator.post_api("translator.api.get_strings_for_translation", params=locals())
-
-
-@frappe.whitelist()
-def get_source_additional_info(source, language=""):
-	from frappe.frappeclient import FrappeClient
-
-	translator = FrappeClient(get_translator_url())
-	return translator.post_api("translator.api.get_source_additional_info", params=locals())
-
-
-@frappe.whitelist()
-def get_contributions(language):
-	return frappe.get_all(
-		"Translation",
-		fields=["*"],
-		filters={
-			"contributed": 1,
-		},
-	)
-
-
-@frappe.whitelist()
-def get_contribution_status(message_id):
-	from frappe.frappeclient import FrappeClient
-
-	doc = frappe.get_doc("Translation", message_id)
-	translator = FrappeClient(get_translator_url())
-	return translator.get_api(
-		"translator.api.get_contribution_status",
-		params={"translation_id": doc.contribution_docname},
-	)
-
-
-def get_translator_url():
-	return frappe.get_hooks()["translator_url"][0]
-
-
 @frappe.whitelist(allow_guest=True)
 def get_all_languages(with_language_name: bool = False) -> list:
 	"""Return all enabled language codes ar, ch etc."""