feat: rearranging standard fields in customize form (#19822)

* feat: rearranging standing fields

* fix: fixed creation of property setter

* refactor: renamed setup_sortable

* fix: loading field_order property

* refactor: removed redundant db call

* fix: field_order not found

* test: Added tests for field order in customize form

* refactor: better naming

* refactor: simplified logic

* feat: Updating field order on custom field creation

* feat: Added support for custom fiels

* refactor: moving to meta

* refactor: changed property type to json

* fix: new standard field insert order.

* fix: don't modify insert_after of system generated custom fields.

# This is because system generated fields are to be treated as standard fields. If the user restores the form to default, this value will be used to reset the original position.

# The new position of form fields are stored in the field_order Property Setter.

* fix: treat system generated fields as standard fields when sorting.

* revert: check for is_system_generated

* Revert "fix: new standard field insert order."

This reverts commit 6cdbe42.

* fix: prioritize field_order over insert_after.

# Use insert_after as fallback in event the field doesn't exist in field_order

* fix(test): delete existing custom field

* fix: order of standard fields without field_order property.

* Revert "Revert "fix: new standard field insert order.""

This reverts commit c830f1b.

* test: field order of newly migrated standard fields.

* fix(test): clear test_standard_field from previous test run.

* fix: sort with insert_after for system generated fields.

* fix(test): reset standard field creation before re-run and after successful test.

* fix: insert_after position should be + 1

* chore: remove debug statement

* test: system generated customized fields

* chore: remove print

* chore: lint all

* fix: show quick link to Table MultiSelect DocTypes

* refactor: change backend implementation of `CustomizeForm` and `Meta`

* test: simplify tests

* fix: rename `idx` to `index` for clarity

* perf: define `existing_fields` conditionally

---------

Co-authored-by: Aradhya <aradhyatripathi51@gmail.com>
Co-authored-by: Aradhya Tripathi <67282231+Aradhya-Tripathi@users.noreply.github.com>
Co-authored-by: Sagar Vora <sagar@resilient.tech>

frappe/custom/doctype/customize_form/customize_form.js

@@ -49,14 +49,6 @@ frappe.ui.form.on("Customize Form", {
 				grid_row.row.addClass("highlight");
 			}
 		});
-
-		$(frm.wrapper).on("grid-make-sortable", function (e, frm) {
-			frm.trigger("setup_sortable");
-		});
-
-		$(frm.wrapper).on("grid-move-row", function (e, frm) {
-			frm.trigger("setup_sortable");
-		});
 	},
 
 	doc_type: function (frm) {
@@ -71,7 +63,7 @@ frappe.ui.form.on("Customize Form", {
 							frm.set_value("doc_type", "");
 						} else {
 							frm.refresh();
-							frm.trigger("setup_sortable");
+							frm.trigger("add_customize_child_table_button");
 							frm.trigger("setup_default_views");
 						}
 					}
@@ -87,23 +79,16 @@ frappe.ui.form.on("Customize Form", {
 		frm.trigger("setup_default_views");
 	},
 
-	setup_sortable: function (frm) {
+	add_customize_child_table_button: function (frm) {
 		frm.doc.fields.forEach(function (f) {
-			if (!f.is_custom_field || f.is_system_generated) {
-				f._sortable = false;
-			}
+			if (!in_list(["Table", "Table MultiSelect"], f.fieldtype)) return;
 
-			if (f.fieldtype == "Table") {
-				frm.add_custom_button(
-					f.options,
-					function () {
-						frm.set_value("doc_type", f.options);
-					},
-					__("Customize Child Table")
-				);
-			}
+			frm.add_custom_button(
+				f.options,
+				() => frm.set_value("doc_type", f.options),
+				__("Customize Child Table")
+			);
 		});
-		frm.fields_dict.fields.grid.refresh();
 	},
 
 	refresh: function (frm) {

frappe/custom/doctype/customize_form/customize_form.py

@@ -214,11 +214,39 @@ def set_property_setters(self):
 		# action and links
 		self.set_property_setters_for_actions_and_links(meta)
 
+	def set_property_setter_for_field_order(self, meta):
+		new_order = [df.fieldname for df in self.fields]
+		existing_order = getattr(meta, "field_order", None)
+		default_order = [
+			fieldname for fieldname, df in meta._fields.items() if not getattr(df, "is_custom_field", False)
+		]
+
+		if new_order == default_order:
+			if existing_order:
+				delete_property_setter(self.doc_type, "field_order")
+
+			return
+
+		if existing_order and new_order == json.loads(existing_order):
+			return
+
+		frappe.make_property_setter(
+			{
+				"doctype": self.doc_type,
+				"doctype_or_field": "DocType",
+				"property": "field_order",
+				"value": json.dumps(new_order),
+			},
+			is_system_generated=False,
+		)
+
 	def set_property_setters_for_doctype(self, meta):
 		for prop, prop_type in doctype_properties.items():
 			if self.get(prop) != meta.get(prop):
 				self.make_property_setter(prop, self.get(prop), prop_type)
 
+		self.set_property_setter_for_field_order(meta)
+
 	def set_property_setters_for_docfield(self, meta, df, meta_df):
 		for prop, prop_type in docfield_properties.items():
 			if prop != "idx" and (df.get(prop) or "") != (meta_df[0].get(prop) or ""):

frappe/custom/doctype/customize_form/test_customize_form.py

@@ -425,3 +425,15 @@ def test_system_generated_fields(self):
 		self.assertEqual(
 			frappe.db.get_value("Property Setter", property_setter_filters, "value"), "Test Description"
 		)
+
+	def test_custom_field_order(self):
+		# shuffle fields
+		customize_form = self.get_customize_form(doctype="ToDo")
+		customize_form.fields.insert(0, customize_form.fields.pop())
+		customize_form.save_customization()
+
+		field_order_property = json.loads(
+			frappe.db.get_value("Property Setter", {"doc_type": "ToDo", "property": "field_order"}, "value")
+		)
+
+		self.assertEqual(field_order_property, [df.fieldname for df in frappe.get_meta("ToDo").fields])

frappe/model/meta.py

@@ -134,13 +134,10 @@ def process(self):
 			self.init_field_caches()
 			return
 
-		has_custom_fields = self.add_custom_fields()
+		self.add_custom_fields()
 		self.apply_property_setters()
 		self.init_field_caches()
-
-		if has_custom_fields:
-			self.sort_fields()
-
+		self.sort_fields()
 		self.get_valid_columns()
 		self.set_custom_permissions()
 		self.add_custom_links_and_actions()
@@ -361,7 +358,6 @@ def add_custom_fields(self):
 			return
 
 		self.extend("fields", custom_fields)
-		return True
 
 	def apply_property_setters(self):
 		"""
@@ -372,11 +368,11 @@ def apply_property_setters(self):
 		if not frappe.db.table_exists("Property Setter"):
 			return
 
-		property_setters = frappe.db.sql(
-			"""select * from `tabProperty Setter` where
-			doc_type=%s""",
-			(self.name,),
-			as_dict=1,
+		property_setters = frappe.db.get_values(
+			"Property Setter",
+			filters={"doc_type": self.name},
+			fieldname="*",
+			as_dict=True,
 		)
 
 		if not property_setters:
@@ -452,14 +448,56 @@ def init_field_caches(self):
 			self._table_fields = self.get("fields", {"fieldtype": ["in", table_fields]})
 
 	def sort_fields(self):
-		"""Sort custom fields on the basis of insert_after"""
+		"""
+		Sort fields on the basis of following rules (priority descending):
+		- `field_order` property setter
+		- `insert_after` computed based on default order for standard fields
+		- `insert_after` property for custom fields
+		"""
+
+		if field_order := getattr(self, "field_order", []):
+			field_order = [fieldname for fieldname in json.loads(field_order) if fieldname in self._fields]
+
+			# all fields match, best case scenario
+			if len(field_order) == len(self.fields):
+				self._update_fields_based_on_order(field_order)
+				return
+
+			# if the first few standard fields are not in the field order, prepare to prepend them
+			if self.fields[0].fieldname not in field_order:
+				fields_to_prepend = []
+				standard_field_found = False
+
+				for fieldname, field in self._fields.items():
+					if getattr(field, "is_custom_field", False):
+						# all custom fields from here on
+						break
+
+					if fieldname in field_order:
+						standard_field_found = True
+						break
+
+					fields_to_prepend.append(fieldname)
+
+				if standard_field_found:
+					field_order = fields_to_prepend + field_order
+				else:
+					# worst case scenario, invalidate field_order
+					field_order = fields_to_prepend
 
-		field_order = []
+		existing_fields = set(field_order) if field_order else False
 		insert_after_map = {}
 
-		for field in self.fields:
+		for index, field in enumerate(self.fields):
+			if existing_fields and field.fieldname in existing_fields:
+				continue
+
 			if not getattr(field, "is_custom_field", False):
-				field_order.append(field.fieldname)
+				if existing_fields:
+					# compute insert_after from previous field
+					insert_after_map.setdefault(self.fields[index - 1].fieldname, []).append(field.fieldname)
+				else:
+					field_order.append(field.fieldname)
 
 			elif insert_after := getattr(field, "insert_after", None):
 				insert_after_map.setdefault(insert_after, []).append(field.fieldname)
@@ -471,6 +509,9 @@ def sort_fields(self):
 		if insert_after_map:
 			_update_field_order_based_on_insert_after(field_order, insert_after_map)
 
+		self._update_fields_based_on_order(field_order)
+
+	def _update_fields_based_on_order(self, field_order):
 		sorted_fields = []
 
 		for idx, fieldname in enumerate(field_order, 1):