diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index 6561f12f56b..d6d990d2348 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -427,23 +427,20 @@ def add_data_row(self, rows, dt, parentfield, doc, rowidx): row[_column_start_end.start + i + 1] = value def build_response_as_excel(self): + from frappe.desk.utils import provide_binary_file + from frappe.utils.xlsxutils import make_xlsx + filename = frappe.generate_hash(length=10) with open(filename, "wb") as f: f.write(cstr(self.writer.getvalue()).encode("utf-8")) f = open(filename) reader = csv.reader(f) - - from frappe.utils.xlsxutils import make_xlsx - xlsx_file = make_xlsx(reader, "Data Import Template" if self.template else "Data Export") f.close() os.remove(filename) - # write out response as a xlsx type - frappe.response["filename"] = _(self.doctype) + ".xlsx" - frappe.response["filecontent"] = xlsx_file.getvalue() - frappe.response["type"] = "binary" + provide_binary_file(self.doctype, "xlsx", xlsx_file.getvalue()) def _append_name_column(self, dt=None): self.append_field_column( diff --git a/frappe/core/doctype/data_import/exporter.py b/frappe/core/doctype/data_import/exporter.py index b3a79e46761..a7dcd1acd71 100644 --- a/frappe/core/doctype/data_import/exporter.py +++ b/frappe/core/doctype/data_import/exporter.py @@ -1,8 +1,6 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import typing - import frappe from frappe import _ from frappe.model import display_fieldtypes, no_value_fields @@ -243,15 +241,9 @@ def get_csv_array_for_export(self): def build_response(self): if self.file_type == "CSV": - self.build_csv_response() + build_csv_response(self.get_csv_array_for_export(), _(self.doctype)) elif self.file_type == "Excel": - self.build_xlsx_response() - - def build_csv_response(self): - build_csv_response(self.get_csv_array_for_export(), _(self.doctype)) - - def build_xlsx_response(self): - build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype)) + build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype)) def group_children_data_by_parent(self, children_data: dict[str, list]): return groupby_metric(children_data, key="parent") diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 48c1c9f4e25..25c2ee7093b 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -45,6 +45,7 @@ "disable_user_pass_login", "login_with_email_link", "login_with_email_link_expiry", + "rate_limit_email_link_login", "allow_error_traceback", "strip_exif_metadata_from_uploaded_images", "allow_older_web_view_links", @@ -437,11 +438,11 @@ "label": "Include Web View Link in Email" }, { - "collapsible": 1, - "fieldname": "prepared_report_section", - "fieldtype": "Section Break", - "label": "Reports" - }, + "collapsible": 1, + "fieldname": "prepared_report_section", + "fieldtype": "Section Break", + "label": "Reports" + }, { "default": "Frappe", "description": "The application name will be used in the Login page.", @@ -600,12 +601,19 @@ "fieldname": "store_attached_pdf_document", "fieldtype": "Check", "label": "Store Attached PDF Document" + }, + { + "depends_on": "login_with_email_link", + "description": "You can set a high value here if multiple users will be logging in from the same network.", + "fieldname": "rate_limit_email_link_login", + "fieldtype": "Int", + "label": "Rate limit for email link login" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2024-03-14 15:18:01.465057", + "modified": "2024-03-22 16:35:52.338727", "modified_by": "Administrator", "module": "Core", "name": "System Settings", @@ -624,4 +632,4 @@ "sort_order": "ASC", "states": [], "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 3fa27c7b3ee..afe70ed2d6a 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -10,6 +10,7 @@ import frappe.desk.reportview from frappe import _ from frappe.core.utils import ljust_list +from frappe.desk.reportview import clean_params, parse_json from frappe.model.utils import render_include from frappe.modules import get_module_path, scrub from frappe.monitor import add_data_to_monitor @@ -318,47 +319,49 @@ def get_report_data(doc, data): @frappe.whitelist() def export_query(): """export from query reports""" - data = frappe._dict(frappe.local.form_dict) - data.pop("cmd", None) - data.pop("csrf_token", None) + from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file - if isinstance(data.get("filters"), str): - filters = json.loads(data["filters"]) + form_params = frappe._dict(frappe.local.form_dict) + csv_params = pop_csv_params(form_params) + clean_params(form_params) + parse_json(form_params) - if data.get("report_name"): - report_name = data["report_name"] - frappe.permissions.can_export( - frappe.get_cached_value("Report", report_name, "ref_doctype"), - raise_exception=True, - ) + report_name = form_params.report_name + frappe.permissions.can_export( + frappe.get_cached_value("Report", report_name, "ref_doctype"), + raise_exception=True, + ) - file_format_type = data.get("file_format_type") - custom_columns = frappe.parse_json(data.get("custom_columns", "[]")) - include_indentation = data.get("include_indentation") - visible_idx = data.get("visible_idx") + file_format_type = form_params.file_format_type + custom_columns = frappe.parse_json(form_params.custom_columns or "[]") + include_indentation = form_params.include_indentation + visible_idx = form_params.visible_idx if isinstance(visible_idx, str): visible_idx = json.loads(visible_idx) - if file_format_type == "Excel": - data = run(report_name, filters, custom_columns=custom_columns, are_default_filters=False) - data = frappe._dict(data) - if not data.columns: - frappe.respond_as_web_page( - _("No data to export"), - _("You can try changing the filters of your report."), - ) - return + data = run(report_name, form_params.filters, custom_columns=custom_columns) + data = frappe._dict(data) + if not data.columns: + frappe.respond_as_web_page( + _("No data to export"), + _("You can try changing the filters of your report."), + ) + return + + format_duration_fields(data) + xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation) + if file_format_type == "CSV": + content = get_csv_bytes(xlsx_data, csv_params) + file_extension = "csv" + elif file_format_type == "Excel": from frappe.utils.xlsxutils import make_xlsx - format_duration_fields(data) - xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation) - xlsx_file = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths) + file_extension = "xlsx" + content = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths).getvalue() - frappe.response["filename"] = _(report_name) + ".xlsx" - frappe.response["filecontent"] = xlsx_file.getvalue() - frappe.response["type"] = "binary" + provide_binary_file(report_name, file_extension, content) def format_duration_fields(data: frappe._dict) -> None: diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index c0a2eb12aa7..806ee451b14 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -4,7 +4,6 @@ """build query for doclistview and return results""" import json -from io import StringIO import frappe import frappe.permissions @@ -347,30 +346,21 @@ def delete_report(name): @frappe.read_only() def export_query(): """export from report builder""" - title = frappe.form_dict.title - frappe.form_dict.pop("title", None) + from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file form_params = get_form_params() form_params["limit_page_length"] = None form_params["as_list"] = True - doctype = form_params.doctype - add_totals_row = None - file_format_type = form_params["file_format_type"] - title = title or doctype - - del form_params["doctype"] - del form_params["file_format_type"] - - if "add_totals_row" in form_params and form_params["add_totals_row"] == "1": - add_totals_row = 1 - del form_params["add_totals_row"] + doctype = form_params.pop("doctype") + file_format_type = form_params.pop("file_format_type") + title = form_params.pop("title", doctype) + csv_params = pop_csv_params(form_params) + add_totals_row = 1 if form_params.pop("add_totals_row", None) == "1" else None frappe.permissions.can_export(doctype, raise_exception=True) - if "selected_items" in form_params: - si = json.loads(frappe.form_dict.get("selected_items")) - form_params["filters"] = {"name": ("in", si)} - del form_params["selected_items"] + if selection := form_params.pop("selected_items", None): + form_params["filters"] = {"name": ("in", json.loads(selection))} make_access_log( doctype=doctype, @@ -386,36 +376,24 @@ def export_query(): ret = append_totals_row(ret) data = [[_("Sr"), *get_labels(db_query.fields, doctype)]] - for i, row in enumerate(ret): - data.append([i + 1, *list(row)]) - + data.extend([i + 1, *list(row)] for i, row in enumerate(ret)) data = handle_duration_fieldtype_values(doctype, data, db_query.fields) if file_format_type == "CSV": - # convert to csv - import csv - from frappe.utils.xlsxutils import handle_html - f = StringIO() - writer = csv.writer(f) - for r in data: - # encode only unicode type strings and not int, floats etc. - writer.writerow([handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r]) - - f.seek(0) - frappe.response["result"] = cstr(f.read()) - frappe.response["type"] = "csv" - frappe.response["doctype"] = title - + file_extension = "csv" + content = get_csv_bytes( + [[handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r] for r in data], + csv_params, + ) elif file_format_type == "Excel": from frappe.utils.xlsxutils import make_xlsx - xlsx_file = make_xlsx(data, doctype) + file_extension = "xlsx" + content = make_xlsx(data, doctype).getvalue() - frappe.response["filename"] = _(title) + ".xlsx" - frappe.response["filecontent"] = xlsx_file.getvalue() - frappe.response["type"] = "binary" + provide_binary_file(title, file_extension, content) def append_totals_row(data): @@ -442,16 +420,12 @@ def get_labels(fields, doctype): """get column labels based on column names""" labels = [] for key in fields: - key = key.split(" as ")[0] - - if key.startswith(("count(", "sum(", "avg(")): + try: + parenttype, fieldname = parse_field(key) + except ValueError: continue - if "." in key: - parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") - else: - parenttype = doctype - fieldname = fieldname.strip("`") + parenttype = parenttype or doctype if parenttype == doctype and fieldname == "name": label = _("ID", context="Label of name column in report") @@ -470,17 +444,12 @@ def get_labels(fields, doctype): def handle_duration_fieldtype_values(doctype, data, fields): for field in fields: - key = field.split(" as ")[0] - - if key.startswith(("count(", "sum(", "avg(")): + try: + parenttype, fieldname = parse_field(field) + except ValueError: continue - if "." in key: - parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") - else: - parenttype = doctype - fieldname = field.strip("`") - + parenttype = parenttype or doctype df = frappe.get_meta(parenttype).get_field(fieldname) if df and df.fieldtype == "Duration": @@ -493,6 +462,19 @@ def handle_duration_fieldtype_values(doctype, data, fields): return data +def parse_field(field: str) -> tuple[str | None, str]: + """Parse a field into parenttype and fieldname.""" + key = field.split(" as ")[0] + + if key.startswith(("count(", "sum(", "avg(")): + raise ValueError + + if "." in key: + return key.split(".")[0][4:-1], key.split(".")[1].strip("`") + + return None, key.strip("`") + + @frappe.whitelist() def delete_items(): """delete selected items""" diff --git a/frappe/desk/search.py b/frappe/desk/search.py index c4bb5e1fb39..ba04610a172 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -179,7 +179,7 @@ def search_widget( formatted_fields = [f"`tab{meta.name}`.`{f.strip()}`" for f in fields] # Insert title field query after name - if meta.show_title_field_in_link: + if meta.show_title_field_in_link and meta.title_field: formatted_fields.insert(1, f"`tab{meta.name}`.{meta.title_field} as `label`") # In order_by, `idx` gets second priority, because it stores link count diff --git a/frappe/desk/utils.py b/frappe/desk/utils.py index a42fe25b910..77edf88d7a4 100644 --- a/frappe/desk/utils.py +++ b/frappe/desk/utils.py @@ -25,3 +25,34 @@ def validate_route_conflict(doctype, name): def slug(name): return name.lower().replace(" ", "-") + + +def pop_csv_params(form_dict): + """Pop csv params from form_dict and return them as a dict.""" + from csv import QUOTE_NONNUMERIC + + from frappe.utils.data import cint, cstr + + return { + "delimiter": cstr(form_dict.pop("csv_delimiter", ","))[0], + "quoting": cint(form_dict.pop("csv_quoting", QUOTE_NONNUMERIC)), + } + + +def get_csv_bytes(data: list[list], csv_params: dict) -> bytes: + """Convert data to csv bytes.""" + from csv import writer + from io import StringIO + + file = StringIO() + csv_writer = writer(file, **csv_params) + csv_writer.writerows(data) + + return file.getvalue().encode("utf-8") + + +def provide_binary_file(filename: str, extension: str, content: bytes) -> None: + """Provide a binary file to the client.""" + frappe.response["type"] = "binary" + frappe.response["filecontent"] = content + frappe.response["filename"] = f"{filename}.{extension}" diff --git a/frappe/handler.py b/frappe/handler.py index c2f1342ed3b..2b0175ca0d6 100644 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -12,6 +12,7 @@ import frappe.utils from frappe import _, is_whitelisted from frappe.core.doctype.server_script.server_script_utils import get_server_script_map +from frappe.monitor import add_data_to_monitor from frappe.utils import cint from frappe.utils.csvutils import build_csv_response from frappe.utils.image import optimize_image @@ -332,6 +333,8 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None): frappe.response["message"] = response + add_data_to_monitor(methodname=method) + # for backwards compatibility runserverobj = run_doc_method diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 173314c2d6f..43fdee26c9c 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -644,7 +644,7 @@ def apply_fieldlevel_read_permissions(self): # field: 'distinct name' # column: 'name' else: - column = field.split(" ", 2)[1].replace("`", "") + column = field.split(" ", 1)[1].replace("`", "") else: # field: 'count(`tabPhoto`.name) as total_count' # column: 'tabPhoto.name' @@ -1079,11 +1079,6 @@ def set_order_by(self, args): f"`tab{self.doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}" ) - # draft docs always on top - if hasattr(self.doctype_meta, "is_submittable") and self.doctype_meta.is_submittable: - if self.order_by: - args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}" - def validate_order_by_and_group_by(self, parameters: str): """Check order by, group by so that atleast one column is selected and does not have subquery""" if not parameters: diff --git a/frappe/public/js/frappe/form/footer/version_timeline_content_builder.js b/frappe/public/js/frappe/form/footer/version_timeline_content_builder.js index f69ddaae60c..33c87e458af 100644 --- a/frappe/public/js/frappe/form/footer/version_timeline_content_builder.js +++ b/frappe/public/js/frappe/form/footer/version_timeline_content_builder.js @@ -224,6 +224,7 @@ function format_content_for_timeline(content) { // limits content to 40 characters // escapes HTML // and makes it bold + content = frappe.utils.html2text(content); content = frappe.ellipsis(content, 40) || '""'; content = frappe.utils.escape_html(content); return content.bold(); diff --git a/frappe/public/js/frappe/list/base_list.js b/frappe/public/js/frappe/list/base_list.js index a4265ff4589..06f055e3d67 100644 --- a/frappe/public/js/frappe/list/base_list.js +++ b/frappe/public/js/frappe/list/base_list.js @@ -353,7 +353,7 @@ frappe.views.BaseList = class BaseList { } setup_paging_area() { - const paging_values = [20, 100, 500]; + const paging_values = [20, 100, 500, 2500]; this.$paging_area = $( `
@@ -439,15 +439,22 @@ frappe.views.BaseList = class BaseList { } get_args() { + let filters = this.get_filters_for_args(); + let group_by = this.get_group_by(); + let group_by_required = + Array.isArray(filters) && + filters.some((filter) => { + return filter[0] !== this.doctype; + }); return { doctype: this.doctype, fields: this.get_fields(), - filters: this.get_filters_for_args(), + filters, order_by: this.sort_selector && this.sort_selector.get_sql_string(), start: this.start, page_length: this.page_length, view: this.view, - group_by: this.get_group_by(), + group_by: group_by_required ? group_by : null, }; } diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index d5ee8c05441..ff6c0231c42 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1467,75 +1467,55 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { return; } - let export_options = ["Excel"]; - if (this.datatable) { - export_options.push("CSV"); - } - - let export_dialog_fields = [ - { - label: __("Select File Format"), - fieldname: "file_format", - fieldtype: "Select", - options: export_options, - default: "Excel", - reqd: 1, - }, - ]; - + let extra_fields = null; if (this.tree_report) { - export_dialog_fields.push({ - label: __("Include indentation"), - fieldname: "include_indentation", - fieldtype: "Check", - }); + extra_fields = [ + { + label: __("Include indentation"), + fieldname: "include_indentation", + fieldtype: "Check", + }, + ]; } - this.export_dialog = frappe.prompt( - export_dialog_fields, - ({ file_format, include_indentation }) => { + this.export_dialog = frappe.report_utils.get_export_dialog( + __(this.report_name), + extra_fields, + ({ file_format, include_indentation, csv_delimiter, csv_quoting }) => { this.make_access_log("Export", file_format); - if (file_format === "CSV") { - const column_row = this.columns.reduce((acc, col) => { - if (!col.hidden) { - acc.push(__(col.label)); - } - return acc; - }, []); - const data = this.get_data_for_csv(include_indentation); - const out = [column_row].concat(data); - frappe.tools.downloadify(out, null, this.report_name); - } else { - let filters = this.get_filter_values(true); - if (frappe.urllib.get_dict("prepared_report_name")) { - filters = Object.assign( - frappe.urllib.get_dict("prepared_report_name"), - filters - ); - } + let filters = this.get_filter_values(true); + if (frappe.urllib.get_dict("prepared_report_name")) { + filters = Object.assign( + frappe.urllib.get_dict("prepared_report_name"), + filters + ); + } - const visible_idx = this.datatable?.bodyRenderer.visibleRowIndices || []; - if (visible_idx.length + 1 === this.data?.length) { - visible_idx.push(visible_idx.length); - } + const visible_idx = this.datatable.bodyRenderer.visibleRowIndices; + if (visible_idx.length + 1 === this.data.length) { + visible_idx.push(visible_idx.length); + } - const args = { - cmd: "frappe.desk.query_report.export_query", - report_name: this.report_name, - custom_columns: this.custom_columns?.length ? this.custom_columns : [], - file_format_type: file_format, - filters: filters, - visible_idx, - include_indentation, - }; + const args = { + cmd: "frappe.desk.query_report.export_query", + report_name: this.report_name, + custom_columns: this.custom_columns.length ? this.custom_columns : [], + file_format_type: file_format, + filters: filters, + visible_idx, + csv_delimiter, + csv_quoting, + include_indentation, + }; - open_url_post(frappe.request.url, args); - } - }, - __("Export Report: {0}", [this.report_name]), - __("Download") + open_url_post(frappe.request.url, args); + + this.export_dialog.hide(); + } ); + + this.export_dialog.show(); } get_data_for_csv(include_indentation) { diff --git a/frappe/public/js/frappe/views/reports/report_utils.js b/frappe/public/js/frappe/views/reports/report_utils.js index d40cf07ce6e..16b02c28a10 100644 --- a/frappe/public/js/frappe/views/reports/report_utils.js +++ b/frappe/public/js/frappe/views/reports/report_utils.js @@ -165,4 +165,145 @@ frappe.report_utils = { }; return get_result[fn](values); }, + + get_export_dialog(report_name, extra_fields, callback) { + const fields = [ + { + label: "File Format", + fieldname: "file_format", + fieldtype: "Select", + options: ["Excel", "CSV"], + default: "Excel", + reqd: 1, + }, + { + fieldtype: "Section Break", + fieldname: "csv_settings", + label: "Settings", + collapsible: 1, + depends_on: "eval:doc.file_format=='CSV'", + }, + { + fieldtype: "Data", + label: "CSV Delimiter", + fieldname: "csv_delimiter", + default: ",", + length: 1, + depends_on: "eval:doc.file_format=='CSV'", + }, + { + fieldtype: "Select", + label: "CSV Quoting", + fieldname: "csv_quoting", + options: [ + { value: 0, label: "Minimal" }, + { value: 1, label: "All" }, + { value: 2, label: "Non-numeric" }, + { value: 3, label: "None" }, + ], + default: 2, + depends_on: "eval:doc.file_format=='CSV'", + }, + { + fieldtype: "Small Text", + label: "CSV Preview", + fieldname: "csv_preview", + read_only: 1, + depends_on: "eval:doc.file_format=='CSV'", + }, + ]; + + if (extra_fields) { + fields.push( + { + fieldtype: "Section Break", + fieldname: "extra_fields", + collapsible: 0, + }, + ...extra_fields + ); + } + + const dialog = new frappe.ui.Dialog({ + title: __("Export Report: {0}", [report_name], "Export report"), + fields: fields, + primary_action_label: __("Download", null, "Export report"), + primary_action: callback, + }); + + function update_csv_preview(dialog) { + const is_query_report = frappe.get_route()[0] === "query-report"; + const report = is_query_report ? frappe.query_report : cur_list; + const columns = report.columns.filter((col) => col.hidden !== 1); + PREVIEW_DATA = [ + columns.map((col) => __(is_query_report ? col.label : col.name)), + ...report.data + .slice(0, 3) + .map((row) => + columns.map((col) => row[is_query_report ? col.fieldname : col.field]) + ), + ]; + + dialog.set_value( + "csv_preview", + frappe.report_utils.get_csv_preview( + PREVIEW_DATA, + dialog.get_value("csv_quoting"), + dialog.get_value("csv_delimiter") + ) + ); + } + + dialog.fields_dict["file_format"].df.onchange = () => update_csv_preview(dialog); + dialog.fields_dict["csv_quoting"].df.onchange = () => update_csv_preview(dialog); + dialog.fields_dict["csv_delimiter"].df.onchange = () => update_csv_preview(dialog); + + return dialog; + }, + + get_csv_preview(data, quoting, delimiter) { + // data: array of arrays + // quoting: 0 - minimal, 1 - all, 2 - non-numeric, 3 - none + // delimiter: any single character + quoting = cint(quoting); + const QUOTING = { + Minimal: 0, + All: 1, + NonNumeric: 2, + None: 3, + }; + + if (delimiter.length > 1) { + frappe.throw(__("Delimiter must be a single character")); + } + + if (0 > quoting || quoting > 3) { + frappe.throw(__("Quoting must be between 0 and 3")); + } + + return data + .map((row) => { + return row + .map((col) => { + if (typeof col == "string" && col.includes('"')) { + col = col.replace(/"/g, '""'); + } + + switch (quoting) { + case QUOTING.Minimal: + return typeof col === "string" && col.includes(delimiter) + ? `"${col}"` + : `${col}`; + case QUOTING.All: + return `"${col}"`; + case QUOTING.NonNumeric: + return isNaN(col) ? `"${col}"` : `${col}`; + case QUOTING.None: + return `${col}`; + } + }) + .join(delimiter); + }) + .join("\n"); + }, }; diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index f19237a5386..09b08cc6ec1 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -1472,7 +1472,8 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { if (this.add_totals_row) { const total_data = this.get_columns_totals(this.data); - total_data["name"] = __("Totals").bold(); + total_data["name"] = __("Total"); + total_data.is_total_row = true; rows_in_order.push(total_data); } @@ -1551,33 +1552,31 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { action: () => { const args = this.get_args(); const selected_items = this.get_checked_items(true); - let fields = [ - { - fieldtype: "Select", - label: __("Select File Type"), - fieldname: "file_format_type", - options: ["Excel", "CSV"], - default: "Excel", - }, - ]; - if (this.total_count > this.count_without_children || args.page_length) { - fields.push({ - fieldtype: "Check", - fieldname: "export_all_rows", - label: __("Export All {0} rows?", [(this.total_count + "").bold()]), - }); + let extra_fields = null; + if (this.total_count > (this.count_without_children || args.page_length)) { + extra_fields = [ + { + fieldtype: "Check", + fieldname: "export_all_rows", + label: __("Export All {0} rows?", [`${this.total_count}`]), + }, + ]; } - const d = new frappe.ui.Dialog({ - title: __("Export Report: {0}", [__(this.doctype)]), - fields: fields, - primary_action_label: __("Download"), - primary_action: (data) => { + const d = frappe.report_utils.get_export_dialog( + __(this.doctype), + extra_fields, + (data) => { args.cmd = "frappe.desk.reportview.export_query"; - args.file_format_type = data.file_format_type; + args.file_format_type = data.file_format; args.title = this.report_name || this.doctype; + if (data.file_format == "CSV") { + args.csv_delimiter = data.csv_delimiter; + args.csv_quoting = data.csv_quoting; + } + if (this.add_totals_row) { args.add_totals_row = 1; } @@ -1597,8 +1596,8 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { open_url_post(frappe.request.url, args); d.hide(); - }, - }); + } + ); d.show(); }, diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index af2093a11ae..2335fc28f88 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -1220,6 +1220,20 @@ def test_get_count(self): self.assertIsInstance(count, int) self.assertLessEqual(count, limit) + # doctype with space in name + limit = 2 + frappe.local.form_dict = frappe._dict( + { + "doctype": "Role Profile", + "fields": [], + "distinct": "true", + "limit": limit, + } + ) + count = execute_cmd("frappe.desk.reportview.get_count") + self.assertIsInstance(count, int) + self.assertLessEqual(count, limit) + def test_reportview_get(self): user = frappe.get_doc("User", "test@example.com") add_child_table_to_blog_post() diff --git a/frappe/tests/test_query_report.py b/frappe/tests/test_query_report.py index 6f77a3380c9..1c02b9da8b3 100644 --- a/frappe/tests/test_query_report.py +++ b/frappe/tests/test_query_report.py @@ -188,3 +188,44 @@ def test_report_for_duplicate_column_names(self): except Exception as e: raise e frappe.db.rollback() + + def test_csv(self): + from csv import QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC, DictReader + from io import StringIO + + REPORT_NAME = "Test CSV Report" + REF_DOCTYPE = "DocType" + REPORT_COLUMNS = ["name", "module", "issingle"] + + if not frappe.db.exists("Report", REPORT_NAME): + report = frappe.new_doc("Report") + report.report_name = REPORT_NAME + report.ref_doctype = "User" + report.report_type = "Query Report" + report.query = frappe.qb.from_(REF_DOCTYPE).select(*REPORT_COLUMNS).limit(10).get_sql() + report.is_standard = "No" + report.save() + + for delimiter in (",", ";", "\t", "|"): + for quoting in (QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC): + frappe.local.form_dict = frappe._dict( + { + "report_name": REPORT_NAME, + "file_format_type": "CSV", + "csv_quoting": quoting, + "csv_delimiter": delimiter, + "include_indentation": 0, + "visible_idx": [0, 1, 2], + } + ) + export_query() + + self.assertTrue(frappe.response["filename"].endswith(".csv")) + self.assertEqual(frappe.response["type"], "binary") + with StringIO(frappe.response["filecontent"].decode("utf-8")) as result: + reader = DictReader(result, delimiter=delimiter, quoting=quoting) + row = reader.__next__() + for column in REPORT_COLUMNS: + self.assertIn(column, row) + + frappe.delete_doc("Report", REPORT_NAME, delete_permanently=True) diff --git a/frappe/tests/test_reportview.py b/frappe/tests/test_reportview.py new file mode 100644 index 00000000000..4a24514acf8 --- /dev/null +++ b/frappe/tests/test_reportview.py @@ -0,0 +1,34 @@ +# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors +# License: MIT. See LICENSE + +import frappe +from frappe.desk.reportview import export_query +from frappe.tests.utils import FrappeTestCase + + +class TestReportview(FrappeTestCase): + def test_csv(self): + from csv import QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC, DictReader + from io import StringIO + + frappe.local.form_dict = frappe._dict( + doctype="DocType", + file_format_type="CSV", + fields=("name", "module", "issingle"), + filters={"issingle": 1, "module": "Core"}, + ) + + for delimiter in (",", ";", "\t", "|"): + frappe.local.form_dict.csv_delimiter = delimiter + for quoting in (QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC): + frappe.local.form_dict.csv_quoting = quoting + + export_query() + + self.assertTrue(frappe.response["filename"].endswith(".csv")) + self.assertEqual(frappe.response["type"], "binary") + with StringIO(frappe.response["filecontent"].decode("utf-8")) as result: + reader = DictReader(result, delimiter=delimiter, quoting=quoting) + for row in reader: + self.assertEqual(int(row["Is Single"]), 1) + self.assertEqual(row["Module"], "Core") diff --git a/frappe/translations/de.csv b/frappe/translations/de.csv index 54da9a309d2..06fa4b45df5 100644 --- a/frappe/translations/de.csv +++ b/frappe/translations/de.csv @@ -158,7 +158,7 @@ No Data,Keine Daten, No address added yet.,Noch keine Adresse hinzugefügt., No contacts added yet.,Noch keine Kontakte hinzugefügt., No items found.,Keine Elemente gefunden., -None,Keiner, +None,Keine, Not Permitted,Nicht zulässig, Not active,Nicht aktiv, Notes,Hinweise, @@ -2220,7 +2220,6 @@ Select Columns,Spalten auswählen, Select Document Type,Dokumenttyp auswählen, Select Document Type or Role to start.,"Dokumententyp oder Rolle auswählen, um zu beginnen.", Select Document Types to set which User Permissions are used to limit access.,"Dokumentenarten auswählen, um die Benutzerrechte, die den Zugriff einschränken, anzuwenden", -Select File Format,Wählen Sie Dateiformat, Select File Type,Dateityp auswählen, Select Language...,Sprache auswählen..., Select Languages,Sprachenauswahl, @@ -4870,3 +4869,9 @@ Filters:,Filter:, {0} is not like {1},{0} ist nicht wie {1}, {0} is set,{0} ist eingetragen, {0} is not set,{0} ist nicht eingetragen, +File Format,Dateiformat, +CSV Delimiter,Trennzeichen, +CSV Quoting,Anführungszeichen, +CSV Preview,Vorschau, +Non-numeric,Nicht-numerische, +Minimal,Minimal, diff --git a/frappe/utils/redis_queue.py b/frappe/utils/redis_queue.py index 31b8ebc39db..791ff12a9b0 100644 --- a/frappe/utils/redis_queue.py +++ b/frappe/utils/redis_queue.py @@ -17,6 +17,21 @@ def add_user(self, username, password=None): @classmethod def get_connection(cls, username=None, password=None): + if frappe.conf.redis_queue_sentinel_enabled: + from frappe.utils.redis_wrapper import get_sentinel_connection + + sentinels = [tuple(node.split(":")) for node in frappe.conf.get("redis_queue_sentinels", [])] + sentinel = get_sentinel_connection( + sentinels=sentinels, + sentinel_username=frappe.conf.get("redis_queue_sentinel_username"), + sentinel_password=frappe.conf.get("redis_queue_sentinel_password"), + master_username=frappe.conf.get("redis_queue_master_username", username), + master_password=frappe.conf.get("redis_queue_master_password", password), + ) + conn = sentinel.master_for(frappe.conf.get("redis_queue_master_service")) + conn.ping() + return conn + rq_url = frappe.local.conf.redis_queue domain = rq_url.split("redis://", 1)[-1] url = (username and f"redis://{username}:{password or ''}@{domain}") or rq_url diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 506183cf942..94764e7e284 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -109,8 +109,6 @@ def read_xls_file_from_attached_file(content): def build_xlsx_response(data, filename): - xlsx_file = make_xlsx(data, filename) - # write out response as a xlsx type - frappe.response["filename"] = _(filename) + ".xlsx" - frappe.response["filecontent"] = xlsx_file.getvalue() - frappe.response["type"] = "binary" + from frappe.desk.utils import provide_binary_file + + provide_binary_file(filename, "xlsx", make_xlsx(data, filename).getvalue()) diff --git a/frappe/www/contact.html b/frappe/www/contact.html index a6672d63d2a..d34c63589d8 100644 --- a/frappe/www/contact.html +++ b/frappe/www/contact.html @@ -1,7 +1,7 @@ {% extends "templates/web.html" %} {% set title = heading or "Contact Us" %} -{% block header %}

{{ heading or "Contact Us" }}

{% endblock %} +{% block header %}

{{ heading or _("Contact Us") }}

{% endblock %} {% block page_content %}