Merge pull request #11083 from netchampfaris/dib-fixes-2020-07-22

frappe/core/doctype/data_import/data_import.js

@@ -317,6 +317,7 @@ frappe.ui.form.on('Data Import', {
 	},
 
 	show_import_warnings(frm, preview_data) {
+		let columns = preview_data.columns;
 		let warnings = JSON.parse(frm.doc.template_warnings || '[]');
 		warnings = warnings.concat(preview_data.warnings || []);
 
@@ -367,11 +368,13 @@ frappe.ui.form.on('Data Import', {
 			.map(warning => {
 				let header = '';
 				if (warning.col) {
-					header = __('Column {0}', [warning.col]);
+					let column_number = `<span class="text-uppercase">${__('Column {0}', [warning.col])}</span>`;
+					let column_header = columns[warning.col].header_title;
+					header = `${column_number} (${column_header})`;
 				}
 				return `
 					<div class="warning" data-col="${warning.col}">
-						<h5 class="text-uppercase">${header}</h5>
+						<h5>${header}</h5>
 						<div class="body">${warning.message}</div>
 					</div>
 				`;

frappe/core/doctype/data_import/data_import_list.js

@@ -17,6 +17,7 @@ frappe.listview_settings['Data Import'] = {
 	get_indicator: function(doc) {
 		var colors = {
 			'Pending': 'orange',
+			'Not Started': 'orange',
 			'Partial Success': 'orange',
 			'Success': 'green',
 			'In Progress': 'orange',
@@ -26,6 +27,9 @@ frappe.listview_settings['Data Import'] = {
 		if (imports_in_progress.includes(doc.name)) {
 			status = 'In Progress';
 		}
+		if (status == 'Pending') {
+			status = 'Not Started';
+		}
 		return [__(status), colors[status], 'status,=,' + doc.status];
 	},
 	formatters: {

frappe/core/doctype/data_import/importer.py

@@ -7,7 +7,7 @@
 import frappe
 import timeit
 import json
-from datetime import datetime
+from datetime import datetime, date
 from frappe import _
 from frappe.utils import cint, flt, update_progress_bar, cstr
 from frappe.utils.csvutils import read_csv_content, get_csv_content_from_google_sheets
@@ -233,7 +233,7 @@ def update_record(self, doc):
 			return updated_doc
 		else:
 			# throw if no changes
-			frappe.throw('No changes to update')
+			frappe.throw("No changes to update")
 
 	def get_eta(self, current, total, processing_time):
 		self.last_eta = getattr(self, "last_eta", 0)
@@ -322,7 +322,7 @@ def __init__(self, doctype, file, template_options=None, import_type=None):
 		if isinstance(file, frappe.string_types):
 			if frappe.db.exists("File", {"file_url": file}):
 				self.file_doc = frappe.get_doc("File", {"file_url": file})
-			elif 'docs.google.com/spreadsheets' in file:
+			elif "docs.google.com/spreadsheets" in file:
 				self.google_sheets_url = file
 			elif os.path.exists(file):
 				self.file_path = file
@@ -348,7 +348,7 @@ def get_data_from_template_file(self):
 
 		elif self.google_sheets_url:
 			content = get_csv_content_from_google_sheets(self.google_sheets_url)
-			extension = 'csv'
+			extension = "csv"
 
 		if not content:
 			frappe.throw(_("Invalid or corrupted content for import"))
@@ -602,29 +602,33 @@ def _parse_doc(self, doctype, columns, values, parent_doc=None, table_df=None):
 
 		is_table = frappe.get_meta(doctype).istable
 		is_update = self.import_type == UPDATE
-		if is_table and is_update and doc.get("name") in INVALID_VALUES:
-			# for table rows being inserted in update
-			# create a new doc with defaults set
-			new_doc = frappe.new_doc(doctype, as_dict=True)
-			new_doc.update(doc)
-			doc = new_doc
+		if is_table and is_update:
+			# check if the row already exists
+			# if yes, fetch the original doc so that it is not updated
+			# if no, create a new doc
+			id_field = get_id_field(doctype)
+			id_value = doc.get(id_field.fieldname)
+			if id_value and frappe.db.exists(doctype, id_value):
+				doc = frappe.get_doc(doctype, id_value)
+			else:
+				# for table rows being inserted in update
+				# create a new doc with defaults set
+				new_doc = frappe.new_doc(doctype, as_dict=True)
+				new_doc.update(doc)
+				doc = new_doc
 
 		self.check_mandatory_fields(doctype, doc, table_df)
 		return doc
 
 	def validate_value(self, value, col):
 		df = col.df
 		if df.fieldtype == "Select":
-			select_options = [d for d in (df.options or '').split('\n') if d]
+			select_options = get_select_options(df)
 			if select_options and value not in select_options:
 				options_string = ", ".join([frappe.bold(d) for d in select_options])
 				msg = _("Value must be one of {0}").format(options_string)
 				self.warnings.append(
-					{
-						"row": self.row_number,
-						"field": df_as_json(df),
-						"message": msg,
-					}
+					{"row": self.row_number, "field": df_as_json(df), "message": msg,}
 				)
 				return
 
@@ -635,11 +639,7 @@ def validate_value(self, value, col):
 					frappe.bold(value), frappe.bold(df.options)
 				)
 				self.warnings.append(
-					{
-						"row": self.row_number,
-						"field": df_as_json(df),
-						"message": msg,
-					}
+					{"row": self.row_number, "field": df_as_json(df), "message": msg,}
 				)
 				return
 		elif df.fieldtype in ["Date", "Datetime"]:
@@ -668,7 +668,7 @@ def link_exists(self, value, df):
 
 	def parse_value(self, value, col):
 		df = col.df
-		if isinstance(value, datetime) and df.fieldtype in ["Date", "Datetime"]:
+		if isinstance(value, (datetime, date)) and df.fieldtype in ["Date", "Datetime"]:
 			return value
 
 		value = cstr(value)
@@ -689,7 +689,7 @@ def parse_value(self, value, col):
 		return value
 
 	def get_date(self, value, column):
-		if isinstance(value, datetime):
+		if isinstance(value, (datetime, date)):
 			return value
 
 		date_format = column.date_format
@@ -786,9 +786,7 @@ def __init__(self, index, row, doctype, raw_data, column_to_field_map=None):
 		for j, header in enumerate(row):
 			column_values = [get_item_at_index(r, j) for r in raw_data]
 			map_to_field = column_to_field_map.get(str(j))
-			column = Column(
-				j, header, self.doctype, column_values, map_to_field, self.seen
-			)
+			column = Column(j, header, self.doctype, column_values, map_to_field, self.seen)
 			self.seen.append(header)
 			self.columns.append(column)
 
@@ -918,13 +916,20 @@ def parse(self):
 		self.skip_import = skip_import
 
 	def guess_date_format_for_column(self):
-		""" Guesses date format for a column by parsing all the values in the column,
+		"""Guesses date format for a column by parsing all the values in the column,
 		getting the date format and then returning the one which has the maximum frequency
 		"""
 
-		date_formats = [
-			frappe.utils.guess_date_format(d) for d in self.column_values if isinstance(d, str)
-		]
+		def guess_date_format(d):
+			if isinstance(d, (datetime, date)):
+				if self.df.fieldtype == "Date":
+					return "%Y-%m-%d"
+				if self.df.fieldtype == "Datetime":
+					return "%Y-%m-%d %H:%M:%S"
+			if isinstance(d, str):
+				return frappe.utils.guess_date_format(d)
+
+		date_formats = [guess_date_format(d) for d in self.column_values]
 		date_formats = [d for d in date_formats if d]
 		if not date_formats:
 			return
@@ -958,28 +963,58 @@ def validate_values(self):
 		if self.skip_import:
 			return
 
-		if self.df.fieldtype == 'Link':
+		if self.df.fieldtype == "Link":
 			# find all values that dont exist
 			values = list(set([cstr(v) for v in self.column_values[1:] if v]))
-			exists = [d.name for d in frappe.db.get_all(self.df.options, filters={'name': ('in', values)})]
+			exists = [
+				d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)})
+			]
 			not_exists = list(set(values) - set(exists))
 			if not_exists:
-				missing_values = ', '.join(not_exists)
-				self.warnings.append({
-					'col': self.column_number,
-					'message': "The following values do not exist for {}: {}".format(self.df.options, missing_values),
-					'type': 'warning'
-				})
+				missing_values = ", ".join(not_exists)
+				self.warnings.append(
+					{
+						"col": self.column_number,
+						"message": (
+							"The following values do not exist for {}: {}".format(
+								self.df.options, missing_values
+							)
+						),
+						"type": "warning",
+					}
+				)
 		elif self.df.fieldtype in ("Date", "Time", "Datetime"):
 			# guess date format
 			self.date_format = self.guess_date_format_for_column()
 			if not self.date_format:
-				self.date_format = '%Y-%m-%d'
-				self.warnings.append({
-					'col': self.column_number,
-					'message': _("Date format could not determined from the values in this column. Defaulting to yyyy-mm-dd."),
-					'type': 'info'
-				})
+				self.date_format = "%Y-%m-%d"
+				self.warnings.append(
+					{
+						"col": self.column_number,
+						"message": _(
+							"Date format could not be determined from the values in"
+							" this column. Defaulting to yyyy-mm-dd."
+						),
+						"type": "info",
+					}
+				)
+		elif self.df.fieldtype == "Select":
+			options = get_select_options(self.df)
+			if options:
+				values = list(set([cstr(v) for v in self.column_values[1:] if v]))
+				invalid = list(set(values) - set(options))
+				if invalid:
+					valid_values = ", ".join([frappe.bold(o) for o in options])
+					invalid_values = ", ".join([frappe.bold(i) for i in invalid])
+					self.warnings.append(
+						{
+							"col": self.column_number,
+							"message": (
+								"The following values are invalid: {0}. Values must be"
+								" one of {1}".format(invalid_values, valid_values)
+							),
+						}
+					)
 
 	def as_dict(self):
 		d = frappe._dict()
@@ -990,7 +1025,7 @@ def as_dict(self):
 		d.map_to_field = self.map_to_field
 		d.date_format = self.date_format
 		d.df = self.df
-		if hasattr(self.df, 'is_child_table_field'):
+		if hasattr(self.df, "is_child_table_field"):
 			d.is_child_table_field = self.df.is_child_table_field
 			d.child_table_df = self.df.child_table_df
 		d.skip_import = self.skip_import
@@ -1070,7 +1105,7 @@ def get_standard_fields(doctype):
 		# other fields
 		fields = get_standard_fields(doctype) + frappe.get_meta(doctype).fields
 		for df in fields:
-			label = (df.label or '').strip()
+			label = (df.label or "").strip()
 			fieldtype = df.fieldtype or "Data"
 			parent = df.parent or parent_doctype
 			if fieldtype not in no_value_fields:
@@ -1164,12 +1199,17 @@ def get_user_format(date_format):
 		.replace("%d", "dd")
 	)
 
+
 def df_as_json(df):
 	return {
-		'fieldname': df.fieldname,
-		'fieldtype': df.fieldtype,
-		'label': df.label,
-		'options': df.options,
-		'parent': df.parent,
-		'default': df.default
+		"fieldname": df.fieldname,
+		"fieldtype": df.fieldtype,
+		"label": df.label,
+		"options": df.options,
+		"parent": df.parent,
+		"default": df.default,
 	}
+
+
+def get_select_options(df):
+	return [d for d in (df.options or "").split("\n") if d]

frappe/public/js/frappe/data_import/data_exporter.js

@@ -274,23 +274,29 @@ frappe.data_import.DataExporter = class DataExporter {
 			? this.column_map[child_fieldname]
 			: this.column_map[doctype];
 
-		let is_field_mandatory = df => (df.fieldname === 'name' && !child_fieldname)
-			|| (df.reqd && this.exporting_for == 'Insert New Records');
+		let is_field_mandatory = df => {
+			if (df.reqd && this.exporting_for == 'Insert New Records') {
+				return true;
+			}
+			if (autoname_field && df.fieldname == autoname_field.fieldname) {
+				return true;
+			}
+			if (df.fieldname === 'name') {
+				return true;
+			}
+			return false;
+		};
 
 		return fields
 			.filter(df => {
-				if (autoname_field && df.fieldname === autoname_field.fieldname) {
+				if (autoname_field && df.fieldname === 'name') {
 					return false;
 				}
 				return true;
 			})
 			.map(df => {
-				let label = __(df.label);
-				if (autoname_field && df.fieldname === 'name') {
-					label = label + ` (${__(autoname_field.label)})`;
-				}
 				return {
-					label,
+					label: __(df.label),
 					value: df.fieldname,
 					danger: is_field_mandatory(df),
 					checked: false,

frappe/public/js/frappe/data_import/import_preview.js

@@ -98,6 +98,9 @@ frappe.data_import.ImportPreview = class ImportPreview {
 					.replace('%y', 'yy')
 					.replace('%m', 'mm')
 					.replace('%d', 'dd')
+					.replace('%H', 'HH')
+					.replace('%M', 'mm')
+					.replace('%S', 'ss')
 				: null;
 
 			let column_title = `<span class="indicator green">
@@ -354,4 +357,4 @@ function get_fields_as_options(doctype, column_map) {
 			});
 		})
 	);
-}
+}