From 9a84a99ac027724c429e3b62b9a9eb68e23a6f70 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Wed, 20 Feb 2019 13:04:35 +0530 Subject: [PATCH 1/8] feat(tally): Add Tally Migration DocType --- .../doctype/tally_migration/__init__.py | 0 .../tally_migration/tally_migration.js | 8 + .../tally_migration/tally_migration.json | 159 ++++++++++++++++++ .../tally_migration/tally_migration.py | 12 ++ .../tally_migration/test_tally_migration.js | 23 +++ .../tally_migration/test_tally_migration.py | 10 ++ 6 files changed, 212 insertions(+) create mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/__init__.py create mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js create mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json create mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py create mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.js create mode 100644 erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.py diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/__init__.py b/erpnext/erpnext_integrations/doctype/tally_migration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js new file mode 100644 index 000000000000..065fcc722e13 --- /dev/null +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js @@ -0,0 +1,8 @@ +// Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Tally Migration', { + refresh: function(frm) { + + } +}); diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json new file mode 100644 index 000000000000..b54e6beb208e --- /dev/null +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json @@ -0,0 +1,159 @@ +{ + "allow_copy": 0, + "allow_events_in_timeline": 0, + "allow_guest_to_view": 0, + "allow_import": 0, + "allow_rename": 0, + "autoname": "", + "beta": 0, + "creation": "2019-02-01 14:27:09.485238", + "custom": 0, + "docstatus": 0, + "doctype": "DocType", + "document_type": "", + "editable_grid": 1, + "engine": "InnoDB", + "fields": [ + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "master_data", + "fieldtype": "Attach", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 1, + "in_standard_filter": 0, + "label": "Master Data", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "column_break_2", + "fieldtype": "Column Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "day_book", + "fieldtype": "Attach", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 1, + "in_standard_filter": 0, + "label": "Day Book", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + } + ], + "has_web_view": 0, + "hide_heading": 0, + "hide_toolbar": 0, + "idx": 0, + "image_view": 0, + "in_create": 0, + "is_submittable": 0, + "issingle": 0, + "istable": 0, + "max_attachments": 0, + "modified": "2019-02-20 12:59:17.746113", + "modified_by": "Administrator", + "module": "ERPNext Integrations", + "name": "Tally Migration", + "name_case": "", + "owner": "Administrator", + "permissions": [ + { + "amend": 0, + "cancel": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "if_owner": 0, + "import": 0, + "permlevel": 0, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "set_user_permissions": 0, + "share": 1, + "submit": 0, + "write": 1 + } + ], + "quick_entry": 0, + "read_only": 0, + "read_only_onload": 0, + "show_name_in_global_search": 0, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1, + "track_seen": 0, + "track_views": 0 +} \ No newline at end of file diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py new file mode 100644 index 000000000000..f4e064ae1aef --- /dev/null +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals + +import frappe +from frappe.model.document import Document + + +class TallyMigration(Document): + pass diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.js new file mode 100644 index 000000000000..433c5e2cda8e --- /dev/null +++ b/erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.js @@ -0,0 +1,23 @@ +/* eslint-disable */ +// rename this file from _test_[name] to test_[name] to activate +// and remove above this line + +QUnit.test("test: Tally Migration", function (assert) { + let done = assert.async(); + + // number of asserts + assert.expect(1); + + frappe.run_serially([ + // insert a new Tally Migration + () => frappe.tests.make('Tally Migration', [ + // values to be set + {key: 'value'} + ]), + () => { + assert.equal(cur_frm.doc.key, 'value'); + }, + () => done() + ]); + +}); diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.py new file mode 100644 index 000000000000..9f67e55ca165 --- /dev/null +++ b/erpnext/erpnext_integrations/doctype/tally_migration/test_tally_migration.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors +# See license.txt +from __future__ import unicode_literals + +import frappe +import unittest + +class TestTallyMigration(unittest.TestCase): + pass From 55d465f3ad15f75e972bacfd602f58927b676b9d Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Fri, 1 Mar 2019 15:34:47 +0530 Subject: [PATCH 2/8] feat(tally): Preprocess master data to generate COA --- .../tally_migration/tally_migration.js | 33 ++- .../tally_migration/tally_migration.json | 230 +++++++++++++++++- .../tally_migration/tally_migration.py | 122 +++++++++- 3 files changed, 381 insertions(+), 4 deletions(-) diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js index 065fcc722e13..4652d60bff70 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js @@ -3,6 +3,35 @@ frappe.ui.form.on('Tally Migration', { refresh: function(frm) { - - } + if (frm.doc.master_data && frm.doc.day_book) { + frm.disable_save(); + if(frm.doc.status != "In Progress") { + frm.page.set_primary_action("Preprocess", () => frm.trigger("preprocess")); + } + } else { + frm.set_value("status", "Attach File"); + } + if (frm.doc.tally_company && frm.doc.erpnext_company) { + frm.set_df_property("company_section", "hidden", 0); + frm.page.set_primary_action("Start Import", () => frm.trigger("start_import")); + } + }, + preprocess: function(frm) { + frm.call({ + doc: frm.doc, + method: "preprocess", + freeze: true + }).then((r) => { + frm.set_value("status", "Preprocessing In Progress"); + }); + }, + start_import: function(frm) { + frm.call({ + doc: frm.doc, + method: "start_import", + freeze: true + }).then((r) => { + frm.set_value("status", "Import In Progress"); + }); + }, }); diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json index b54e6beb208e..7d51850aff75 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json @@ -14,6 +14,38 @@ "editable_grid": 1, "engine": "InnoDB", "fields": [ + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "status", + "fieldtype": "Data", + "hidden": 1, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Status", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, { "allow_bulk_edit": 0, "allow_in_quick_entry": 0, @@ -46,6 +78,40 @@ "translatable": 0, "unique": 0 }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "Sundry Creditors", + "fieldname": "tally_creditors_account", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Tally Creditors Account", + "length": 0, + "no_copy": 0, + "options": "", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 1, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, { "allow_bulk_edit": 0, "allow_in_quick_entry": 0, @@ -108,6 +174,168 @@ "set_only_once": 0, "translatable": 0, "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "Sundry Debtors", + "fieldname": "tally_debtors_account", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Tally Debtors Account", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 1, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "company_section", + "fieldtype": "Section Break", + "hidden": 1, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Company Section", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "tally_company", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Tally Company", + "length": 0, + "no_copy": 0, + "options": "", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 1, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "column_break_8", + "fieldtype": "Column Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "erpnext_company", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "ERPNext Company", + "length": 0, + "no_copy": 0, + "options": "", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 } ], "has_web_view": 0, @@ -120,7 +348,7 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2019-02-20 12:59:17.746113", + "modified": "2019-03-01 15:02:47.992385", "modified_by": "Administrator", "module": "ERPNext Integrations", "name": "Tally Migration", diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py index f4e064ae1aef..4c27d743fb01 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -4,9 +4,129 @@ from __future__ import unicode_literals +import json +import re +import zipfile import frappe from frappe.model.document import Document +from bs4 import BeautifulSoup as bs +PRIMARY_ACCOUNT = "Primary" class TallyMigration(Document): - pass + def _preprocess(self): + company, chart_of_accounts_tree, customers, suppliers = self._process_master_data() + self.tally_company = company + self.erpnext_company = company + self.status = "Preprocessed" + self.save() + + def _process_master_data(self): + def get_master_collection(master_data): + master_file = frappe.get_doc("File", {"file_url": master_data}) + + with zipfile.ZipFile(master_file.get_full_path()) as zf: + content = zf.read(zf.namelist()[0]).decode("utf-16") + + master = bs(sanitize(emptify(content)), "xml") + collection = master.BODY.IMPORTDATA.REQUESTDATA + return collection + + def get_company_name(collection): + return collection.find_all("REMOTECMPINFO.LIST")[0].REMOTECMPNAME.string + + def get_coa_customers_suppliers(collection): + root_type_map = { + "Application of Funds (Assets)": "Asset", + "Expenses": "Expense", + "Income": "Income", + "Source of Funds (Liabilities)": "Liability" + } + roots = set(root_type_map.keys()) + accounts = list(get_groups(collection.find_all("GROUP"))) + list(get_ledgers(collection.find_all("LEDGER"))) + children, parents = get_children_and_parent_dict(accounts) + group_set = [acc[1] for acc in accounts if acc[2]] + children, customers, suppliers = remove_parties(parents, children, group_set) + coa = traverse({}, children, roots, roots, group_set) + + for account in coa: + coa[account]["root_type"] = root_type_map[account] + + return coa, customers, suppliers + + def get_groups(accounts): + for account in accounts: + if account["NAME"] in (self.tally_creditors_account, self.tally_debtors_account): + yield get_parent(account), account["NAME"], 0 + else: + yield get_parent(account), account["NAME"], 1 + + def get_ledgers(accounts): + for account in accounts: + # If Ledger doesn't have PARENT field then don't create Account + # For example "Profit & Loss A/c" + if account.PARENT: + yield account.PARENT.string, account["NAME"], 0 + + def get_parent(account): + if account.PARENT: + return account.PARENT.string + return { + ("Yes", "No"): "Application of Funds (Assets)", + ("Yes", "Yes"): "Expenses", + ("No", "Yes"): "Income", + ("No", "No"): "Source of Funds (Liabilities)", + }[(account.ISDEEMEDPOSITIVE.string, account.ISREVENUE.string)] + + def get_children_and_parent_dict(accounts): + children, parents = {}, {} + for parent, account, is_group in accounts: + children.setdefault(parent, set()).add(account) + parents.setdefault(account, set()).add(parent) + return children, parents + + def remove_parties(parents, children, group_set): + customers, suppliers = set(), set() + for account in parents: + if self.tally_creditors_account in parents[account]: + children.pop(account, None) + if account not in group_set: + customers.add(account) + elif self.tally_debtors_account in parents[account]: + children.pop(account, None) + if account not in group_set: + suppliers.add(account) + return children, customers, suppliers + + def traverse(tree, children, accounts, roots, group_set): + for account in accounts: + if account in group_set or account in roots: + if account in children: + tree[account] = traverse({}, children, children[account], roots, group_set) + else: + tree[account] = {"is_group": 1} + else: + tree[account] = {} + return tree + + collection = get_master_collection(self.master_data) + + company = get_company_name(collection) + chart_of_accounts_tree, customer_names, supplier_names = get_coa_customers_suppliers(collection) + + return company, chart_of_accounts_tree, customer_names, supplier_names + + def preprocess(self): + frappe.enqueue_doc(self.doctype, self.name, "_preprocess") + + def start_import(self): + pass + +def sanitize(string): + return re.sub("", "", string) + +def emptify(string): + string = re.sub(r"<\w+/>", "", string) + string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string) + string = re.sub(r"\r\n", "", string) + return string From 0668482f82e447c628ff8ae00e39e2505228bf26 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Fri, 1 Mar 2019 21:35:56 +0530 Subject: [PATCH 3/8] feat(tally): Create chart of accounts from tally master data --- .../tally_migration/tally_migration.json | 34 ++++++++++++++++++- .../tally_migration/tally_migration.py | 24 ++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json index 7d51850aff75..2087c7f7bee6 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json @@ -328,6 +328,38 @@ "precision": "", "print_hide": 0, "print_hide_if_no_value": 0, + "read_only": 1, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "chart_of_accounts", + "fieldtype": "Attach", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Chart of Accounts", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, "read_only": 0, "remember_last_selected_value": 0, "report_hide": 0, @@ -348,7 +380,7 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2019-03-01 15:02:47.992385", + "modified": "2019-03-01 20:58:04.320605", "modified_by": "Administrator", "module": "ERPNext Integrations", "name": "Tally Migration", diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py index 4c27d743fb01..aac371dd3ee1 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -10,6 +10,7 @@ import frappe from frappe.model.document import Document from bs4 import BeautifulSoup as bs +from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts PRIMARY_ACCOUNT = "Primary" @@ -19,6 +20,15 @@ def _preprocess(self): self.tally_company = company self.erpnext_company = company self.status = "Preprocessed" + + coa_file = frappe.get_doc({ + "doctype": "File", + "file_name": "COA.json", + "attached_to_doctype": self.doctype, + "attached_to_name": self.name, + "content": json.dumps(chart_of_accounts_tree) + }).insert() + self.chart_of_accounts = coa_file.file_url self.save() def _process_master_data(self): @@ -120,7 +130,19 @@ def preprocess(self): frappe.enqueue_doc(self.doctype, self.name, "_preprocess") def start_import(self): - pass + def create_company_and_coa(coa_file_url): + coa_file = frappe.get_doc("File", {"file_url": coa_file_url}) + frappe.local.flags.ignore_chart_of_accounts = True + company = frappe.get_doc({ + "doctype": "Company", + "company_name": self.erpnext_company, + "default_currency": "INR", + }).insert() + frappe.local.flags.ignore_chart_of_accounts = False + create_charts(company.name, json.loads(coa_file.get_content())) + + create_company_and_coa(self.chart_of_accounts) + def sanitize(string): return re.sub("", "", string) From f2202cf638a789c379fb0fc38d6d7adb57728e18 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sat, 2 Mar 2019 17:23:23 +0530 Subject: [PATCH 4/8] feat(tally): Preprocess and create parties and addresses --- .../tally_migration/tally_migration.json | 66 +++++++++++- .../tally_migration/tally_migration.py | 101 +++++++++++++++++- 2 files changed, 164 insertions(+), 3 deletions(-) diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json index 2087c7f7bee6..6478cdb9cf37 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json @@ -368,6 +368,70 @@ "set_only_once": 0, "translatable": 0, "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "parties", + "fieldtype": "Attach", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Parties", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "addresses", + "fieldtype": "Attach", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Addresses", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 } ], "has_web_view": 0, @@ -380,7 +444,7 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2019-03-01 20:58:04.320605", + "modified": "2019-03-01 22:44:04.042954", "modified_by": "Administrator", "module": "ERPNext Integrations", "name": "Tally Migration", diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py index aac371dd3ee1..c8e2eba67acc 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -6,6 +6,7 @@ import json import re +import traceback import zipfile import frappe from frappe.model.document import Document @@ -17,6 +18,7 @@ class TallyMigration(Document): def _preprocess(self): company, chart_of_accounts_tree, customers, suppliers = self._process_master_data() + parties, addresses = self._process_parties(customers, suppliers) self.tally_company = company self.erpnext_company = company self.status = "Preprocessed" @@ -29,6 +31,25 @@ def _preprocess(self): "content": json.dumps(chart_of_accounts_tree) }).insert() self.chart_of_accounts = coa_file.file_url + + parties_file = frappe.get_doc({ + "doctype": "File", + "file_name": "Parties.json", + "attached_to_doctype": self.doctype, + "attached_to_name": self.name, + "content": json.dumps(parties) + }).insert() + self.parties = parties_file.file_url + + addresses_file = frappe.get_doc({ + "doctype": "File", + "file_name": "Addresses.json", + "attached_to_doctype": self.doctype, + "attached_to_name": self.name, + "content": json.dumps(addresses) + }).insert() + self.addresses = addresses_file.file_url + self.save() def _process_master_data(self): @@ -93,6 +114,7 @@ def get_children_and_parent_dict(accounts): for parent, account, is_group in accounts: children.setdefault(parent, set()).add(account) parents.setdefault(account, set()).add(parent) + parents[account].update(parents.get(parent, [])) return children, parents def remove_parties(parents, children, group_set): @@ -105,7 +127,7 @@ def remove_parties(parents, children, group_set): elif self.tally_debtors_account in parents[account]: children.pop(account, None) if account not in group_set: - suppliers.add(account) + suppliers.add(account) return children, customers, suppliers def traverse(tree, children, accounts, roots, group_set): @@ -126,10 +148,63 @@ def traverse(tree, children, accounts, roots, group_set): return company, chart_of_accounts_tree, customer_names, supplier_names + def _process_parties(self, customers, suppliers): + def get_master_collection(master_data): + master_file = frappe.get_doc("File", {"file_url": master_data}) + + with zipfile.ZipFile(master_file.get_full_path()) as zf: + content = zf.read(zf.namelist()[0]).decode("utf-16") + + master = bs(sanitize(emptify(content)), "xml") + collection = master.BODY.IMPORTDATA.REQUESTDATA + return collection + + def get_parties_addresses(collection, customers, suppliers): + parties, addresses = [], [] + for account in collection.find_all("LEDGER"): + party_type = None + if account.NAME.string in customers: + party_type = "Customer" + parties.append({ + "doctype": party_type, + "customer_name": account.NAME.string, + "tax_id": account.INCOMETAXNUMBER.string if account.INCOMETAXNUMBER else None, + "customer_group": "All Customer Groups", + "territory": "All Territories", + "customer_type": "Individual", + }) + elif account.NAME.string in suppliers: + party_type = "Supplier" + parties.append({ + "doctype": party_type, + "supplier_name": account.NAME.string, + "pan": account.INCOMETAXNUMBER.string if account.INCOMETAXNUMBER else None, + "supplier_group": "All Supplier Groups", + "supplier_type": "Individual", + }) + if party_type: + address = "\n".join([a.string for a in account.find_all("ADDRESS")[:2]]) + addresses.append({ + "doctype": "Address", + "address_line1": address[:140].strip(), + "address_line2": address[140:].strip(), + "country": account.COUNTRYNAME.string if account.COUNTRYNAME else None, + "state": account.STATENAME.string if account.STATENAME else None, + "gst_state": account.STATENAME.string if account.STATENAME else None, + "pin_code": account.PINCODE.string if account.PINCODE else None, + "gstin": account.PARTYGSTIN.string if account.PARTYGSTIN else None, + "links": [{"link_doctype": party_type, "link_name": account["NAME"]}], + }) + return parties, addresses + + collection = get_master_collection(self.master_data) + parties, addresses = get_parties_addresses(collection, customers, suppliers) + return parties, addresses + def preprocess(self): frappe.enqueue_doc(self.doctype, self.name, "_preprocess") - def start_import(self): + def _start_import(self): def create_company_and_coa(coa_file_url): coa_file = frappe.get_doc("File", {"file_url": coa_file_url}) frappe.local.flags.ignore_chart_of_accounts = True @@ -141,8 +216,30 @@ def create_company_and_coa(coa_file_url): frappe.local.flags.ignore_chart_of_accounts = False create_charts(company.name, json.loads(coa_file.get_content())) + def create_parties_addresses(parties_file_url, addresses_file_url): + parties_file = frappe.get_doc("File", {"file_url": parties_file_url}) + for party in json.loads(parties_file.get_content()): + try: + frappe.get_doc(party).insert() + except: + log(party) + addresses_file = frappe.get_doc("File", {"file_url": addresses_file_url}) + for address in json.loads(addresses_file.get_content()): + try: + frappe.get_doc(address).insert(ignore_mandatory=True) + except: + log(address) + create_company_and_coa(self.chart_of_accounts) + create_parties_addresses(self.parties, self.addresses) + + def start_import(self): + frappe.enqueue_doc(self.doctype, self.name, "_start_import") + +def log(data=None): + message = json.dumps({"data": data, "exception": traceback.format_exc()}, indent=4) + frappe.log_error(title="Tally Migration Error", message=message) def sanitize(string): return re.sub("", "", string) From 27ab5b540b78212ccd16b528d9770977e364f946 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sat, 2 Mar 2019 20:36:11 +0530 Subject: [PATCH 5/8] fix(tally): Use LEDSTATENAME instead of STATENAME --- .../doctype/tally_migration/tally_migration.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py index c8e2eba67acc..8f3bf5d6cd5e 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -189,9 +189,11 @@ def get_parties_addresses(collection, customers, suppliers): "address_line1": address[:140].strip(), "address_line2": address[140:].strip(), "country": account.COUNTRYNAME.string if account.COUNTRYNAME else None, - "state": account.STATENAME.string if account.STATENAME else None, - "gst_state": account.STATENAME.string if account.STATENAME else None, + "state": account.LEDSTATENAME.string if account.LEDSTATENAME else None, + "gst_state": account.LEDSTATENAME.string if account.LEDSTATENAME else None, "pin_code": account.PINCODE.string if account.PINCODE else None, + "mobile": account.LEDGERPHONE.string if account.LEDGERPHONE else None, + "phone": account.LEDGERPHONE.string if account.LEDGERPHONE else None, "gstin": account.PARTYGSTIN.string if account.PARTYGSTIN else None, "links": [{"link_doctype": party_type, "link_name": account["NAME"]}], }) From b6e9eb9784dfa70f14b2f2ab07599418aa7934ee Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Sat, 2 Mar 2019 22:12:41 +0530 Subject: [PATCH 6/8] feat(tally): Preprocess and create items and uoms --- .../tally_migration/tally_migration.json | 129 +++++++++++++++++- .../tally_migration/tally_migration.py | 86 +++++++++--- 2 files changed, 194 insertions(+), 21 deletions(-) diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json index 6478cdb9cf37..54e162b4af46 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json @@ -337,6 +337,38 @@ "translatable": 0, "unique": 0 }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "processed_files", + "fieldtype": "Section Break", + "hidden": 1, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Processed Files", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, { "allow_bulk_edit": 0, "allow_in_quick_entry": 0, @@ -432,6 +464,101 @@ "set_only_once": 0, "translatable": 0, "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "column_break_17", + "fieldtype": "Column Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "uoms", + "fieldtype": "Attach", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "UOMs", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "items", + "fieldtype": "Attach", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Items", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 } ], "has_web_view": 0, @@ -444,7 +571,7 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2019-03-01 22:44:04.042954", + "modified": "2019-03-02 21:51:48.869051", "modified_by": "Administrator", "module": "ERPNext Integrations", "name": "Tally Migration", diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py index 8f3bf5d6cd5e..cedf0167ab43 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -19,6 +19,7 @@ class TallyMigration(Document): def _preprocess(self): company, chart_of_accounts_tree, customers, suppliers = self._process_master_data() parties, addresses = self._process_parties(customers, suppliers) + items, uoms = self._process_stock_items() self.tally_company = company self.erpnext_company = company self.status = "Preprocessed" @@ -50,18 +51,38 @@ def _preprocess(self): }).insert() self.addresses = addresses_file.file_url + uoms_file = frappe.get_doc({ + "doctype": "File", + "file_name": "UOMs.json", + "attached_to_doctype": self.doctype, + "attached_to_name": self.name, + "content": json.dumps(uoms) + }).insert() + self.uoms = uoms_file.file_url + + items_file = frappe.get_doc({ + "doctype": "File", + "file_name": "Items.json", + "attached_to_doctype": self.doctype, + "attached_to_name": self.name, + "content": json.dumps(items) + }).insert() + self.items = items_file.file_url + + self.save() - def _process_master_data(self): - def get_master_collection(master_data): - master_file = frappe.get_doc("File", {"file_url": master_data}) + def get_master_collection(self): + master_file = frappe.get_doc("File", {"file_url": self.master_data}) - with zipfile.ZipFile(master_file.get_full_path()) as zf: - content = zf.read(zf.namelist()[0]).decode("utf-16") + with zipfile.ZipFile(master_file.get_full_path()) as zf: + content = zf.read(zf.namelist()[0]).decode("utf-16") - master = bs(sanitize(emptify(content)), "xml") - collection = master.BODY.IMPORTDATA.REQUESTDATA - return collection + master = bs(sanitize(emptify(content)), "xml") + collection = master.BODY.IMPORTDATA.REQUESTDATA + return collection + + def _process_master_data(self): def get_company_name(collection): return collection.find_all("REMOTECMPINFO.LIST")[0].REMOTECMPNAME.string @@ -141,7 +162,7 @@ def traverse(tree, children, accounts, roots, group_set): tree[account] = {} return tree - collection = get_master_collection(self.master_data) + collection = self.get_master_collection() company = get_company_name(collection) chart_of_accounts_tree, customer_names, supplier_names = get_coa_customers_suppliers(collection) @@ -149,16 +170,6 @@ def traverse(tree, children, accounts, roots, group_set): return company, chart_of_accounts_tree, customer_names, supplier_names def _process_parties(self, customers, suppliers): - def get_master_collection(master_data): - master_file = frappe.get_doc("File", {"file_url": master_data}) - - with zipfile.ZipFile(master_file.get_full_path()) as zf: - content = zf.read(zf.namelist()[0]).decode("utf-16") - - master = bs(sanitize(emptify(content)), "xml") - collection = master.BODY.IMPORTDATA.REQUESTDATA - return collection - def get_parties_addresses(collection, customers, suppliers): parties, addresses = [], [] for account in collection.find_all("LEDGER"): @@ -199,10 +210,29 @@ def get_parties_addresses(collection, customers, suppliers): }) return parties, addresses - collection = get_master_collection(self.master_data) + collection = self.get_master_collection() parties, addresses = get_parties_addresses(collection, customers, suppliers) return parties, addresses + def _process_stock_items(self): + collection = self.get_master_collection() + uoms = [] + for uom in collection.find_all("UNIT"): + uoms.append({"doctype": "UOM", "uom_name": uom.NAME.string}) + + items = [] + for item in collection.find_all("STOCKITEM"): + hsn_code = item.find_all("GSTDETAILS.LIST")[0].HSNCODE + items.append({ + "doctype": "Item", + "item_code" : item.NAME.string, + "stock_uom": item.BASEUNITS.string, + "gst_hsn_code": hsn_code.string if hsn_code else None, + "is_stock_item": 0, + "item_group": "All Item Groups", + }) + return items, uoms + def preprocess(self): frappe.enqueue_doc(self.doctype, self.name, "_preprocess") @@ -232,8 +262,24 @@ def create_parties_addresses(parties_file_url, addresses_file_url): except: log(address) + def create_items_uoms(items_file_url, uoms_file_url): + uoms_file = frappe.get_doc("File", {"file_url": uoms_file_url}) + for uom in json.loads(uoms_file.get_content()): + try: + frappe.get_doc(uom).insert() + except: + log(uom) + + items_file = frappe.get_doc("File", {"file_url": items_file_url}) + for item in json.loads(items_file.get_content()): + try: + frappe.get_doc(item).insert() + except: + log(item) + create_company_and_coa(self.chart_of_accounts) create_parties_addresses(self.parties, self.addresses) + create_items_uoms(self.items, self.uoms) def start_import(self): frappe.enqueue_doc(self.doctype, self.name, "_start_import") From ba61be93ae305e42b7b0411de16804d5a4549813 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Tue, 5 Mar 2019 19:16:00 +0530 Subject: [PATCH 7/8] refactor: Major changes --- .../tally_migration/tally_migration.js | 67 +- .../tally_migration/tally_migration.json | 639 ++++-------------- .../tally_migration/tally_migration.py | 445 ++++++++---- 3 files changed, 493 insertions(+), 658 deletions(-) diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js index 4652d60bff70..104ac570c690 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js @@ -2,36 +2,49 @@ // For license information, please see license.txt frappe.ui.form.on('Tally Migration', { + onload: function(frm) { + frappe.realtime.on("tally_migration_progress_update", function (data) { + frm.dashboard.show_progress(data.title, (data.count / data.total) * 100, data.message); + if (data.count == data.total) { + window.setTimeout(title => frm.dashboard.hide_progress(title), 1500, data.title); + } + }); + }, refresh: function(frm) { - if (frm.doc.master_data && frm.doc.day_book) { - frm.disable_save(); - if(frm.doc.status != "In Progress") { - frm.page.set_primary_action("Preprocess", () => frm.trigger("preprocess")); + if (frm.doc.master_data && !frm.doc.is_master_data_imported) { + if (frm.doc.is_master_data_processed) { + if (frm.doc.status != "Importing Master Data") { + frm.events.add_button(frm, __("Import Master Data"), "import_master_data"); + } + } else { + if (frm.doc.status != "Processing Master Data") { + frm.events.add_button(frm, __("Process Master Data"), "process_master_data"); + } } - } else { - frm.set_value("status", "Attach File"); } - if (frm.doc.tally_company && frm.doc.erpnext_company) { - frm.set_df_property("company_section", "hidden", 0); - frm.page.set_primary_action("Start Import", () => frm.trigger("start_import")); + if (frm.doc.day_book_data && !frm.doc.is_day_book_data_imported) { + if (frm.doc.is_day_book_data_processed) { + if (frm.doc.status != "Importing Day Book Data") { + frm.events.add_button(frm, __("Import Day Book Data"), "import_day_book_data"); + } + } else { + if (frm.doc.status != "Processing Day Book Data") { + frm.events.add_button(frm, __("Process Day Book Data"), "process_day_book_data"); + } + } } }, - preprocess: function(frm) { - frm.call({ - doc: frm.doc, - method: "preprocess", - freeze: true - }).then((r) => { - frm.set_value("status", "Preprocessing In Progress"); - }); - }, - start_import: function(frm) { - frm.call({ - doc: frm.doc, - method: "start_import", - freeze: true - }).then((r) => { - frm.set_value("status", "Import In Progress"); - }); - }, + add_button: function(frm, label, method) { + frm.add_custom_button( + label, + () => frm.call({ + doc: frm.doc, + method: method, + freeze: true, + callback: () => { + frm.remove_custom_button(label); + } + }) + ); + } }); diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json index 54e162b4af46..26415caf8d35 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json @@ -1,610 +1,219 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "", - "beta": 0, + "beta": 1, "creation": "2019-02-01 14:27:09.485238", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "status", + "master_data", + "is_master_data_processed", + "is_master_data_imported", + "column_break_2", + "tally_creditors_account", + "tally_debtors_account", + "company_section", + "tally_company", + "column_break_8", + "erpnext_company", + "processed_files_section", + "chart_of_accounts", + "parties", + "addresses", + "column_break_17", + "uoms", + "items", + "vouchers", + "accounts_section", + "default_warehouse", + "round_off_account", + "column_break_21", + "default_cost_center", + "day_book_section", + "day_book_data", + "column_break_27", + "is_day_book_data_processed", + "is_day_book_data_imported" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "status", "fieldtype": "Data", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Status", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Status" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "master_data", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, - "label": "Master Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Master Data" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "default": "Sundry Creditors", "fieldname": "tally_creditors_account", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Tally Creditors Account", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "column_break_2", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "day_book", - "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Day Book", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Column Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "default": "Sundry Debtors", "fieldname": "tally_debtors_account", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Tally Debtors Account", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, + "depends_on": "is_master_data_processed", "fieldname": "company_section", - "fieldtype": "Section Break", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Company Section", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Section Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "tally_company", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Tally Company", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "column_break_8", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Column Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "erpnext_company", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "ERPNext Company", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "ERPNext Company" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "processed_files", + "fieldname": "processed_files_section", "fieldtype": "Section Break", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Processed Files", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Processed Files" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "chart_of_accounts", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Chart of Accounts", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Chart of Accounts" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "parties", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Parties", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Parties" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "addresses", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Addresses", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Addresses" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "column_break_17", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Column Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "uoms", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "UOMs", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "UOMs" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "items", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Items", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Items" + }, + { + "fieldname": "vouchers", + "fieldtype": "Attach", + "label": "Vouchers" + }, + { + "depends_on": "is_master_data_imported", + "fieldname": "accounts_section", + "fieldtype": "Section Break", + "label": "Accounts" + }, + { + "fieldname": "default_warehouse", + "fieldtype": "Link", + "label": "Default Warehouse", + "options": "Warehouse" + }, + { + "fieldname": "round_off_account", + "fieldtype": "Link", + "label": "Round Off Account", + "options": "Account" + }, + { + "fieldname": "column_break_21", + "fieldtype": "Column Break" + }, + { + "fieldname": "default_cost_center", + "fieldtype": "Link", + "label": "Default Cost Center", + "options": "Cost Center" + }, + { + "fieldname": "is_master_data_processed", + "fieldtype": "Check", + "label": "Is Master Data Processed", + "read_only": 1 + }, + { + "fieldname": "is_day_book_data_processed", + "fieldtype": "Check", + "label": "Is Day Book Data Processed", + "read_only": 1 + }, + { + "fieldname": "is_day_book_data_imported", + "fieldtype": "Check", + "label": "Is Day Book Data Imported", + "read_only": 1 + }, + { + "fieldname": "is_master_data_imported", + "fieldtype": "Check", + "label": "Is Master Data Imported", + "read_only": 1 + }, + { + "depends_on": "is_master_data_imported", + "fieldname": "day_book_section", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_27", + "fieldtype": "Column Break" + }, + { + "fieldname": "day_book_data", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Day Book Data" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-03-02 21:51:48.869051", + "modified": "2019-04-29 05:46:54.394967", "modified_by": "Administrator", "module": "ERPNext Integrations", "name": "Tally Migration", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "track_changes": 1 } \ No newline at end of file diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py index cedf0167ab43..5b96ad36f14f 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -4,86 +4,65 @@ from __future__ import unicode_literals +from decimal import Decimal import json import re import traceback import zipfile import frappe +from frappe import _ +from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.model.document import Document +from frappe.model.naming import getseries, revert_series_if_last +from frappe.utils.data import format_datetime from bs4 import BeautifulSoup as bs -from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts +from erpnext import encode_company_abbr +from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts PRIMARY_ACCOUNT = "Primary" +VOUCHER_CHUNK_SIZE = 500 + class TallyMigration(Document): - def _preprocess(self): - company, chart_of_accounts_tree, customers, suppliers = self._process_master_data() - parties, addresses = self._process_parties(customers, suppliers) - items, uoms = self._process_stock_items() - self.tally_company = company - self.erpnext_company = company - self.status = "Preprocessed" - - coa_file = frappe.get_doc({ - "doctype": "File", - "file_name": "COA.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(chart_of_accounts_tree) - }).insert() - self.chart_of_accounts = coa_file.file_url - - parties_file = frappe.get_doc({ - "doctype": "File", - "file_name": "Parties.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(parties) - }).insert() - self.parties = parties_file.file_url - - addresses_file = frappe.get_doc({ - "doctype": "File", - "file_name": "Addresses.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(addresses) - }).insert() - self.addresses = addresses_file.file_url - - uoms_file = frappe.get_doc({ - "doctype": "File", - "file_name": "UOMs.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(uoms) - }).insert() - self.uoms = uoms_file.file_url - - items_file = frappe.get_doc({ - "doctype": "File", - "file_name": "Items.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(items) - }).insert() - self.items = items_file.file_url + def autoname(self): + if not self.name: + self.name = "Tally Migration on " + format_datetime(self.creation) + def get_collection(self, data_file): + def sanitize(string): + return re.sub("", "", string) - self.save() + def emptify(string): + string = re.sub(r"<\w+/>", "", string) + string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string) + string = re.sub(r"\r\n", "", string) + return string - def get_master_collection(self): - master_file = frappe.get_doc("File", {"file_url": self.master_data}) + master_file = frappe.get_doc("File", {"file_url": data_file}) with zipfile.ZipFile(master_file.get_full_path()) as zf: - content = zf.read(zf.namelist()[0]).decode("utf-16") + encoded_content = zf.read(zf.namelist()[0]) + try: + content = encoded_content.decode("utf-8-sig") + except UnicodeDecodeError: + content = encoded_content.decode("utf-16") master = bs(sanitize(emptify(content)), "xml") collection = master.BODY.IMPORTDATA.REQUESTDATA return collection - def _process_master_data(self): + def dump_processed_data(self, data): + for key, value in data.items(): + f = frappe.get_doc({ + "doctype": "File", + "file_name": key + ".json", + "attached_to_doctype": self.doctype, + "attached_to_name": self.name, + "content": json.dumps(value) + }).insert() + setattr(self, key, f.file_url) + def _process_master_data(self): def get_company_name(collection): return collection.find_all("REMOTECMPINFO.LIST")[0].REMOTECMPNAME.string @@ -144,11 +123,11 @@ def remove_parties(parents, children, group_set): if self.tally_creditors_account in parents[account]: children.pop(account, None) if account not in group_set: - customers.add(account) + suppliers.add(account) elif self.tally_debtors_account in parents[account]: children.pop(account, None) if account not in group_set: - suppliers.add(account) + customers.add(account) return children, customers, suppliers def traverse(tree, children, accounts, roots, group_set): @@ -162,14 +141,6 @@ def traverse(tree, children, accounts, roots, group_set): tree[account] = {} return tree - collection = self.get_master_collection() - - company = get_company_name(collection) - chart_of_accounts_tree, customer_names, supplier_names = get_coa_customers_suppliers(collection) - - return company, chart_of_accounts_tree, customer_names, supplier_names - - def _process_parties(self, customers, suppliers): def get_parties_addresses(collection, customers, suppliers): parties, addresses = [], [] for account in collection.find_all("LEDGER"): @@ -210,33 +181,49 @@ def get_parties_addresses(collection, customers, suppliers): }) return parties, addresses - collection = self.get_master_collection() + def get_stock_items_uoms(collection): + uoms = [] + for uom in collection.find_all("UNIT"): + uoms.append({"doctype": "UOM", "uom_name": uom.NAME.string}) + + items = [] + for item in collection.find_all("STOCKITEM"): + items.append({ + "doctype": "Item", + "item_code" : item.NAME.string, + "stock_uom": item.BASEUNITS.string, + "is_stock_item": 0, + "item_group": "All Item Groups", + "item_defaults": [{"company": self.erpnext_company}] + }) + return items, uoms + + + self.publish("Process Master Data", _("Reading Uploaded File"), 1, 5) + collection = self.get_collection(self.master_data) + + company = get_company_name(collection) + self.tally_company = company + self.erpnext_company = company + + self.publish("Process Master Data", _("Processing Chart of Accounts and Parties"), 2, 5) + chart_of_accounts, customers, suppliers = get_coa_customers_suppliers(collection) + self.publish("Process Master Data", _("Processing Party Addresses"), 3, 5) parties, addresses = get_parties_addresses(collection, customers, suppliers) - return parties, addresses - - def _process_stock_items(self): - collection = self.get_master_collection() - uoms = [] - for uom in collection.find_all("UNIT"): - uoms.append({"doctype": "UOM", "uom_name": uom.NAME.string}) - - items = [] - for item in collection.find_all("STOCKITEM"): - hsn_code = item.find_all("GSTDETAILS.LIST")[0].HSNCODE - items.append({ - "doctype": "Item", - "item_code" : item.NAME.string, - "stock_uom": item.BASEUNITS.string, - "gst_hsn_code": hsn_code.string if hsn_code else None, - "is_stock_item": 0, - "item_group": "All Item Groups", - }) - return items, uoms - - def preprocess(self): - frappe.enqueue_doc(self.doctype, self.name, "_preprocess") - - def _start_import(self): + self.publish("Process Master Data", _("Processing Items and UOMs"), 4, 5) + items, uoms = get_stock_items_uoms(collection) + data = {"chart_of_accounts": chart_of_accounts, "parties": parties, "addresses": addresses, "items": items, "uoms": uoms} + self.publish("Process Master Data", _("Done"), 5, 5) + + self.dump_processed_data(data) + self.is_master_data_processed = 1 + self.status = "" + self.save() + + def publish(self, title, message, count, total): + frappe.publish_realtime("tally_migration_progress_update", {"title": title, "message": message, "count": count, "total": total}) + + def _import_master_data(self): def create_company_and_coa(coa_file_url): coa_file = frappe.get_doc("File", {"file_url": coa_file_url}) frappe.local.flags.ignore_chart_of_accounts = True @@ -244,56 +231,282 @@ def create_company_and_coa(coa_file_url): "doctype": "Company", "company_name": self.erpnext_company, "default_currency": "INR", + "enable_perpetual_inventory": 0, }).insert() frappe.local.flags.ignore_chart_of_accounts = False - create_charts(company.name, json.loads(coa_file.get_content())) + create_charts(company.name, custom_chart=json.loads(coa_file.get_content())) + company.create_default_warehouses() - def create_parties_addresses(parties_file_url, addresses_file_url): + def create_parties_and_addresses(parties_file_url, addresses_file_url): parties_file = frappe.get_doc("File", {"file_url": parties_file_url}) for party in json.loads(parties_file.get_content()): try: frappe.get_doc(party).insert() except: - log(party) + self.log(party) addresses_file = frappe.get_doc("File", {"file_url": addresses_file_url}) for address in json.loads(addresses_file.get_content()): try: frappe.get_doc(address).insert(ignore_mandatory=True) except: - log(address) + try: + gstin = address.pop("gstin", None) + frappe.get_doc(address).insert(ignore_mandatory=True) + self.log({"address": address, "message": "Invalid GSTIN: {}. Address was created without GSTIN".format(gstin)}) + except: + self.log(address) + def create_items_uoms(items_file_url, uoms_file_url): uoms_file = frappe.get_doc("File", {"file_url": uoms_file_url}) for uom in json.loads(uoms_file.get_content()): - try: - frappe.get_doc(uom).insert() - except: - log(uom) + if not frappe.db.exists(uom): + try: + frappe.get_doc(uom).insert() + except: + self.log(uom) items_file = frappe.get_doc("File", {"file_url": items_file_url}) for item in json.loads(items_file.get_content()): try: frappe.get_doc(item).insert() except: - log(item) + self.log(item) + self.publish("Import Master Data", _("Creating Company and Importing Chart of Accounts"), 1, 4) create_company_and_coa(self.chart_of_accounts) - create_parties_addresses(self.parties, self.addresses) + self.publish("Import Master Data", _("Importing Parties and Addresses"), 2, 4) + create_parties_and_addresses(self.parties, self.addresses) + self.publish("Import Master Data", _("Importing Items and UOMs"), 3, 4) create_items_uoms(self.items, self.uoms) + self.publish("Import Master Data", _("Done"), 4, 4) + self.status = "" + self.is_master_data_imported = 1 + self.save() - def start_import(self): - frappe.enqueue_doc(self.doctype, self.name, "_start_import") + def _process_day_book_data(self): + def get_vouchers(collection): + vouchers = [] + for voucher in collection.find_all("VOUCHER"): + if voucher.ISCANCELLED.string == "Yes": + continue + inventory_entries = voucher.find_all("INVENTORYENTRIES.LIST") + voucher.find_all("ALLINVENTORYENTRIES.LIST") + voucher.find_all("INVENTORYENTRIESIN.LIST") + voucher.find_all("INVENTORYENTRIESOUT.LIST") + if voucher.VOUCHERTYPENAME.string not in ["Journal", "Receipt", "Payment", "Contra"] and inventory_entries: + function = voucher_to_invoice + else: + function = voucher_to_journal_entry + try: + vouchers.append(function(voucher)) + except: + self.log(voucher) + return vouchers + + def voucher_to_journal_entry(voucher): + accounts = [] + ledger_entries = voucher.find_all("ALLLEDGERENTRIES.LIST") + voucher.find_all("LEDGERENTRIES.LIST") + for entry in ledger_entries: + account = {"account": encode_company_abbr(entry.LEDGERNAME.string, self.erpnext_company), "cost_center": self.default_cost_center} + if entry.ISPARTYLEDGER.string == "Yes": + party_details = get_party(entry.LEDGERNAME.string) + if party_details: + party_type, party_account = party_details + account["party_type"] = party_type + account["account"] = party_account + account["party"] = entry.LEDGERNAME.string + amount = Decimal(entry.AMOUNT.string) + if amount > 0: + account["credit_in_account_currency"] = str(abs(amount)) + else: + account["debit_in_account_currency"] = str(abs(amount)) + accounts.append(account) + + journal_entry = { + "doctype": "Journal Entry", + "tally_guid": voucher.GUID.string, + "posting_date": voucher.DATE.string, + "company": self.erpnext_company, + "accounts": accounts, + } + return journal_entry + + def voucher_to_invoice(voucher): + if voucher.VOUCHERTYPENAME.string in ["Sales", "Credit Note"]: + doctype = "Sales Invoice" + party_field = "customer" + account_field = "debit_to" + account_name = encode_company_abbr(self.tally_debtors_account, self.erpnext_company) + price_list_field = "selling_price_list" + elif voucher.VOUCHERTYPENAME.string in ["Purchase", "Debit Note"]: + doctype = "Purchase Invoice" + party_field = "supplier" + account_field = "credit_to" + account_name = encode_company_abbr(self.tally_creditors_account, self.erpnext_company) + price_list_field = "buying_price_list" + + invoice = { + "doctype": doctype, + party_field: voucher.PARTYNAME.string, + "tally_guid": voucher.GUID.string, + "posting_date": voucher.DATE.string, + "due_date": voucher.DATE.string, + "items": get_voucher_items(voucher, doctype), + "taxes": get_voucher_taxes(voucher), + account_field: account_name, + price_list_field: "Tally Price List", + "set_posting_time": 1, + "disable_rounded_total": 1, + "company": self.erpnext_company, + } + return invoice + + def get_voucher_items(voucher, doctype): + inventory_entries = voucher.find_all("INVENTORYENTRIES.LIST") + voucher.find_all("ALLINVENTORYENTRIES.LIST") + voucher.find_all("INVENTORYENTRIESIN.LIST") + voucher.find_all("INVENTORYENTRIESOUT.LIST") + if doctype == "Sales Invoice": + account_field = "income_account" + elif doctype == "Purchase Invoice": + account_field = "expense_account" + items = [] + for entry in inventory_entries: + qty, uom = entry.ACTUALQTY.string.strip().split() + items.append({ + "item_code": entry.STOCKITEMNAME.string, + "description": entry.STOCKITEMNAME.string, + "qty": qty.strip(), + "uom": uom.strip(), + "conversion_factor": 1, + "price_list_rate": entry.RATE.string.split("/")[0], + "cost_center": self.default_cost_center, + "warehouse": self.default_warehouse, + account_field: encode_company_abbr(entry.find_all("ACCOUNTINGALLOCATIONS.LIST")[0].LEDGERNAME.string, self.erpnext_company), + }) + return items + + def get_voucher_taxes(voucher): + ledger_entries = voucher.find_all("ALLLEDGERENTRIES.LIST") + voucher.find_all("LEDGERENTRIES.LIST") + taxes = [] + for entry in ledger_entries: + if entry.ISPARTYLEDGER.string == "No": + tax_account = encode_company_abbr(entry.LEDGERNAME.string, self.erpnext_company) + taxes.append({ + "charge_type": "Actual", + "account_head": tax_account, + "description": tax_account, + "tax_amount": entry.AMOUNT.string, + "cost_center": self.default_cost_center, + }) + return taxes + + def get_party(party): + if frappe.db.exists({"doctype": "Supplier", "supplier_name": party}): + return "Supplier", encode_company_abbr(self.tally_creditors_account, self.erpnext_company) + elif frappe.db.exists({"doctype": "Customer", "customer_name": party}): + return "Customer", encode_company_abbr(self.tally_debtors_account, self.erpnext_company) + + self.publish("Process Day Book Data", _("Reading Uploaded File"), 1, 3) + collection = self.get_collection(self.day_book_data) + self.publish("Process Day Book Data", _("Processing Vouchers"), 2, 3) + vouchers = get_vouchers(collection) + self.publish("Process Day Book Data", _("Done"), 3, 3) + self.dump_processed_data({"vouchers": vouchers}) + self.status = "" + self.is_day_book_data_processed = 1 + self.save() + def _import_day_book_data(self): + def create_fiscal_years(vouchers): + from frappe.utils.data import add_years, getdate + earliest_date = getdate(min(voucher["posting_date"] for voucher in vouchers)) + oldest_year = frappe.get_all("Fiscal Year", fields=["year_start_date", "year_end_date"], order_by="year_start_date")[0] + while earliest_date < oldest_year.year_start_date: + new_year = frappe.get_doc({"doctype": "Fiscal Year"}) + new_year.year_start_date = add_years(oldest_year.year_start_date, -1) + new_year.year_end_date = add_years(oldest_year.year_end_date, -1) + if new_year.year_start_date.year == new_year.year_end_date.year: + new_year.year = new_year.year_start_date.year + else: + new_year.year = "{}-{}".format(new_year.year_start_date.year, new_year.year_end_date.year) + new_year.save() + oldest_year = new_year + + def create_custom_fields(doctypes): + for doctype in doctypes: + df = { + "fieldtype": "Data", + "fieldname": "tally_guid", + "read_only": 1, + "label": "Tally GUID" + } + create_custom_field(doctype, df) + + def create_price_list(): + frappe.get_doc({ + "doctype": "Price List", + "price_list_name": "Tally Price List", + "selling": 1, + "buying": 1, + "enabled": 1, + "currency": "INR" + }).insert() -def log(data=None): - message = json.dumps({"data": data, "exception": traceback.format_exc()}, indent=4) - frappe.log_error(title="Tally Migration Error", message=message) + frappe.db.set_value("Account", encode_company_abbr(self.tally_creditors_account, self.erpnext_company), "account_type", "Payable") + frappe.db.set_value("Account", encode_company_abbr(self.tally_debtors_account, self.erpnext_company), "account_type", "Receivable") + frappe.db.set_value("Company", self.erpnext_company, "round_off_account", self.round_off_account) + + vouchers_file = frappe.get_doc("File", {"file_url": self.vouchers}) + vouchers = json.loads(vouchers_file.get_content()) + + create_fiscal_years(vouchers) + create_price_list() + create_custom_fields(["Journal Entry", "Purchase Invoice", "Sales Invoice"]) + + total = len(vouchers) + is_last = False + for index in range(0, total, VOUCHER_CHUNK_SIZE): + if index + VOUCHER_CHUNK_SIZE >= total: + is_last = True + frappe.enqueue_doc(self.doctype, self.name, "_import_vouchers", queue="long", timeout=3600, start=index+1, total=total, is_last=is_last) + + def _import_vouchers(self, start, total, is_last=False): + frappe.flags.in_migrate = True + vouchers_file = frappe.get_doc("File", {"file_url": self.vouchers}) + vouchers = json.loads(vouchers_file.get_content()) + chunk = vouchers[start: start + VOUCHER_CHUNK_SIZE] + + for index, voucher in enumerate(chunk, start=start): + try: + doc = frappe.get_doc(voucher).insert() + doc.submit() + self.publish("Importing Vouchers", _("{} of {}").format(index, total), index, total) + except: + self.log(voucher) + + if is_last: + self.status = "" + self.is_day_book_data_imported = 1 + self.save() + frappe.db.set_value("Price List", "Tally Price List", "enabled", 0) + frappe.flags.in_migrate = False + + def process_master_data(self): + self.status = "Processing Master Data" + self.save() + frappe.enqueue_doc(self.doctype, self.name, "_process_master_data", queue="long", timeout=3600) -def sanitize(string): - return re.sub("", "", string) + def import_master_data(self): + self.status = "Importing Master Data" + self.save() + frappe.enqueue_doc(self.doctype, self.name, "_import_master_data", queue="long", timeout=3600) + + def process_day_book_data(self): + self.status = "Processing Day Book Data" + self.save() + frappe.enqueue_doc(self.doctype, self.name, "_process_day_book_data", queue="long", timeout=3600) + + def import_day_book_data(self): + self.status = "Importing Day Book Data" + self.save() + frappe.enqueue_doc(self.doctype, self.name, "_import_day_book_data", queue="long", timeout=3600) -def emptify(string): - string = re.sub(r"<\w+/>", "", string) - string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string) - string = re.sub(r"\r\n", "", string) - return string + def log(self, data=None): + message = "\n".join(["Data", json.dumps(data, default=str, indent=4), "Exception", traceback.format_exc()]) + return frappe.log_error(title="Tally Migration Error", message=message) From 8afeda924d0a4485dc5e1922aaf7c60aadfe6785 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Mon, 29 Apr 2019 13:09:23 +0530 Subject: [PATCH 8/8] fix: Use all address lines instead of only first two --- .../doctype/tally_migration/tally_migration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py index 5b96ad36f14f..12b646dad771 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -165,7 +165,7 @@ def get_parties_addresses(collection, customers, suppliers): "supplier_type": "Individual", }) if party_type: - address = "\n".join([a.string for a in account.find_all("ADDRESS")[:2]]) + address = "\n".join([a.string for a in account.find_all("ADDRESS")]) addresses.append({ "doctype": "Address", "address_line1": address[:140].strip(),