[IMP] models: ORM speedup

This branch is the combination of several optimizations in the ORM:

* store field values once in the cache: the cache reflects more
faithfully the database, only fields that explicitly depend on the
context have an extra indirection in the cache;

* delay recomputations by default: use method `recompute` to explicitly
flush out pending recomputations;

* delay updates in method `write`: updates are stored in a data
structure that can be flushed efficiently to the database with method
`flush` (which also flush out recomputations);

* make method `modified` take advantage of inverse fields to inverse
dependencies;

* filter records by evaluating a domain on records in Python;

* a computed field with `readonly=False` behaves like a normal field
with an onchange method;

* computed fields are computed in superuser mode by default.

Work done by Toufik Ben Jaa, Raphael Collet, Denis Ledoux and Fabien
Pinckaers.

closes #35659

Signed-off-by: Denis Ledoux <beledouxdenis@users.noreply.github.com>

addons/account/models/account.py

@@ -1148,16 +1148,18 @@ def _check_repartition_lines(self, lines):
     @api.constrains('invoice_repartition_line_ids', 'refund_repartition_line_ids')
     def _validate_repartition_lines(self):
         for record in self:
-            record._check_repartition_lines(record.invoice_repartition_line_ids)
-            record._check_repartition_lines(record.refund_repartition_line_ids)
+            invoice_repartition_line_ids = record.invoice_repartition_line_ids.sorted()
+            refund_repartition_line_ids = record.refund_repartition_line_ids.sorted()
+            record._check_repartition_lines(invoice_repartition_line_ids)
+            record._check_repartition_lines(refund_repartition_line_ids)
 
-            if len(record.invoice_repartition_line_ids) != len(record.refund_repartition_line_ids):
+            if len(invoice_repartition_line_ids) != len(refund_repartition_line_ids):
                 raise ValidationError(_("Invoice and credit note repartition should have the same number of lines."))
 
             index = 0
-            while index < len(record.invoice_repartition_line_ids):
-                inv_rep_ln = record.invoice_repartition_line_ids[index]
-                ref_rep_ln = record.refund_repartition_line_ids[index]
+            while index < len(invoice_repartition_line_ids):
+                inv_rep_ln = invoice_repartition_line_ids[index]
+                ref_rep_ln = refund_repartition_line_ids[index]
                 if inv_rep_ln.repartition_type != ref_rep_ln.repartition_type or inv_rep_ln.factor_percent != ref_rep_ln.factor_percent:
                     raise ValidationError(_("Invoice and credit note repartitions should match (same percentages, in the same order)."))
                 index += 1

addons/account/models/account_journal_dashboard.py

@@ -25,6 +25,8 @@ def _kanban_dashboard_graph(self):
                 journal.kanban_dashboard_graph = json.dumps(journal.get_bar_graph_datas())
             elif (journal.type in ['cash', 'bank']):
                 journal.kanban_dashboard_graph = json.dumps(journal.get_line_graph_datas())
+            else:
+                journal.kanban_dashboard_graph = False
 
     def _get_json_activity_data(self):
         for journal in self:
@@ -243,6 +245,7 @@ def get_journal_dashboard_datas(self):
         #TODO need to check if all invoices are in the same currency than the journal!!!!
         elif self.type in ['sale', 'purchase']:
             title = _('Bills to pay') if self.type == 'purchase' else _('Invoices owed to you')
+            self.env['account.move'].flush(['amount_residual', 'currency_id', 'type', 'invoice_date', 'company_id', 'journal_id', 'date', 'state', 'invoice_payment_state'])
 
             (query, query_args) = self._get_open_bills_to_pay_query()
             self.env.cr.execute(query, query_args)

addons/account/models/account_payment.py

@@ -160,9 +160,11 @@ def _compute_hide_payment_method(self):
 
     @api.depends('invoice_ids', 'amount', 'payment_date', 'currency_id', 'payment_type')
     def _compute_payment_difference(self):
-        for pay in self.filtered(lambda p: p.invoice_ids and p.state == 'draft'):
+        draft_payments = self.filtered(lambda p: p.invoice_ids and p.state == 'draft')
+        for pay in draft_payments:
             payment_amount = -pay.amount if pay.payment_type == 'outbound' else pay.amount
             pay.payment_difference = pay._compute_payment_amount(pay.invoice_ids, pay.currency_id, pay.journal_id, pay.payment_date) - payment_amount
+        (self - draft_payments).payment_difference = 0
 
     @api.onchange('journal_id')
     def _onchange_journal(self):
@@ -298,6 +300,10 @@ def _compute_payment_amount(self, invoices, currency, journal, date):
         if not invoices:
             return 0.0
 
+        self.env['account.move'].flush(['type', 'currency_id'])
+        self.env['account.move.line'].flush(['amount_residual', 'amount_residual_currency', 'move_id', 'account_id'])
+        self.env['account.account'].flush(['user_type_id'])
+        self.env['account.account.type'].flush(['type'])
         self._cr.execute('''
             SELECT
                 move.type AS type,

addons/account/models/account_reconcile_model.py

@@ -636,6 +636,9 @@ def _apply_rules(self, st_lines, excluded_ids=None, partner_map=None):
         # Type == 'invoice_matching'.
         # Map each (st_line.id, model_id) with matching amls.
         invoices_models = ordered_models.filtered(lambda m: m.rule_type == 'invoice_matching')
+        self.env['account.move'].flush(['state'])
+        self.env['account.move.line'].flush(['balance', 'reconciled'])
+        self.env['account.bank.statement.line'].flush(['company_id'])
         if invoices_models:
             query, params = invoices_models._get_invoice_matching_query(st_lines, excluded_ids=excluded_ids, partner_map=partner_map)
             self._cr.execute(query, params)

addons/account/models/chart_template.py

@@ -922,7 +922,11 @@ def _generate_tax(self, company):
                 }
 
                 # We also have to delay the assignation of accounts to repartition lines
-                all_tax_rep_lines = tax.invoice_repartition_line_ids + tax.refund_repartition_line_ids
+                # The below code assigns the account_id to the repartition lines according
+                # to the corresponding repartition line in the template, based on the order.
+                # As we just created the repartition lines, tax.invoice_repartition_line_ids is not well sorted.
+                # But we can force the sort by calling sort()
+                all_tax_rep_lines = tax.invoice_repartition_line_ids.sorted() + tax.refund_repartition_line_ids.sorted()
                 all_template_rep_lines = template.invoice_repartition_line_ids + template.refund_repartition_line_ids
                 for i in range(0, len(all_template_rep_lines)):
                     # We assume template and tax repartition lines are in the same order

addons/account/models/partner.py

@@ -223,6 +223,7 @@ class ResPartner(models.Model):
     _name = 'res.partner'
     _inherit = 'res.partner'
 
+    @api.depends_context('force_company')
     def _credit_debit_get(self):
         tables, where_clause, where_params = self.env['account.move.line'].with_context(company_id=self.env.company.id)._query_get()
         where_params = [tuple(self.ids)] + where_params
@@ -238,12 +239,20 @@ def _credit_debit_get(self):
                       """ + where_clause + """
                       GROUP BY account_move_line.partner_id, act.type
                       """, where_params)
+        treated = self.browse()
         for pid, type, val in self._cr.fetchall():
             partner = self.browse(pid)
             if type == 'receivable':
                 partner.credit = val
+                partner.debit = False
+                treated |= partner
             elif type == 'payable':
                 partner.debit = -val
+                partner.credit = False
+                treated |= partner
+        remaining = (self - treated)
+        remaining.debit = False
+        remaining.credit = False
 
     def _asset_difference_search(self, account_type, operator, operand):
         if operator not in ('<', '=', '>', '>=', '<='):
@@ -337,6 +346,7 @@ def _compute_has_unreconciled_entries(self):
         for partner in self:
             # Avoid useless work if has_unreconciled_entries is not relevant for this partner
             if not partner.active or not partner.is_company and partner.parent_id:
+                partner.has_unreconciled_entries = False
                 continue
             self.env.cr.execute(
                 """ SELECT 1 FROM(

addons/account/tests/account_test_no_chart.py

@@ -90,6 +90,12 @@ def setUpAccountJournal(cls):
             'type': 'sale',
             'company_id': cls.env.user.company_id.id,
         })
+        cls.journal_general = cls.env['account.journal'].create({
+            'name': 'General Journal - Test',
+            'code': 'AJ-GENERAL',
+            'type': 'general',
+            'company_id': cls.env.user.company_id.id,
+        })
 
     @classmethod
     def setUpUsers(cls):

addons/account/tests/invoice_test_common.py

@@ -142,6 +142,6 @@ def init_invoice(cls, move_type):
     def assertInvoiceValues(self, move, expected_lines_values, expected_move_values):
         def sort_lines(lines):
             return lines.sorted(lambda line: (line.exclude_from_invoice_tab, not bool(line.tax_line_id), line.name or '', line.balance))
-        self.assertRecordValues(sort_lines(move.line_ids), expected_lines_values)
-        self.assertRecordValues(sort_lines(move.invoice_line_ids), expected_lines_values[:len(move.invoice_line_ids)])
+        self.assertRecordValues(sort_lines(move.line_ids.sorted()), expected_lines_values)
+        self.assertRecordValues(sort_lines(move.invoice_line_ids.sorted()), expected_lines_values[:len(move.invoice_line_ids)])
         self.assertRecordValues(move, [expected_move_values])

addons/account/tests/test_account_move_entry.py

@@ -77,6 +77,7 @@ def test_misc_tax_lock_date_1(self):
             ],
         })
 
+        self.test_move.flush()
         self.cr.execute('SAVEPOINT test_misc_tax_lock_date_1')
 
         # Writing something affecting a tax is not allowed.
@@ -106,6 +107,8 @@ def test_misc_tax_lock_date_1(self):
         with self.assertRaises(ValidationError):
             self.test_move.unlink()
 
+        self.test_move.flush()
+        self.test_move.invalidate_cache()
         self.cr.execute('ROLLBACK TO SAVEPOINT test_misc_tax_lock_date_1')
 
         with self.assertRaises(UserError):

addons/account/tests/test_invoice_taxes.py

@@ -213,7 +213,7 @@ def test_tax_repartition(self):
         self.assertEqual(len(inv_tax_lines), 2, "There should be two tax lines, one for each repartition line.")
         self.assertEqual(abs(inv_tax_lines.filtered(lambda x: x.account_id == account_1).balance), 4.2, "Tax line on account 1 should amount to 4.2 (10% of 42)")
         self.assertEqual(inv_tax_lines.filtered(lambda x: x.account_id == account_1).tag_ids, inv_tax_tag_10, "Tax line on account 1 should have 10% tag")
-        self.assertEqual(abs(inv_tax_lines.filtered(lambda x: x.account_id == account_2).balance), 37.8, "Tax line on account 2 should amount to 37.8 (90% of 42)")
+        self.assertAlmostEqual(abs(inv_tax_lines.filtered(lambda x: x.account_id == account_2).balance), 37.8, 2, "Tax line on account 2 should amount to 37.8 (90% of 42)")
         self.assertEqual(inv_tax_lines.filtered(lambda x: x.account_id == account_2).tag_ids, inv_tax_tag_90, "Tax line on account 2 should have 90% tag")
 
         # Test refund repartition
@@ -228,5 +228,5 @@ def test_tax_repartition(self):
         ref_tax_lines = refund.line_ids.filtered(lambda x: x.tax_repartition_line_id.repartition_type == 'tax')
         self.assertEqual(len(ref_tax_lines), 2, "There should be two refund tax lines")
         self.assertEqual(abs(ref_tax_lines.filtered(lambda x: x.account_id == ref_base_line.account_id).balance), 4.2, "Refund tax line on base account should amount to 4.2 (10% of 42)")
-        self.assertEqual(abs(ref_tax_lines.filtered(lambda x: x.account_id == account_1).balance), 37.8, "Refund tax line on account 1 should amount to 37.8 (90% of 42)")
+        self.assertAlmostEqual(abs(ref_tax_lines.filtered(lambda x: x.account_id == account_1).balance), 37.8, 2, "Refund tax line on account 1 should amount to 37.8 (90% of 42)")
         self.assertEqual(ref_tax_lines.mapped('tag_ids'), ref_tax_tag, "Refund tax lines should have the right tag")

addons/account/tests/test_reconciliation.py

@@ -1040,6 +1040,7 @@ def test_aged_report_future_payment(self):
         self.env.cr.execute('UPDATE account_partial_reconcile SET create_date = %(date)s WHERE id = %(partial_id)s',
             {'date': invoice.invoice_date,
              'partial_id': statement_partial_id.id})
+        statement.flush()
 
         # Case 1: report date is invoice date
         # There should be an entry for the partner
@@ -1249,6 +1250,7 @@ def test_reconciliation_cash_basis01(self):
         })
         payment_move.post()
 
+        (purchase_move + payment_move).invalidate_cache(['line_ids'])
         to_reconcile = (purchase_move + payment_move).mapped('line_ids').filtered(lambda l: l.account_id.internal_type == 'payable')
         to_reconcile.reconcile()
 
@@ -1258,15 +1260,17 @@ def test_reconciliation_cash_basis01(self):
         self.assertTrue(cash_basis_moves.exists())
 
         # check reconciliation in Payable account
-        self.assertTrue(purchase_move.line_ids[0].full_reconcile_id.exists())
-        self.assertEqual(purchase_move.line_ids[0].full_reconcile_id.reconciled_line_ids,
-             purchase_move.line_ids[0] + purchase_move.line_ids[1] + payment_move.line_ids[0])
+        purchase_move_line_ids = purchase_move.line_ids.sorted()
+        payment_move_line_ids = payment_move.line_ids.sorted()
+        self.assertTrue(purchase_move_line_ids[0].full_reconcile_id.exists())
+        self.assertEqual(purchase_move_line_ids[0].full_reconcile_id.reconciled_line_ids,
+             purchase_move_line_ids[0] + purchase_move_line_ids[1] + payment_move_line_ids[0])
 
         cash_basis_aml_ids = cash_basis_moves.mapped('line_ids')
         # check reconciliation in the tax waiting account
-        self.assertTrue(purchase_move.line_ids[4].full_reconcile_id.exists())
-        self.assertEqual(purchase_move.line_ids[4].full_reconcile_id.reconciled_line_ids,
-            cash_basis_aml_ids.filtered(lambda l: l.account_id == self.tax_waiting_account) + purchase_move.line_ids[4])
+        self.assertTrue(purchase_move_line_ids[4].full_reconcile_id.exists())
+        self.assertEqual(purchase_move_line_ids[4].full_reconcile_id.reconciled_line_ids,
+            cash_basis_aml_ids.filtered(lambda l: l.account_id == self.tax_waiting_account) + purchase_move_line_ids[4])
 
         self.assertEqual(len(cash_basis_aml_ids), 8)
 
@@ -1362,29 +1366,30 @@ def test_reconciliation_cash_basis02(self):
         })
         payment_move1.post()
 
-        (purchase_move + payment_move0).mapped('line_ids').filtered(lambda l: l.account_id.internal_type == 'payable').reconcile()
-        (purchase_move + payment_move1).mapped('line_ids').filtered(lambda l: l.account_id.internal_type == 'payable').reconcile()
+        (purchase_move + payment_move0).mapped('line_ids').sorted().filtered(lambda l: l.account_id.internal_type == 'payable').reconcile()
+        (purchase_move + payment_move1).mapped('line_ids').sorted().filtered(lambda l: l.account_id.internal_type == 'payable').reconcile()
 
         cash_basis_moves = self.env['account.move'].search([('journal_id', '=', self.cash_basis_journal.id)])
 
         self.assertEqual(len(cash_basis_moves), 3)
         self.assertTrue(cash_basis_moves.exists())
 
         # check reconciliation in Payable account
-        self.assertTrue(purchase_move.line_ids[0].full_reconcile_id.exists())
-        self.assertEqual(purchase_move.line_ids[0].full_reconcile_id.reconciled_line_ids,
+        purchase_move_line_ids = purchase_move.line_ids.sorted()
+        self.assertTrue(purchase_move_line_ids[0].full_reconcile_id.exists())
+        self.assertEqual(purchase_move_line_ids[0].full_reconcile_id.reconciled_line_ids,
             (purchase_move + payment_move0 + payment_move1).mapped('line_ids').filtered(lambda l: l.account_id.internal_type == 'payable'))
 
         cash_basis_aml_ids = cash_basis_moves.mapped('line_ids')
 
         # check reconciliation in the tax waiting account
-        self.assertTrue(purchase_move.line_ids[3].full_reconcile_id.exists())
-        self.assertEqual(purchase_move.line_ids[3].full_reconcile_id.reconciled_line_ids,
-            cash_basis_aml_ids.filtered(lambda l: l.account_id == tax_waiting_account10) + purchase_move.line_ids[3])
+        self.assertTrue(purchase_move_line_ids[3].full_reconcile_id.exists())
+        self.assertEqual(purchase_move_line_ids[3].full_reconcile_id.reconciled_line_ids,
+            cash_basis_aml_ids.filtered(lambda l: l.account_id == tax_waiting_account10) + purchase_move_line_ids[3])
 
-        self.assertTrue(purchase_move.line_ids[5].full_reconcile_id.exists())
-        self.assertEqual(purchase_move.line_ids[5].full_reconcile_id.reconciled_line_ids,
-            cash_basis_aml_ids.filtered(lambda l: l.account_id == self.tax_waiting_account) + purchase_move.line_ids[5])
+        self.assertTrue(purchase_move_line_ids[5].full_reconcile_id.exists())
+        self.assertEqual(purchase_move_line_ids[5].full_reconcile_id.reconciled_line_ids,
+            cash_basis_aml_ids.filtered(lambda l: l.account_id == self.tax_waiting_account) + purchase_move_line_ids[5])
 
         self.assertEqual(len(cash_basis_aml_ids), 24)
 
@@ -1582,10 +1587,13 @@ def test_reconciliation_cash_basis_fx_01(self):
         to_reconcile.reconcile()
 
         # check reconciliation in Payable account
-        self.assertTrue(purchase_move.line_ids[2].full_reconcile_id.exists())
+        purchase_line_ids = purchase_move.line_ids.sorted()
+        fx_move_01_line_ids = fx_move_01.line_ids.sorted()
+        payment_move_line_ids = payment_move.line_ids.sorted()
+        self.assertTrue(purchase_line_ids[2].full_reconcile_id.exists())
         self.assertEqual(
-            purchase_move.line_ids[2].full_reconcile_id.reconciled_line_ids,
-            purchase_move.line_ids[2] + fx_move_01.line_ids[0] + payment_move.line_ids[0])
+            purchase_line_ids[2].full_reconcile_id.reconciled_line_ids,
+            purchase_line_ids[2] + fx_move_01_line_ids[0] + payment_move_line_ids[0])
 
         # check cash basis
         cash_basis_moves = self.env['account.move'].search(
@@ -1761,11 +1769,15 @@ def test_reconciliation_cash_basis_fx_02(self):
         to_reconcile.reconcile()
 
         # check reconciliation in Payable account
-        self.assertTrue(purchase_move.line_ids[2].full_reconcile_id.exists())
+        purchase_move_line_ids = purchase_move.line_ids.sorted()
+        fx_move_01_line_ids = fx_move_01.line_ids.sorted()
+        fx_move_02_line_ids = fx_move_02.line_ids.sorted()
+        payment_move_line_ids = payment_move.line_ids.sorted()
+        self.assertTrue(purchase_move_line_ids[2].full_reconcile_id.exists())
         self.assertEqual(
-            purchase_move.line_ids[2].full_reconcile_id.reconciled_line_ids,
-            purchase_move.line_ids[2] + fx_move_01.line_ids[0] + fx_move_02.line_ids[0] +
-            payment_move.line_ids[0])
+            purchase_move_line_ids[2].full_reconcile_id.reconciled_line_ids,
+            purchase_move_line_ids[2] + fx_move_01_line_ids[0] + fx_move_02_line_ids[0] +
+            payment_move_line_ids[0])
 
         # check cash basis
         cash_basis_moves = self.env['account.move'].search(
@@ -2059,7 +2071,7 @@ def test_reconciliation_process_move_lines_with_mixed_currencies(self):
             'mv_line_ids': [move_payment_lines[1].id, move_product_lines[1].id],
             'new_mv_line_dicts': [{
                 'account_id': liquidity_account.id,
-                'analytic_tag_ids': [6, None, []],
+                'analytic_tag_ids': [(6, None, [])],
                 'credit': 0,
                 'date': time.strftime('%Y') + '-01-01',
                 'debit': 15.0,

addons/account/tests/test_reconciliation_matching_rules.py

@@ -23,7 +23,7 @@ def _create_invoice_line(self, amount, partner, type):
     def _check_statement_matching(self, rules, expected_values, statements=None):
         if statements is None:
             statements = self.bank_st + self.cash_st
-        statement_lines = statements.mapped('line_ids')
+        statement_lines = statements.mapped('line_ids').sorted()
         matching_values = rules._apply_rules(statement_lines)
         for st_line_id, values in matching_values.items():
             values.pop('reconciled_lines', None)
@@ -56,7 +56,7 @@ def setUp(self):
             'name': 'write-off model',
             'rule_type': 'writeoff_suggestion',
             'match_partner': True,
-            'match_partner_ids': [6, 0, (self.partner_1 + self.partner_2).ids],
+            'match_partner_ids': [],
             'account_id': current_assets_account.id,
         })
 

addons/account/wizard/account_invoice_send.py

@@ -66,6 +66,8 @@ def _compute_invoice_without_email(self):
                         )
                 else:
                     wizard.invoice_without_email = False
+            else:
+                wizard.invoice_without_email = False
 
     def _send_email(self):
         if self.is_email: