[16.0][ADD] Connector typesense

connector_search_engine/models/se_index.py

@@ -219,6 +219,8 @@ def _jobify_batch_sync(self, force_export: bool = False) -> None:
     def _jobify_batch_recompute(self, force_export: bool = False) -> None:
         self.ensure_one()
         description = _("Prepare a batch recompute of index '%s'") % self.name
+        print(">>> run _jobify_batch_recompute")
+        # breakpoint()
         self.with_delay(
             description=description, identity_key=identity_exact
         ).batch_recompute(force_export)
@@ -231,9 +233,12 @@ def generate_batch_sync_per_index(self, domain: list | None = None) -> None:
         index to sync. The sync will export the bindings marked as to_export.
         and will delete the bindings marked as to_delete.
         """
+        print(">>>>>  run generate_batch_sync_per_index")
         if domain is None:
             domain = []
         for record in self.search(domain):
+            print(domain)
+            print(record)
             record._jobify_batch_sync()
 
     @api.model
@@ -244,9 +249,12 @@ def generate_batch_recompute_per_index(self, domain: list | None = None) -> None
         index to recompute. The recompute process will recompute the bindings
         marked as to_recompute.
         """
+        print(">>>>>  run generate_batch_recompute_per_index")
         if domain is None:
             domain = []
         for record in self.search(domain):
+            print(domain)
+            print(record)
             record._jobify_batch_recompute()
 
     def _get_domain_for_recomputing_binding(self, force_export: bool = False) -> list:
@@ -259,6 +267,7 @@ def _get_domain_for_recomputing_binding(self, force_export: bool = False) -> lis
     def batch_recompute(self, force_export: bool = False) -> None:
         """Recompute all the bindings of the index marked as to_recompute."""
         self.ensure_one()
+        print("###### >>> run batch_recompute")
         domain = self._get_domain_for_recomputing_binding(force_export)
         bindings = self.env["se.binding"].search(domain)
         bindings_count = len(bindings)
@@ -272,6 +281,7 @@ def batch_recompute(self, force_export: bool = False) -> None:
                 "index_name": self.name,
             }
             batch.write({"state": "recomputing"})
+            print(description)
             batch.with_delay(description=description).recompute_json()
 
     def _get_domain_for_exporting_binding(self, force_export: bool = False):

connector_typesense/README.rst

@@ -0,0 +1,252 @@
+===================
+connector_typesense
+===================
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:f09633c3af59b153f0eba3f876a9835676c430a6f39dd0dfa3545a924c80bc11
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Beta
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fsearch--engine-lightgray.png?logo=github
+    :target: https://github.com/OCA/search-engine/tree/16.0/connector_typesense
+    :alt: OCA/search-engine
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/search-engine-16-0/search-engine-16-0-connector_typesense
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/search-engine&target_branch=16.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+Base module for connecting Odoo with external search engines. This addon is
+intended to be used as a base for other addons that implement specific search
+engines. It's designed to be easily extensible and modular.
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Installation
+============
+
+This addon uses the native json python package provided by python. When
+a json for a record is recomputed, the new value is compared to the original
+one to see if an export to the search engine index is needed.  This is
+done by comparing the md5 of the two json strings. This process when done on
+a large number of records can be slow when the json is large and complex. To speed
+up this process you can install the orjson package.
+
+.. code-block:: bash
+
+    pip install orjson
+
+This package requires a typesense search engine running.
+Please read this for a [quick docker based setup](https://typesense.org/docs/guide/install-typesense.html#option-2-local-machine-self-hosting).
+
+Usage
+=====
+
+Overview
+~~~~~~~~
+
+A search engine is a system designed to store information in a way that makes
+it easy to find through search and analytics queries. The main difference
+between a search engine and a database is that a search engine is optimized
+for search and analytics queries, while a database is optimized for
+transactional and relational queries.
+
+This addons is designed around 4 main concepts:
+
+* **The search engine backend** is used to define into Odoo the kind
+  of search engine that will be used to index the data. It's main responsibility
+  is to provide an instance of `odoo.addons.search_engine.tools.adapter.SearchEngineAdapter`
+  that will be used to communicate with the search engine.
+
+* **The search engine index** is used to define into Odoo the index where
+  the data will be indexed. An index is always linked to a search engine backend.
+  The index provides methods to use to manage the lifecycle of the data put into
+  the index for the records of a given model. To do so, it uses:
+
+  * **The SearchEngineAdapter** provided by the backend to communicate with the
+    search engine.
+  * **A ModelSerializer** that is used to transform an odoo record into
+    a dictionary that can be indexed into the search engine.
+  * **A JsonValidator** that is used to validate the data that is to be
+    indexed into the search engine.
+
+  The RecordSerializer and IndexDataValidator are defined on the index itself.
+  The current addon provides a default implementation only for the IndexDataValidator.
+  You can find into the github repository `search-engine <https://github.com:
+  OCA/search-engine/tree/16.0>`_ An implementation of the RecordSerializer based
+  on the jsonifier addon `connector_search_engine_jsonifier`.
+
+* **The search engine indexable record** is a mixin that is used to define
+  the records that can be indexed into a search engine index. The mixin
+  provides methods:
+
+  * To add a record to an index.
+  * To remove a record from an index.
+  * To mark the record into an index (*the search engine bindings*) as to be
+    recomputed (This method should be called when modifications are made on
+    the record that could impact the data that are indexed into the search
+    engine. It will instruct the index that the record must be recomputed and
+    re-indexed).
+
+  It also ensures that when the record is unlinked, it is removed from the indexes
+  it was indexed into.
+
+* **The search engine binding** is a model that represents the link between
+  an index and an indexable odoo record. It give you access to the data
+  that are indexed into the search engine for the record. It's also used to
+  manage the lifecycle of the data into the search engine. When a binding is
+  created, it's marked as to be computed. Once the data are computed, the
+  binding is marked as to be indexed. Once the data are indexed, the binding
+  is marked as indexed. If the linked record is unlinked, the binding is
+  marked as to be removed. Once the data are removed from the search engine,
+  the binding is deleted.
+
+Indexing lifecycle
+~~~~~~~~~~~~~~~~~~
+
+The indexing lifecycle is based on the following steps:
+
+* When a record is added to an index, a binding is created and marked as to be
+  computed.
+* A cron job scheduled every 5 minutes will look for bindings that are to be
+  computed and for each of them will schedule a job to re compute the json data.
+* When the json data is computed, the binding is marked as to be exported if the
+  json is valid and is different from the one that has been computed last time.
+* A cron job scheduled every 5 minutes will ensure the syncing with the search
+  engine. It will:
+
+  * look for bindings that are to be exported and for each of them will schedule
+    a job to export the json data into the search engine. Once exported, the
+    binding is marked as 'done'.
+  * look for bindings that are to be removed and for each of them will schedule
+    a job to remove the data from the search engine. Once removed, the binding
+    is deleted.
+
+To keep in sync the data from your model instance and the data that are indexed
+into the search engine, you should call the method `_se_mark_to_update` on the
+mode instance when you make modifications that could impact the data that are
+indexed into the search engine.
+
+* When the method `_se_mark_to_update` is called, the binding is marked as to be
+  computed.
+* From there, the same process as described above will be used to recompute the
+  data and reindex them into the search engine.
+
+When a model instance is unlinked, the binding is marked as to be removed. From
+there if will be processed by the job syncing the data with the search engine.
+
+.. note::
+
+  In previous versions of this addon, there was no method to mark a record as
+  to be recomputed. As a consequence, all the records were re-computed every day
+  to ensure that the data in the search engine were up to date. This was a
+  performance issue and consumed a lot of resources. If despite this, you want
+  to recompute all the records every day, you can activate the cron jon
+  `Search engine: recompute all index` and deactivate the one named
+  `earch engine: Generate job for recompute binding to recompute per index`.
+
+Known issues / Roadmap
+======================
+
+* Implement generic trigger for binding
+  based on ir.export linked to the index
+  (the aim is to set the binding to be updated
+  if we modify a field configured in the exporter)
+
+Changelog
+=========
+
+16.0.0.1.1 (2023-10-13)
+~~~~~~~~~~~~~~~~~~~~~~~
+
+**Bugfixes**
+
+- Fixes cache issue with the *se_binding_ids* field on the *s.indexable.record*
+  model. When a binding is created or updated or deleted, the cache for the
+  *se_binding_ids* field for referenced records is now invalidated. That way,
+  the next time the field is accessed after such an operation, the value is
+  recomputed to reflect the change. (`#163 <https://github.com/OCA/search-engine/issues/163>`_)
+
+
+16.0.0.1.0 (2023-10-13)
+~~~~~~~~~~~~~~~~~~~~~~~
+
+**Features**
+
+- A new action **Update state** is now available on *Search Engine Record* objects.
+  This action allows you to update the state of selected records on the tree view.
+
+  Add a smart button to quickly access to the bound records from the
+  *Search Engine Backend* and *Search Engine Record* views. (`#162 <https://github.com/OCA/search-engine/issues/162>`__)
+
+
+**Bugfixes**
+
+- Fix Search Engine Binding form view. The fields data and error are now
+  properly displayed and fit the width of the form.
+
+  Makes the Odoo's admin user a member of the *Search Engine Connector Manager* group. (`#162 <https://github.com/OCA/search-engine/issues/162>`__)
+
+
+12.0.x.y.z (YYYY-MM-DD)
+~~~~~~~~~~~~~~~~~~~~~~~
+
+TODO
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/search-engine/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/search-engine/issues/new?body=module:%20connector_typesense%0Aversion:%2016.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+~~~~~~~
+
+* Derico
+
+Contributors
+~~~~~~~~~~~~
+
+* Sébastien BEAU <sebastien.beau@akretion.com>
+* Laurent Mignon <laurent.mignon@acsone.eu>
+* Simone Orsi <simone.orsi@camptocamp.com>
+* Raphaël Reverdy <raphael.reverdy@akretion.com>
+
+Maintainers
+~~~~~~~~~~~
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+This module is part of the `OCA/search-engine <https://github.com/OCA/search-engine/tree/16.0/connector_typesense>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

connector_typesense/__init__.py

@@ -0,0 +1,2 @@
+from . import models
+from . import tools

connector_typesense/__manifest__.py

@@ -0,0 +1,20 @@
+# Copyright 2024 Derico
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+    "name": "connector_typesense",
+    "category": "Connector",
+    "summary": "Connector For Typesense Search Engine",
+    "version": "16.0.0.0.2",
+    "license": "AGPL-3",
+    "author": "Derico, Odoo Community Association (OCA)",
+    "website": "https://github.com/OCA/search-engine",
+    "depends": ["connector_search_engine"],
+    "data": [
+        "views/se_backend.xml",
+    ],
+    # "demo": ["demo/backend_demo.xml"],
+    # TODO: Get latest improvements from elasticsearch library
+    "external_dependencies": {"python": ["typesense", "requests"]},
+    "installable": True,
+}

connector_typesense/models/__init__.py

@@ -0,0 +1 @@
+from . import se_backend, se_index

connector_typesense/models/se_backend.py

@@ -0,0 +1,48 @@
+# Copyright 2024 Derico
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+from odoo import fields, models
+
+from ..tools.adapter import TypesenseAdapter
+
+
+class SeBackend(models.Model):
+    _inherit = "se.backend"
+
+    backend_type = fields.Selection(
+        selection_add=[("typesense", "Typesense")],
+        ondelete={"typesense": "cascade"},
+        string="Type",
+        required=True,
+    )
+    ts_server_host = fields.Char(
+        string="Typesense host",
+        groups="connector_search_engine.group_connector_search_engine_manager",
+    )
+    ts_server_port = fields.Char(
+        string="Typesense port",
+        groups="connector_search_engine.group_connector_search_engine_manager",
+    )
+    ts_server_protocol = fields.Char(
+        string="Typesense protocol",
+        groups="connector_search_engine.group_connector_search_engine_manager",
+    )
+    ts_server_timeout = fields.Integer(
+        string="Typesense server timeout",
+        groups="connector_search_engine.group_connector_search_engine_manager",
+    )
+    api_key_id = fields.Char(
+        help="Typesense Api Key ID",
+        string="Api Key ID",
+        groups="connector_search_engine.group_connector_search_engine_manager",
+    )
+    api_key = fields.Char(
+        help="Typesense Api Key",
+        groups="connector_search_engine.group_connector_search_engine_manager",
+    )
+
+    def _get_adapter_class(self):
+        if self.backend_type == "typesense":
+            return TypesenseAdapter
+        else:
+            return super()._get_adapter_class()

connector_typesense/models/se_index.py

@@ -0,0 +1,10 @@
+# Copyright 2024 Derico
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+from odoo import _, api, models
+from odoo.exceptions import ValidationError
+
+
+class SeIndex(models.Model):
+
+    _inherit = "se.index"