[IMP] website : add website visitors activity tracking

This commit adds the website_visitor model that will be used
to track website visitor activity (page viewed, number of visits and
more general info about the visitor (country, lang, etc..)

This model will, in later commit, be used to send chat requests
and push notification from the operators (or backend users)
directly to the visitor.

- A website_visitor is created once the visitor is requesting
a website.page that is tracked.
- A website_visitor is considered as connected if his last tracked
website_page request is within the last 5 minutes.
- The number of visits for a website_visitor is incremented
if his last tracked website_page request was at least 8 hours ago.
- A website_visitor is only handled by the system. Users cannot
create, edit or delete a website_visitor.
- A unique website_visitor is created per website.
That means that the same real person can triggers multiple visitor
creation if visits multiple websites.
This is because, for livechat purpose on later commit, for example,
the chat request can be created on the correct livecaht channel
(linked to the correct website)
- The visitor is recognized via his cookie (visitor_id). So if the visitor
flush his cookies, a new visitor will be created the next time he will
request a tracked website_page.
- Link user's res.partner to website.visitor.
    If a website_visitor log in
    (a visitor that has visitor_id in his cookie),
    the website_visitor is linked to the res.partner.
    The website visitor name is than adapted to match the name of
    the first res.partner linked to the visitor.
    A visitor can have multiple partners as the same session
    can be used by multiple person (one PC for a team for example).

To keep a detailed history of the visitor page views,
we add a website.visitor.page model that makes the link
between visitor and website.page but that keeps the visit date.
So that we can see if a visitor went mulitple times
on the same page and when. It's usefull to see his last page views.

Task ID : 2028059
PR #34624

addons/website/__manifest__.py

@@ -19,12 +19,14 @@
     'installable': True,
     'data': [
         'data/website_data.xml',
+        'data/website_visitor_cron.xml',
         'security/website_security.xml',
         'security/ir.model.access.csv',
         'views/website_templates.xml',
         'views/website_navbar_templates.xml',
         'views/snippets.xml',
         'views/website_views.xml',
+        'views/website_visitor_views.xml',
         'views/res_config_settings_views.xml',
         'views/ir_actions_views.xml',
         'views/ir_attachment_views.xml',

addons/website/data/website_data.xml

@@ -120,16 +120,19 @@
             <field name="is_published">True</field>
             <field name="url">/</field>
             <field name="view_id" ref="homepage"/>
+            <field name="is_tracked">True</field>
         </record>
         <record id="contactus_page" model="website.page">
             <field name="url">/contactus</field>
             <field name="is_published">True</field>
             <field name="view_id" ref="contactus"/>
+            <field name="is_tracked">True</field>
         </record>
         <record id="aboutus_page" model="website.page">
             <field name="is_published">True</field>
             <field name="url">/aboutus</field>
             <field name="view_id" ref="aboutus"/>
+            <field name="is_tracked">True</field>
         </record>
 
         <!-- Default Menu to store module menus for new website -->

addons/website/data/website_visitor_cron.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding='UTF-8'?>
+<odoo>
+    <record id="website_visitor_cron" model="ir.cron">
+        <field name="name">Website Visitor : Archive old visitors</field>
+        <field name="model_id" ref="model_website_visitor"/>
+        <field name="state">code</field>
+        <field name="code">model._cron_archive_visitors()</field>
+        <field name="interval_number">1</field>
+        <field name="interval_type">days</field>
+        <field name="numbercall">-1</field>
+        <field name="active" eval="True"/>
+        <field name="doall" eval="False"/>
+    </record>
+</odoo>

addons/website/models/__init__.py

@@ -19,3 +19,4 @@
 from . import res_users
 from . import res_config_settings
 from . import res_lang
+from . import website_visitor

addons/website/models/ir_http.py

@@ -86,6 +86,20 @@ def _auth_method_public(cls):
         if not request.uid:
             super(Http, cls)._auth_method_public()
 
+    @classmethod
+    def _extract_website_page(cls, response):
+        if getattr(response, 'status_code', 0) != 200:
+            return False
+
+        main_object = getattr(response, 'qcontext', {}).get('main_object')
+        return main_object if getattr(main_object, '_name', False) == 'website.page' else False
+
+    @classmethod
+    def _dispatch(cls):
+        response = super(Http, cls)._dispatch()
+        request.env['website.visitor']._handle_webpage_dispatch(response, cls._extract_website_page(response))
+        return response
+
     @classmethod
     def _add_dispatch_parameters(cls, func):
 

addons/website/models/res_partner.py

@@ -3,7 +3,7 @@
 
 import werkzeug
 
-from odoo import api, models
+from odoo import models, fields
 
 
 def urlplus(url, params):
@@ -14,6 +14,8 @@ class Partner(models.Model):
     _name = 'res.partner'
     _inherit = ['res.partner', 'website.published.multi.mixin']
 
+    visitor_ids = fields.Many2many('website.visitor', 'website_visitor_partner_rel', 'partner_id', 'visitor_id', string='Visitors')
+
     def google_map_img(self, zoom=8, width=298, height=298):
         google_maps_api_key = self.env['website'].get_current_website().google_maps_api_key
         if not google_maps_api_key:

addons/website/models/res_users.py

@@ -53,3 +53,19 @@ def _auto_init(self):
         tools.create_unique_index(self._cr, 'res_users_login_key_unique_website_index',
             self._table, ['login', 'COALESCE(website_id,-1)'])
         return result
+
+    @classmethod
+    def authenticate(cls, db, login, password, user_agent_env):
+        """ Override to link the logged in user's res.partner to website.visitor """
+        uid = super(ResUsers, cls).authenticate(db, login, password, user_agent_env)
+        if uid:
+            with cls.pool.cursor() as cr:
+                env = api.Environment(cr, uid, {})
+                visitor_sudo = env['website.visitor']._get_visitor_from_request()
+                if visitor_sudo:
+                    vals = {
+                        'user_partner_id': env.user.partner_id.id,
+                        'name': env.user.partner_id.name
+                    }
+                    visitor_sudo.write(vals)
+        return uid

addons/website/models/website_page.py

@@ -20,6 +20,7 @@ class Page(models.Model):
     menu_ids = fields.One2many('website.menu', 'page_id', 'Related Menus')
     is_homepage = fields.Boolean(compute='_compute_homepage', inverse='_set_homepage', string='Homepage')
     is_visible = fields.Boolean(compute='_compute_visible', string='Is Visible')
+    is_tracked = fields.Boolean(string='Is Tracked', default=False, help="A tracked page will be included in visitors browsing history.")
 
     # Page options
     header_overlay = fields.Boolean()

addons/website/models/website_visitor.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from datetime import datetime, timedelta
+from hashlib import sha256
+import hmac
+
+from odoo import fields, models, api, _
+from odoo.tools.translate import _format_time_ago
+from odoo.tools.misc import _consteq
+from odoo.http import request
+
+
+class WebsitVisitorPage(models.Model):
+    _name = 'website.visitor.page'
+    _description = 'Visited Pages'
+    _order = 'visit_datetime ASC'
+    _log_access = False
+
+    visitor_id = fields.Many2one('website.visitor', ondelete="cascade", index=True, required=True, readonly=True)
+    page_id = fields.Many2one('website.page', index=True, ondelete='cascade', readonly=True)
+    visit_datetime = fields.Datetime('Visit Date', default=fields.Datetime.now, required=True, readonly=True)
+
+
+class WebsiteVisitor(models.Model):
+    _name = 'website.visitor'
+    _description = 'Website Visitor'
+    _order = 'last_connection_datetime DESC'
+
+    name = fields.Char('Name', default=_('Website Visitor'))
+    active = fields.Boolean('Active', default=True)
+    website_id = fields.Many2one('website', "Website", readonly=True)
+    user_partner_id = fields.Many2one('res.partner', string="Linked Partner", help="Partner of the last logged in user.")
+    create_date = fields.Datetime('First connection date', readonly=True)
+    last_connection_datetime = fields.Datetime('Last Connection', help="Last page view date", readonly=True)
+    country_id = fields.Many2one('res.country', 'Country', readonly=True)
+    country_flag = fields.Binary(related="country_id.image", string="Country Flag")
+    lang_id = fields.Many2one('res.lang', string='Language', help="Language from the website when visitor has been created")
+    visit_count = fields.Integer('Number of visits', default=1, readonly=True, help="A new visit is considered if last connection was more than 8 hours ago.")
+    visitor_page_ids = fields.One2many('website.visitor.page', 'visitor_id', string='Visited Pages History', readonly=True)
+    visitor_page_count = fields.Integer('Page Views', compute="_compute_page_statistics")
+    page_ids = fields.Many2many('website.page', string="Visited Pages", compute="_compute_page_statistics", store=True)
+    page_count = fields.Integer('# Visited Pages', compute="_compute_page_statistics")
+    time_since_last_action = fields.Char('Last action', compute="_compute_time_statistics", help='Time since last page view. E.g.: 2 minutes ago')
+    is_connected = fields.Boolean('Is connected ?', compute='_compute_time_statistics', help='A visitor is considered as connected if his last page view was within the last 5 minutes.')
+
+    @api.depends('visitor_page_ids')
+    def _compute_page_statistics(self):
+        results = self.env['website.visitor.page'].read_group(
+            [('visitor_id', 'in', self.ids)], ['visitor_id', 'page_id'], ['visitor_id', 'page_id'], lazy=False)
+        mapped_data = {}
+        for result in results:
+            visitor_info = mapped_data.get(result['visitor_id'][0], {'page_count': 0, 'page_ids': set()})
+            visitor_info['page_count'] += result['__count']
+            visitor_info['page_ids'].add(result['page_id'][0])
+            mapped_data[result['visitor_id'][0]] = visitor_info
+
+        for visitor in self:
+            visitor_info = mapped_data.get(visitor.id, {'page_ids': [], 'page_count': 0})
+
+            visitor.page_ids = [(6, 0, visitor_info['page_ids'])]
+            visitor.visitor_page_count = visitor_info['page_count']
+            visitor.page_count = len(visitor_info['page_ids'])
+
+    @api.depends('last_connection_datetime')
+    def _compute_time_statistics(self):
+        results = self.env['website.visitor'].search_read([('id', 'in', self.ids)], ['id', 'last_connection_datetime'])
+        mapped_data = {result['id']: result['last_connection_datetime'] for result in results}
+
+        for visitor in self:
+            last_connection_date = mapped_data[visitor.id]
+            visitor.time_since_last_action = _format_time_ago(self.env, (datetime.now() - last_connection_date))
+            visitor.is_connected = (datetime.now() - last_connection_date) < timedelta(minutes=5)
+
+    def _get_visitor_sign(self):
+        return {visitor.id: "%d-%s" % (visitor.id, self._get_visitor_hash(visitor.id)) for visitor in self}
+
+    @api.model
+    def _get_visitor_hash(self, visitor_id):
+        db_secret = request.env['ir.config_parameter'].sudo().get_param('database.secret')
+        return hmac.new(str(visitor_id).encode('utf-8'), db_secret.encode('utf-8'), sha256).hexdigest()
+
+    def _get_visitor_from_request(self):
+        if not request:
+            return None
+        visitor = self.env['website.visitor']
+        cookie_content = request.httprequest.cookies.get('visitor_id')
+        if cookie_content and '-' in cookie_content:
+            visitor_id, visitor_hash = cookie_content.split('-', 1)
+            if _consteq(visitor_hash, self._get_visitor_hash(visitor_id)):
+                return visitor.sudo().with_context(active_test=False).search([('id', '=', visitor_id)])  # search to avoid having to call exists()
+        return visitor
+
+    def _handle_webpage_dispatch(self, response, website_page):
+        if website_page:
+            # get visitor only if page tracked. Done here to avoid having to do it multiple times in case of override.
+            visitor_sudo = self._get_visitor_from_request() if website_page.is_tracked else False
+            self._handle_website_page_visit(response, website_page, visitor_sudo)
+
+    def _handle_website_page_visit(self, response, website_page, visitor_sudo):
+        """ Called on dispatch. This will create a website.visitor if the http request object
+        is a tracked website page. Only on tracked page to avoid having too much operations done on every page
+        or other http requests.
+        Note: The side effect is that the last_connection_datetime is updated ONLY on tracked pages."""
+        if website_page.is_tracked:
+            if not visitor_sudo:
+                # If visitor does not exist
+                visitor_sudo = self._create_visitor(website_page.id)
+                sign = visitor_sudo._get_visitor_sign().get(visitor_sudo.id)
+                response.set_cookie('visitor_id', sign)
+            else:
+                # Add page even if already in visitor_page_ids as checks on relations are done in many2many write method
+                vals = {
+                    'last_connection_datetime': datetime.now(),
+                    'visitor_page_ids': [(0, 0, {'page_id': website_page.id, 'visit_datetime': datetime.now()})],
+                }
+                if visitor_sudo.last_connection_datetime < (datetime.now() - timedelta(hours=8)):
+                    vals['visit_count'] = visitor_sudo.visit_count + 1
+                if not visitor_sudo.active:
+                    vals['active'] = True
+                visitor_sudo.write(vals)
+
+    def _create_visitor(self, website_page_id=False):
+        country_code = request.session.get('geoip', {}).get('country_code', False)
+        country_id = request.env['res.country'].sudo().search([('code', '=', country_code)], limit=1).id if country_code else False
+        lang_id = request.env['res.lang'].sudo().search([('code', '=', request.lang)], limit=1).id
+        vals = {
+            'last_connection_datetime': datetime.now(),
+            'lang_id': lang_id,
+            'country_id': country_id,
+            'website_id': request.website.id
+        }
+        if not self.env.user._is_public():
+            vals['user_partner_id'] = self.env.user.partner_id.id
+        if website_page_id:
+            vals['visitor_page_ids'] = [(0, 0, {'page_id': website_page_id, 'visit_datetime': datetime.now()})]
+        # Set signed visitor id in cookie
+        return self.sudo().create(vals)
+
+    def _cron_archive_visitors(self):
+        one_week_ago = datetime.now() - timedelta(days=7)
+        visitors_to_archive = self.env['website.visitor'].sudo().search([('last_connection_datetime', '<', one_week_ago)])
+        visitors_to_archive.write({'active': False})

addons/website/security/ir.model.access.csv

@@ -12,3 +12,7 @@ access_website_ir_ui_view,access_website_ir_ui_view,model_ir_ui_view,group_websi
 access_seo_public,access_seo_public,model_website_seo_metadata,,1,0,0,0
 access_seo_manager,access_seo_manager,model_website_seo_metadata,group_website_designer,1,1,1,1
 access_seo_designer,access_seo_designer,model_website_seo_metadata,group_website_designer,1,1,1,1
+access_website_visitor_designer,access_website_visitor_designer,model_website_visitor,website.group_website_designer,1,0,0,0
+access_website_visitor_system,access_website_visitor_system,model_website_visitor,base.group_system,1,1,1,1
+access_website_visitor_page_designer,access_website_visitor_page_designer,model_website_visitor_page,website.group_website_designer,1,0,0,0
+access_website_visitor_page_system,access_website_visitor_page_system,model_website_visitor_page,base.group_system,1,1,1,1

addons/website/static/src/scss/website.backend.scss

@@ -216,3 +216,12 @@
         }
     }
 }
+
+.oe_stat_button {
+    &.o_stat_button_info:hover {
+        color: #666666 !important;
+        background-color: transparent !important;
+        opacity: 0.8 !important;
+        cursor: default !important;
+    }
+}

addons/website/tests/__init__.py

@@ -12,3 +12,4 @@
 from . import test_page
 from . import test_website_favicon
 from . import test_website_reset_password
+from . import test_website_visitor

addons/website/tests/test_website_visitor.py

@@ -0,0 +1,38 @@
+# coding: utf-8
+from odoo.tests import HttpCase, tagged
+
+@tagged('dbetest')
+class WebsiteVisitorTests(HttpCase):
+    def test_create_visitor_on_tracked_page(self):
+        Page = self.env['website.page']
+        View = self.env['ir.ui.view']
+        Visitor = self.env['website.visitor']
+        base_view = View.create({
+            'name': 'Base',
+            'type': 'qweb',
+            'arch': '''<t name="Homepage" t-name="website.base_view">
+                        <t t-call="website.layout">
+                            I am a generic page
+                        </t>
+                    </t>''',
+            'key': 'test.base_view',
+        })
+        [untracked_page, tracked_page] = Page.create([
+            {
+                'view_id': base_view.id,
+                'url': '/untracked_page_1',
+                'website_published': True
+            },
+            {
+                'view_id': base_view.id,
+                'url': '/tracked_page_1',
+                'website_published': True,
+                'is_tracked': True
+            }
+        ])
+
+        self.assertEqual(len(Visitor.search([])), 0, "No visitor at the moment")
+        self.url_open(untracked_page.url)
+        self.assertEqual(len(Visitor.search([])), 0, "No visitor created after visiting an untracked page")
+        self.url_open(tracked_page.url)
+        self.assertEqual(len(Visitor.search([])), 1, "A visitor should be created after visiting a tracked page")

addons/website/views/website_views.xml

@@ -147,6 +147,7 @@
                                 <field name="url"/>
                                 <field name="view_id" context="{'display_website': True}"/>
                                 <field name="website_id" options="{'no_create': True}" groups="website.group_multi_website"/>
+                                <field name="is_tracked"/>
                             </group>
                             <group>
                                 <field name="website_indexed"/>
@@ -174,10 +175,25 @@
                     <field name="create_uid" invisible="1"/>
                     <field name="write_uid"/>
                     <field name="write_date"/>
+                    <field name="is_tracked"/>
                 </tree>
             </field>
         </record>
 
+        <record id="website_pages_view_search" model="ir.ui.view">
+            <field name="name">website.page.view.search</field>
+            <field name="model">website.page</field>
+            <field name="arch" type="xml">
+                <search string="Website Pages" >
+                    <filter string="Published" name="published" domain="[('website_published', '=', True)]"/>
+                    <filter string="Not published" name="not_published" domain="[('website_published', '=', False)]"/>
+                    <separator/>
+                    <filter string="Tracked" name="tracked" domain="[('is_tracked', '=', True)]"/>
+                    <filter string="Not tracked" name="not_tracked" domain="[('is_tracked', '=', False)]"/>
+                </search>
+            </field>
+        </record>
+
         <record id="action_website_pages_list" model="ir.actions.act_window">
             <field name="name">Website Pages</field>
             <field name="res_model">website.page</field>