feat: Sentiment Analysis (#7475)

.env.example

@@ -230,3 +230,6 @@ AZURE_APP_SECRET=
 ## Change these values to fine tune performance
 # control the concurrency setting of sidekiq
 # SIDEKIQ_CONCURRENCY=10
+
+# Sentiment analysis model file path
+SENTIMENT_FILE_PATH=

Gemfile

@@ -165,6 +165,9 @@ gem 'omniauth'
 gem 'omniauth-google-oauth2'
 gem 'omniauth-rails_csrf_protection', '~> 1.0'
 
+# Sentiment analysis
+gem 'informers'
+
 ### Gems required only in specific deployment environments ###
 ##############################################################
 

Gemfile.lock

@@ -145,6 +145,7 @@ GEM
       statsd-ruby (~> 1.1)
     bcrypt (3.1.18)
     bindex (0.8.1)
+    blingfire (0.1.8)
     bootsnap (1.16.0)
       msgpack (~> 1.2)
     brakeman (5.4.1)
@@ -361,6 +362,10 @@ GEM
     image_processing (1.12.2)
       mini_magick (>= 4.9.5, < 5)
       ruby-vips (>= 2.0.17, < 3)
+    informers (0.2.0)
+      blingfire (>= 0.1.7)
+      numo-narray
+      onnxruntime (>= 0.5.1)
     jbuilder (2.11.5)
       actionview (>= 5.0.0)
       activesupport (>= 5.0.0)
@@ -474,6 +479,7 @@ GEM
       racc (~> 1.4)
     nokogiri (1.15.2-x86_64-linux)
       racc (~> 1.4)
+    numo-narray (0.9.2.1)
     oauth (1.1.0)
       oauth-tty (~> 1.0, >= 1.0.1)
       snaky_hash (~> 2.0)
@@ -502,6 +508,14 @@ GEM
     omniauth-rails_csrf_protection (1.0.1)
       actionpack (>= 4.2)
       omniauth (~> 2.0)
+    onnxruntime (0.7.6)
+      ffi
+    onnxruntime (0.7.6-arm64-darwin)
+      ffi
+    onnxruntime (0.7.6-x86_64-darwin)
+      ffi
+    onnxruntime (0.7.6-x86_64-linux)
+      ffi
     openssl (3.1.0)
     orm_adapter (0.5.0)
     os (1.1.4)
@@ -846,6 +860,7 @@ DEPENDENCIES
   hashie
   html2text!
   image_processing
+  informers
   jbuilder
   json_refs
   json_schemer

app/models/conversation.rb

@@ -304,3 +304,4 @@ def validate_referer_url
 end
 
 Conversation.include_mod_with('EnterpriseConversationConcern')
+Conversation.include_mod_with('SentimentAnalysisHelper')

app/models/message.rb

@@ -12,6 +12,7 @@
 #  private                   :boolean          default(FALSE)
 #  processed_message_content :text
 #  sender_type               :string
+#  sentiment                 :jsonb
 #  status                    :integer          default("sent")
 #  created_at                :datetime         not null
 #  updated_at                :datetime         not null
@@ -246,6 +247,7 @@ def execute_after_create_commit_callbacks
     reopen_conversation
     notify_via_mail
     set_conversation_activity
+    update_message_sentiments
     dispatch_create_events
     send_reply
     execute_message_template_hooks
@@ -371,4 +373,10 @@ def set_conversation_activity
     conversation.update_columns(last_activity_at: created_at)
     # rubocop:enable Rails/SkipsModelValidations
   end
+
+  def update_message_sentiments
+    # override in the enterprise ::Enterprise::SentimentAnalysisJob.perform_later(self)
+  end
 end
+
+Message.prepend_mod_with('Message')

db/migrate/20230706090122_sentiment_column_to_messages.rb

@@ -0,0 +1,5 @@
+class SentimentColumnToMessages < ActiveRecord::Migration[7.0]
+  def change
+    add_column :messages, :sentiment, :jsonb, default: {}
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_06_20_212340) do
+ActiveRecord::Schema[7.0].define(version: 2023_07_06_090122) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_stat_statements"
   enable_extension "pg_trgm"
@@ -669,6 +669,7 @@
     t.jsonb "external_source_ids", default: {}
     t.jsonb "additional_attributes", default: {}
     t.text "processed_message_content"
+    t.jsonb "sentiment", default: {}
     t.index "((additional_attributes -> 'campaign_id'::text))", name: "index_messages_on_additional_attributes_campaign_id", using: :gin
     t.index ["account_id", "inbox_id"], name: "index_messages_on_account_id_and_inbox_id"
     t.index ["account_id"], name: "index_messages_on_account_id"

enterprise/app/jobs/enterprise/sentiment_analysis_job.rb

@@ -0,0 +1,39 @@
+class Enterprise::SentimentAnalysisJob < ApplicationJob
+  queue_as :default
+
+  def perform(message)
+    return if message.account.locale != 'en'
+    return if valid_incoming_message?(message)
+
+    save_message_sentiment(message)
+  rescue StandardError => e
+    Rails.logger.error("Sentiment Analysis Error for message #{message.id}: #{e}")
+    ChatwootExceptionTracker.new(e, account: message.account).capture_exception
+  end
+
+  def save_message_sentiment(message)
+    # We are truncating the data here to avoind the OnnxRuntime::Error
+    # Indices element out of data bounds, idx=512 must be within the inclusive range [-512,511]
+    # While gathering the maningfull node the Array/tensor index is going out of bound
+
+    text = message.content&.truncate(2900)
+    sentiment = model.predict(text)
+    message.sentiment = sentiment.merge(value: label_val(sentiment))
+
+    message.save!
+  end
+
+  # Model initializes OnnxRuntime::Model, with given file for inference session and to create the tensor
+  def model
+    model_path = ENV.fetch('SENTIMENT_FILE_PATH', nil)
+    Informers::SentimentAnalysis.new(model_path) if model_path.present?
+  end
+
+  def label_val(sentiment)
+    sentiment[:label] == 'positive' ? 1 : -1
+  end
+
+  def valid_incoming_message?(message)
+    !message.incoming? || message.private?
+  end
+end

enterprise/app/models/enterprise/message.rb

@@ -0,0 +1,5 @@
+module Enterprise::Message
+  def update_message_sentiments
+    ::Enterprise::SentimentAnalysisJob.perform_later(self)
+  end
+end

enterprise/app/models/enterprise/sentiment_analysis_helper.rb

@@ -0,0 +1,45 @@
+module Enterprise::SentimentAnalysisHelper
+  extend ActiveSupport::Concern
+
+  included do
+    def opening_sentiments
+      records = incoming_messages.first(average_message_count)
+      average_sentiment(records)
+    end
+
+    def closing_sentiments
+      return unless resolved?
+
+      records = incoming_messages.last(average_message_count)
+      average_sentiment(records)
+    end
+
+    def average_sentiment(records)
+      {
+        label: average_sentiment_label(records),
+        score: average_sentiment_score(records)
+      }
+    end
+
+    private
+
+    def average_sentiment_label(records)
+      value = records.pluck(:sentiment).sum { |a| a['value'].to_i }
+      value.negative? ? 'negative' : 'positive'
+    end
+
+    def average_sentiment_score(records)
+      total = records.pluck(:sentiment).sum { |a| a['score'].to_f }
+      total / average_message_count
+    end
+
+    def average_message_count
+      # incoming_messages.count >= 10 ? 5 : ((incoming_messages.count / 2) - 1)
+      5
+    end
+
+    def incoming_messages
+      messages.incoming.where(private: false)
+    end
+  end
+end

spec/enterprise/jobs/enterprise/sentiment_analysis_job_spec.rb

@@ -0,0 +1,82 @@
+require 'rails_helper'
+
+RSpec.describe Enterprise::SentimentAnalysisJob do
+  context 'when account locale set to english language' do
+    let(:account) { create(:account, locale: 'en') }
+    let(:message) { build(:message, content_type: nil, account: account) }
+
+    context 'when update the message sentiments' do
+      let(:model_path) { 'sentiment-analysis.onnx' }
+      let(:model) { double }
+
+      before do
+        allow(Informers::SentimentAnalysis).to receive(:new).with(model_path).and_return(model)
+        allow(model).to receive(:predict).and_return({ label: 'positive', score: '0.6' })
+      end
+
+      it 'with incoming message' do
+        with_modified_env SENTIMENT_FILE_PATH: 'sentiment-analysis.onnx' do
+          message.update(message_type: :incoming)
+
+          described_class.perform_now(message)
+
+          expect(message.sentiment).not_to be_empty
+        end
+      end
+
+      it 'update sentiment label for positive message' do
+        with_modified_env SENTIMENT_FILE_PATH: 'sentiment-analysis.onnx' do
+          message.update(message_type: :incoming, content: 'I like your product')
+
+          described_class.perform_now(message)
+
+          expect(message.sentiment).not_to be_empty
+          expect(message.sentiment['label']).to eq('positive')
+          expect(message.sentiment['value']).to eq(1)
+        end
+      end
+
+      it 'update sentiment label for negative message' do
+        with_modified_env SENTIMENT_FILE_PATH: 'sentiment-analysis.onnx' do
+          message.update(message_type: :incoming, content: 'I did not like your product')
+          allow(model).to receive(:predict).and_return({ label: 'negative', score: '0.6' })
+
+          described_class.perform_now(message)
+
+          expect(message.sentiment).not_to be_empty
+          expect(message.sentiment['label']).to eq('negative')
+          expect(message.sentiment['value']).to eq(-1)
+        end
+      end
+    end
+
+    context 'when does not update the message sentiments' do
+      it 'with outgoing message' do
+        message.update(message_type: :outgoing)
+
+        described_class.perform_now(message)
+
+        expect(message.sentiment).to be_empty
+      end
+
+      it 'with private message' do
+        message.update(private: true)
+
+        described_class.perform_now(message)
+
+        expect(message.sentiment).to be_empty
+      end
+    end
+  end
+
+  context 'when account locale is not set to english language' do
+    let(:account) { create(:account, locale: 'es') }
+    let(:message) { build(:message, content_type: nil, account: account) }
+
+    it 'does not update the message sentiments' do
+      described_class.perform_now(message)
+
+      expect(message.sentiment).to be_empty
+    end
+  end
+end

spec/enterprise/models/conversation_spec.rb

@@ -4,4 +4,34 @@
   describe 'associations' do
     it { is_expected.to belong_to(:sla_policy).optional }
   end
+
+  describe 'conversation sentiments' do
+    include ActiveJob::TestHelper
+
+    let(:conversation) { create(:conversation, additional_attributes: { referer: 'https://www.chatwoot.com/' }) }
+
+    before do
+      10.times do
+        message = create(:message, conversation_id: conversation.id, account_id: conversation.account_id, message_type: 'incoming')
+        message.update(sentiment: { 'label': 'positive', score: '0.4' })
+      end
+    end
+
+    it 'returns opening sentiments' do
+      sentiments = conversation.opening_sentiments
+      expect(sentiments[:label]).to eq('positive')
+    end
+
+    it 'returns closing sentiments if conversation is not resolved' do
+      sentiments = conversation.closing_sentiments
+      expect(sentiments).to be_nil
+    end
+
+    it 'returns closing sentiments if it is resolved' do
+      conversation.resolved!
+
+      sentiments = conversation.closing_sentiments
+      expect(sentiments[:label]).to eq('positive')
+    end
+  end
 end

spec/enterprise/models/message_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require Rails.root.join 'spec/models/concerns/liquidable_shared.rb'
+
+RSpec.describe Message do
+  context 'with sentiment analysis' do
+    let(:message) { build(:message, message_type: :incoming, content_type: nil, account: create(:account)) }
+
+    it 'calls SentimentAnalysisJob' do
+      allow(Enterprise::SentimentAnalysisJob).to receive(:perform_later).and_return(:perform_later).with(message)
+
+      message.save!
+
+      expect(Enterprise::SentimentAnalysisJob).to have_received(:perform_later)
+    end
+  end
+end