chore: Refactor Response Bot Data Schema (#8011)

This PR refactors the schema we introduced in #7518 based on the feedback from production tests. Here is the change log

- Decouple Inbox association to a new table inbox_response_sources -> this lets us share the same response source between multiple inboxes
- Add a status field to responses. This ensures that, by default, responses are created in pending status. You can do quality assurance before making them active. In future, this status can be leveraged by the bot to auto-generate response questions from conversations which require a handoff
- Add response_source association to responses and remove hard dependency from response_documents. This lets users write free-form question answers based on conversations, which doesn't necessarily need a response source.

app/views/api/v1/models/_response_source.json.jbuilder

@@ -2,7 +2,6 @@ json.id resource.id
 json.name resource.name
 json.source_link resource.source_link
 json.source_type resource.source_type
-json.inbox_id resource.inbox_id
 json.account_id resource.account_id
 json.created_at resource.created_at.to_i
 json.updated_at resource.updated_at.to_i

config/initializers/monkey_patches/schema_dumper.rb

@@ -33,3 +33,4 @@ def extensions(stream)
 ActiveRecord::SchemaDumper.ignore_tables << 'responses'
 ActiveRecord::SchemaDumper.ignore_tables << 'response_sources'
 ActiveRecord::SchemaDumper.ignore_tables << 'response_documents'
+ActiveRecord::SchemaDumper.ignore_tables << 'inbox_response_sources'

enterprise/app/controllers/api/v1/accounts/response_sources_controller.rb

@@ -28,7 +28,7 @@ def find_response_source
   end
 
   def response_source_params
-    params.require(:response_source).permit(:name, :source_link, :inbox_id,
+    params.require(:response_source).permit(:name, :source_link,
                                             response_documents_attributes: [:document_link])
   end
 end

enterprise/app/dashboards/response_dashboard.rb

@@ -10,8 +10,13 @@ class ResponseDashboard < Administrate::BaseDashboard
   ATTRIBUTE_TYPES = {
     id: Field::Number.with_options(searchable: true),
     account: Field::BelongsToSearch.with_options(class_name: 'Account', searchable_field: [:name, :id], order: 'id DESC'),
+    response_source: Field::BelongsToSearch.with_options(class_name: 'ResponseSource', searchable_field: [:name, :id, :source_link],
+                                                         order: 'id DESC'),
     answer: Field::Text.with_options(searchable: true),
     question: Field::String.with_options(searchable: true),
+    status: Field::Select.with_options(searchable: false, collection: lambda { |field|
+      field.resource.class.send(field.attribute.to_s.pluralize).keys
+    }),
     response_document: Field::BelongsToSearch.with_options(class_name: 'ResponseDocument', searchable_field: [:document_link, :content, :id],
                                                            order: 'id DESC'),
     created_at: Field::DateTime,
@@ -27,17 +32,21 @@ class ResponseDashboard < Administrate::BaseDashboard
     id
     question
     answer
+    status
     response_document
+    response_source
     account
   ].freeze
 
   # SHOW_PAGE_ATTRIBUTES
   # an array of attributes that will be displayed on the model's show page.
   SHOW_PAGE_ATTRIBUTES = %i[
     id
+    status
     question
     answer
     response_document
+    response_source
     account
     created_at
     updated_at
@@ -47,10 +56,11 @@ class ResponseDashboard < Administrate::BaseDashboard
   # an array of attributes that will be displayed
   # on the model's form (`new` and `edit`) pages.
   FORM_ATTRIBUTES = %i[
+    response_source
+    response_document
     question
     answer
-    response_document
-    account
+    status
   ].freeze
 
   # COLLECTION_FILTERS
@@ -63,7 +73,12 @@ class ResponseDashboard < Administrate::BaseDashboard
   #   COLLECTION_FILTERS = {
   #     open: ->(resources) { resources.where(open: true) }
   #   }.freeze
-  COLLECTION_FILTERS = {}.freeze
+  COLLECTION_FILTERS = {
+    account: ->(resources, attr) { resources.where(account_id: attr) },
+    response_source: ->(resources, attr) { resources.where(response_source_id: attr) },
+    response_document: ->(resources, attr) { resources.where(response_document_id: attr) },
+    status: ->(resources, attr) { resources.where(status: attr) }
+  }.freeze
 
   # Overwrite this method to customize how responses are displayed
   # across all pages of the admin dashboard.

enterprise/app/dashboards/response_document_dashboard.rb

@@ -38,11 +38,11 @@ class ResponseDocumentDashboard < Administrate::BaseDashboard
   SHOW_PAGE_ATTRIBUTES = %i[
     id
     account
-    content
-    document_id
+    response_source
     document_link
+    document_id
     document_type
-    response_source
+    content
     created_at
     updated_at
     responses
@@ -53,11 +53,11 @@ class ResponseDocumentDashboard < Administrate::BaseDashboard
   # on the model's form (`new` and `edit`) pages.
   FORM_ATTRIBUTES = %i[
     account
-    content
-    document_id
+    response_source
     document_link
+    document_id
     document_type
-    response_source
+    content
   ].freeze
 
   # COLLECTION_FILTERS
@@ -70,7 +70,10 @@ class ResponseDocumentDashboard < Administrate::BaseDashboard
   #   COLLECTION_FILTERS = {
   #     open: ->(resources) { resources.where(open: true) }
   #   }.freeze
-  COLLECTION_FILTERS = {}.freeze
+  COLLECTION_FILTERS = {
+    account: ->(resources, attr) { resources.where(account_id: attr) },
+    response_source: ->(resources, attr) { resources.where(response_source_id: attr) }
+  }.freeze
 
   # Overwrite this method to customize how response documents are displayed
   # across all pages of the admin dashboard.

enterprise/app/dashboards/response_source_dashboard.rb

@@ -8,7 +8,7 @@ class ResponseSourceDashboard < Administrate::BaseDashboard
   # which determines how the attribute is displayed
   # on pages throughout the dashboard.
   ATTRIBUTE_TYPES = {
-    id: Field::Number,
+    id: Field::Number.with_options(searchable: true),
     account: Field::BelongsToSearch.with_options(class_name: 'Account', searchable_field: [:name, :id], order: 'id DESC'),
     name: Field::String.with_options(searchable: true),
     response_documents: Field::HasMany,
@@ -73,7 +73,9 @@ class ResponseSourceDashboard < Administrate::BaseDashboard
   #   COLLECTION_FILTERS = {
   #     open: ->(resources) { resources.where(open: true) }
   #   }.freeze
-  COLLECTION_FILTERS = {}.freeze
+  COLLECTION_FILTERS = {
+    account: ->(resources, attr) { resources.where(account_id: attr) }
+  }.freeze
 
   # Overwrite this method to customize how response sources are displayed
   # across all pages of the admin dashboard.

enterprise/app/jobs/response_builder_job.rb

@@ -63,14 +63,20 @@ def prepare_headers
 
   def create_responses(response, response_document)
     response_body = JSON.parse(response.body)
-    faqs = JSON.parse(response_body['choices'][0]['message']['content'].strip)
+    content = response_body.dig('choices', 0, 'message', 'content')
+
+    return if content.nil?
+
+    faqs = JSON.parse(content.strip)
 
     faqs.each do |faq|
       response_document.responses.create!(
         question: faq['question'],
         answer: faq['answer'],
-        account_id: response_document.account_id
+        response_source: response_document.response_source
       )
     end
+  rescue JSON::ParserError => e
+    Rails.logger.error "Error in parsing GPT processed response document : #{e.message}"
   end
 end

enterprise/app/models/enterprise/audit_log.rb

@@ -1,3 +1,31 @@
+# == Schema Information
+#
+# Table name: audits
+#
+#  id              :bigint           not null, primary key
+#  action          :string
+#  associated_type :string
+#  auditable_type  :string
+#  audited_changes :jsonb
+#  comment         :string
+#  remote_address  :string
+#  request_uuid    :string
+#  user_type       :string
+#  username        :string
+#  version         :integer          default(0)
+#  created_at      :datetime
+#  associated_id   :bigint
+#  auditable_id    :bigint
+#  user_id         :bigint
+#
+# Indexes
+#
+#  associated_index              (associated_type,associated_id)
+#  auditable_index               (auditable_type,auditable_id,version)
+#  index_audits_on_created_at    (created_at)
+#  index_audits_on_request_uuid  (request_uuid)
+#  user_index                    (user_id,user_type)
+#
 class Enterprise::AuditLog < Audited::Audit
   after_save :log_additional_information
 

enterprise/app/models/enterprise/concerns/inbox.rb

@@ -3,9 +3,10 @@ module Enterprise::Concerns::Inbox
 
   included do
     def self.add_response_related_associations
-      has_many :response_sources, dependent: :destroy_async
+      has_many :inbox_response_sources, dependent: :destroy_async
+      has_many :response_sources, through: :inbox_response_sources
       has_many :response_documents, through: :response_sources
-      has_many :responses, through: :response_documents
+      has_many :responses, through: :response_sources
     end
 
     add_response_related_associations if Features::ResponseBotService.new.vector_extension_enabled?

enterprise/app/models/enterprise/inbox.rb

@@ -7,7 +7,7 @@ def member_ids_with_assignment_capacity
 
   def get_responses(query)
     embedding = Openai::EmbeddingsService.new.get_embedding(query)
-    responses.nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5)
+    responses.active.nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5)
   end
 
   def active_bot?

enterprise/app/models/inbox_response_source.rb

@@ -0,0 +1,21 @@
+# == Schema Information
+#
+# Table name: inbox_response_sources
+#
+#  id                 :bigint           not null, primary key
+#  created_at         :datetime         not null
+#  updated_at         :datetime         not null
+#  inbox_id           :bigint           not null
+#  response_source_id :bigint           not null
+#
+# Indexes
+#
+#  index_inbox_response_sources_on_inbox_id                         (inbox_id)
+#  index_inbox_response_sources_on_inbox_id_and_response_source_id  (inbox_id,response_source_id) UNIQUE
+#  index_inbox_response_sources_on_response_source_id               (response_source_id)
+#  index_inbox_response_sources_on_response_source_id_and_inbox_id  (response_source_id,inbox_id) UNIQUE
+#
+class InboxResponseSource < ApplicationRecord
+  belongs_to :inbox
+  belongs_to :response_source
+end

enterprise/app/models/response.rb

@@ -6,22 +6,28 @@
 #  answer               :text             not null
 #  embedding            :vector(1536)
 #  question             :string           not null
+#  status               :integer          default(0)
 #  created_at           :datetime         not null
 #  updated_at           :datetime         not null
 #  account_id           :bigint           not null
 #  response_document_id :bigint
+#  response_source_id   :bigint           not null
 #
 # Indexes
 #
 #  index_responses_on_embedding             (embedding) USING ivfflat
 #  index_responses_on_response_document_id  (response_document_id)
 #
 class Response < ApplicationRecord
-  belongs_to :response_document
+  belongs_to :response_document, optional: true
   belongs_to :account
+  belongs_to :response_source
   has_neighbors :embedding, normalize: true
 
   before_save :update_response_embedding
+  before_validation :ensure_account
+
+  enum status: { pending: 0, active: 1 }
 
   def self.search(query)
     embedding = Openai::EmbeddingsService.new.get_embedding(query)
@@ -30,6 +36,10 @@ def self.search(query)
 
   private
 
+  def ensure_account
+    self.account = response_source.account
+  end
+
   def update_response_embedding
     self.embedding = Openai::EmbeddingsService.new.get_embedding("#{question}: #{answer}")
   end

enterprise/app/models/response_document.rb

@@ -18,7 +18,7 @@
 #  index_response_documents_on_response_source_id  (response_source_id)
 #
 class ResponseDocument < ApplicationRecord
-  has_many :responses, dependent: :destroy
+  has_many :responses, dependent: :destroy_async
   belongs_to :account
   belongs_to :response_source
 

enterprise/app/models/response_source.rb

@@ -10,7 +10,6 @@
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
 #  account_id        :bigint           not null
-#  inbox_id          :bigint           not null
 #  source_model_id   :bigint
 #
 # Indexes
@@ -19,10 +18,11 @@
 #
 class ResponseSource < ApplicationRecord
   enum source_type: { external: 0, kbase: 1, inbox: 2 }
+  has_many :inbox_response_sources, dependent: :destroy_async
+  has_many :inboxes, through: :inbox_response_sources
   belongs_to :account
-  belongs_to :inbox
-  has_many :response_documents, dependent: :destroy
-  has_many :responses, through: :response_documents
+  has_many :response_documents, dependent: :destroy_async
+  has_many :responses, dependent: :destroy_async
 
   accepts_nested_attributes_for :response_documents
 end

enterprise/app/services/features/response_bot_service.rb

@@ -23,19 +23,31 @@ def vector_extension_enabled?
   def create_tables
     return unless vector_extension_enabled?
 
-    %i[response_sources response_documents responses].each do |table|
+    %i[response_sources response_documents responses inbox_response_sources].each do |table|
       send("create_#{table}_table")
     end
   end
 
   def drop_tables
-    %i[responses response_documents response_sources].each do |table|
+    %i[responses response_documents response_sources inbox_response_sources].each do |table|
       MIGRATION_VERSION.drop_table table if MIGRATION_VERSION.table_exists?(table)
     end
   end
 
   private
 
+  def create_inbox_response_sources_table
+    return if MIGRATION_VERSION.table_exists?(:inbox_response_sources)
+
+    MIGRATION_VERSION.create_table :inbox_response_sources do |t|
+      t.references :inbox, null: false
+      t.references :response_source, null: false
+      t.index [:inbox_id, :response_source_id], name: 'index_inbox_response_sources_on_inbox_id_and_response_source_id', unique: true
+      t.index [:response_source_id, :inbox_id], name: 'index_inbox_response_sources_on_response_source_id_and_inbox_id', unique: true
+      t.timestamps
+    end
+  end
+
   def create_response_sources_table
     return if MIGRATION_VERSION.table_exists?(:response_sources)
 
@@ -45,7 +57,6 @@ def create_response_sources_table
       t.string :source_link
       t.references :source_model, polymorphic: true
       t.bigint :account_id, null: false
-      t.bigint :inbox_id, null: false
       t.timestamps
     end
   end
@@ -69,9 +80,11 @@ def create_responses_table
     return if MIGRATION_VERSION.table_exists?(:responses)
 
     MIGRATION_VERSION.create_table :responses do |t|
+      t.bigint :response_source_id, null: false
       t.bigint :response_document_id
       t.string :question, null: false
       t.text :answer, null: false
+      t.integer :status, default: 0
       t.bigint :account_id, null: false
       t.vector :embedding, limit: 1536
       t.timestamps

lib/tasks/auto_annotate_models.rake

@@ -20,7 +20,10 @@ if Rails.env.development?
       'show_complete_foreign_keys' => 'false',
       'show_indexes' => 'true',
       'simple_indexes' => 'false',
-      'model_dir' => 'app/models',
+      'model_dir' => [
+        'app/models',
+        'enterprise/app/models',
+      ],
       'root_dir' => '',
       'include_version' => 'false',
       'require' => '',