Add spam detection engine

.github/workflows/ci_ai.yml

@@ -0,0 +1,34 @@
+name: "[CI] Ai"
+on:
+  push:
+    branches:
+      - develop
+      - release/*
+      - "*-stable"
+  pull_request:
+    branches-ignore:
+      - "chore/l10n*"
+    paths:
+      - "*"
+      - ".github/**"
+      - "decidim-ai/**"
+      - "decidim-core/**"
+      - "decidim-dev/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build_app:
+    uses: ./.github/workflows/build_app.yml
+    secrets: inherit
+    name: Build test application
+  main:
+    needs: build_app
+    name: Tests
+    uses: ./.github/workflows/test_app.yml
+    secrets: inherit
+    with:
+      working-directory: "decidim-ai"
+      test_command: bundle exec parallel_test --type rspec --pattern spec/

.spelling.yml

@@ -5,6 +5,8 @@ exclude_paths:
   - decidim-core/lib/decidim/db/common-passwords.txt
   - decidim-initiatives/spec/types/initiative_type_spec.rb
   - decidim-proposals/app/packs/documents/decidim/proposals/participatory_texts/participatory_text.md
+  - decidim-ai/data/(.*).csv
+  - decidim-ai/spec/support/test.csv
 
 forbidden:
   arent: are not

Gemfile

@@ -5,6 +5,7 @@ source "https://rubygems.org"
 ruby RUBY_VERSION
 
 gem "decidim", path: "."
+gem "decidim-ai", path: "."
 gem "decidim-conferences", path: "."
 gem "decidim-consultations", path: "."
 gem "decidim-elections", path: "."

Gemfile.lock

@@ -31,6 +31,10 @@ PATH
       devise (~> 4.7)
       devise-i18n (~> 1.2)
       devise_invitable (~> 2.0)
+    decidim-ai (0.28.0.dev)
+      classifier-reborn (~> 2.3.0)
+      cld (~> 0.11)
+      decidim-core (= 0.28.0.dev)
     decidim-api (0.28.0.dev)
       commonmarker (~> 0.23.0, >= 0.23.9)
       graphql (~> 2.0)
@@ -312,6 +316,11 @@ GEM
       actionpack (>= 5.0)
       cells (>= 4.1.6, < 5.0.0)
     charlock_holmes (0.7.7)
+    classifier-reborn (2.3.0)
+      fast-stemmer (~> 1.0)
+      matrix (~> 0.4)
+    cld (0.13.0)
+      ffi
     cmdparse (3.0.7)
     commonmarker (0.23.9)
     concurrent-ruby (1.2.2)
@@ -399,6 +408,7 @@ GEM
     faraday-retry (1.0.3)
     faraday_middleware (1.2.0)
       faraday (~> 1.0)
+    fast-stemmer (1.0.2)
     ffi (1.15.5)
     file_validators (3.0.0)
       activemodel (>= 3.2)
@@ -820,6 +830,7 @@ DEPENDENCIES
   brakeman (~> 5.4)
   byebug (~> 11.0)
   decidim!
+  decidim-ai!
   decidim-conferences!
   decidim-consultations!
   decidim-dev!

decidim-ai/README.md

@@ -0,0 +1,55 @@
+# Decidim::Ai
+
+The Decidim::AI is a library that aims to privide Artificial Inteligence tools for Decidim. This plugin has been initially developed aiming to analyze the content and provide spam classification using Naive Bayes algorithm.
+All AI related functionality provided by Decidim should be included in this same module.
+
+## Installation
+
+In order to install use this library, you need at least Decidim 0.25 to be installed.
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "decidim-tools-ai"
+```
+
+And then execute:
+
+```bash
+bundle install
+```
+
+After that, add an initializer file inside your project, having the following content:
+
+```ruby
+# config/initializers/decidim_ai.rb
+```
+
+After the configuration is added, you need to run the below command, so that the reporting user is created.
+
+```ruby
+bundle exec rake decidim:spam:data:create_reporting_user
+```
+
+If you have an existing installation, you can use the below command to train the engine with your existing data:
+
+```ruby
+bundle exec rake decidim:spam:train:moderation
+```
+
+Add the queue name to `config/sidekiq.yml` file:
+
+```yaml
+:queues:
+- ["default", 1]
+- ["spam_analysis", 1]
+# The other yaml entries
+```
+
+## Contributing
+
+See [Decidim](https://github.com/decidim/decidim).
+
+## License
+
+See [Decidim](https://github.com/decidim/decidim).

decidim-ai/Rakefile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "decidim/dev/common_rake"

decidim-ai/decidim-ai.gemspec

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path("lib", __dir__)
+
+require "decidim/ai/version"
+
+Gem::Specification.new do |s|
+  s.version = Decidim::Ai.version
+  s.authors = ["Alexandru-Emil Lupu"]
+  s.email = ["contact@alecslupu.ro"]
+  s.license = "AGPL-3.0"
+  s.homepage = "https://decidim.org"
+  s.metadata = {
+    "bug_tracker_uri" => "https://github.com/decidim/decidim/issues",
+    "documentation_uri" => "https://docs.decidim.org/",
+    "funding_uri" => "https://opencollective.com/decidim",
+    "homepage_uri" => "https://decidim.org",
+    "source_code_uri" => "https://github.com/decidim/decidim"
+  }
+  s.required_ruby_version = ">= 3.1"
+
+  s.name = "decidim-ai"
+  s.summary = "A Decidim module with AI tools"
+  s.description = "A Decidim module with AI tools"
+
+  s.files = Dir["{app,config,db,lib,vendor}/**/*", "Rakefile", "README.md"]
+
+  s.add_dependency "classifier-reborn", "~> 2.3.0"
+  s.add_dependency "cld", "~> 0.11"
+  s.add_dependency "decidim-core", Decidim::Ai.version
+end

decidim-ai/lib/decidim/ai.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "decidim/ai/engine"
+
+module Decidim
+  module Ai
+    autoload :LanguageDetectionService, "decidim/ai/language_detection_service"
+    autoload :SpamDetectionService, "decidim/ai/spam_detection_service"
+    autoload :StrategyRegistry, "decidim/ai/strategy_registry"
+    autoload :LoadDataset, "decidim/ai/load_dataset"
+
+    module SpamContent
+      autoload :BaseStrategy, "decidim/ai/spam_content/base_strategy"
+      autoload :BayesStrategy, "decidim/ai/spam_content/bayes_strategy"
+    end
+
+    include ActiveSupport::Configurable
+
+    # You can configure the spam treshold for the spam detection service.
+    # The treshold is a float value between 0 and 1.
+    # The default value is 0.5
+    # Any value below the treshold will be considered spam.
+    config_accessor :spam_treshold do
+      0.5
+    end
+    # Registered analyzers.
+    # You can register your own analyzer by adding a new entry to this array.
+    # The entry must be a hash with the following keys:
+    # - name: the name of the analyzer
+    # - strategy: the class of the strategy to use
+    # - options: a hash with the options to pass to the strategy
+    # Example:
+    # config.registered_analyzers = [
+    #   {
+    #     name: :bayes,
+    #     strategy: Decidim::Ai::SpamContent::BayesStrategy,
+    #     options: {
+    #       adapter: :redis,
+    #       params: {
+    #         url:                lambda { ENV["REDIS_URL"] }
+    #         scheme:             "redis"
+    #         host:               "127.0.0.1"
+    #         port:               6379
+    #         path:               nil
+    #         timeout:            5.0
+    #         password:           nil
+    #         db:                 0
+    #         driver:             nil
+    #         id:                 nil
+    #         tcp_keepalive:      0
+    #         reconnect_attempts: 1
+    #         inherit_socket:     false
+    #       }
+    #     }
+    #   }
+    # ]
+    config_accessor :registered_analyzers do
+      [
+        { name: :bayes, strategy: Decidim::Ai::SpamContent::BayesStrategy, options: { adapter: :memory, params: {} } }
+      ]
+    end
+
+    # Language detection service class.
+    #
+    # If you want to autodetect the language of the content, you can use a class service having the following contract
+    #
+    # class LanguageDetectionService
+    #   def initialize(text)
+    #     @text = text
+    #   end
+    #
+    #   def language_code
+    #     CLD.detect_language(@text).fetch(:code)
+    #   end
+    # end
+    config_accessor :language_detection_service do
+      "Decidim::Ai::LanguageDetectionService"
+    end
+
+    # Spam detection service class.
+    # If you want to use a different spam detection service, you can use a class service having the following contract
+    #
+    # class SpamDetectionService
+    #   def initialize
+    #     @registry = Decidim::Ai.spam_detection_registry
+    #   end
+    #
+    #   def train(category, text)
+    #     # train the strategy
+    #   end
+    #
+    #   def classify(text)
+    #     # classify the text
+    #   end
+    #
+    #   def untrain(category, text)
+    #     # untrain the strategy
+    #   end
+    #
+    #   def classification_log
+    #     # return the classification log
+    #   end
+    # end
+    config_accessor :spam_detection_service do
+      "Decidim::Ai::SpamDetectionService"
+    end
+
+    # This is the email address used by the spam engine to
+    # properly identify the user that will report users and content
+    config_accessor :reporting_user_email do
+      "reporting.user@domain.tld"
+    end
+
+    def self.spam_detection_instance
+      @spam_detection_instance ||= spam_detection_service.constantize.new
+    end
+
+    def self.spam_detection_registry
+      @spam_detection ||= Decidim::Ai::StrategyRegistry.new
+    end
+
+    def self.create_reporting_users!
+      Decidim::Organization.find_each do |organization|
+        user = organization.users.find_or_initialize_by(email: Decidim::Ai.reporting_user_email)
+        next if user.persisted?
+
+        password = SecureRandom.hex(10)
+        user.password = password
+        user.password_confirmation = password
+
+        user.deleted_at = Time.current
+        user.tos_agreement = true
+        user.name = ""
+        user.skip_confirmation!
+        user.save!
+      end
+    end
+  end
+end

decidim-ai/lib/decidim/ai/engine.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    class Engine < ::Rails::Engine
+      isolate_namespace Decidim::Ai
+
+      paths["db/migrate"] = nil
+      # paths["lib/tasks"] = nil
+
+      initializer "decidim_ai.classifiers" do |_app|
+        Decidim::Ai.registered_analyzers.each do |analyzer|
+          Decidim::Ai.spam_detection_registry.register_analyzer(**analyzer)
+        end
+      end
+
+      def load_seed
+        nil
+      end
+    end
+  end
+end

decidim-ai/lib/decidim/ai/language_detection_service.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "cld"
+
+module Decidim
+  module Ai
+    class LanguageDetectionService
+      def initialize(text)
+        @text = text
+      end
+
+      def language_code
+        CLD.detect_language(@text).fetch(:code)
+      end
+    end
+  end
+end

decidim-ai/lib/decidim/ai/load_dataset.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    class LoadDataset
+      def self.call(file)
+        service = Decidim::Ai.spam_detection_instance
+
+        ext = File.extname(file)[1..-1]
+        reader_class = Decidim::Admin::Import::Readers.search_by_file_extension(ext)
+        reader_class.new(file).read_rows do |row|
+          next unless %w(spam ham).include?(row[0])
+          next if row[1].blank?
+
+          service.train(row[0].to_sym, row[1])
+        end
+      end
+    end
+  end
+end