Feature/grab musicians from el recodo

api/Gemfile.lock

@@ -215,12 +215,11 @@ GEM
       railties (>= 5.0.0)
     faker (3.4.1)
       i18n (>= 1.8.11, < 2)
-    faraday (2.9.2)
-      faraday-net_http (>= 2.0, < 3.2)
+    faraday (1.2.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords
     faraday-multipart (1.0.4)
       multipart-post (~> 2)
-    faraday-net_http (3.1.0)
-      net-http
     ffi (1.17.0-aarch64-linux-gnu)
     ffi (1.17.0-arm64-darwin)
     ffi (1.17.0-x86_64-linux-gnu)
@@ -381,8 +380,6 @@ GEM
     multipart-post (2.4.1)
     mutex_m (0.2.0)
     nenv (0.3.0)
-    net-http (0.4.1)
-      uri
     net-imap (0.4.13)
       date
       net-protocol
@@ -581,6 +578,7 @@ GEM
     ruby-progressbar (1.13.0)
     ruby-vips (2.2.1)
       ffi (~> 1.12)
+    ruby2_keywords (0.0.5)
     searchkick (5.3.1)
       activemodel (>= 6.1)
       hashie
@@ -654,7 +652,6 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
-    uri (0.13.0)
     view_component (3.12.1)
       activesupport (>= 5.2.0, < 8.0)
       concurrent-ruby (~> 1.0)

api/app/models/external_catalog/el_recodo/song_scraper.rb

@@ -23,15 +23,17 @@ class PageNotFoundError < StandardError; end
         :page_updated_at
       ).freeze
 
-      def initialize(music_id:)
-        @music_id = music_id
+      def initialize(email:, password:)
+        @cookie = login(email:, password:)
       end
 
-      def metadata
+      def fetch(music_id:)
+        @music_id = music_id
+        binding.irb
         Metadata.new(
           date:,
           ert_number:,
-          music_id: @music_id,
+          music_id:,
           title:,
           style:,
           orchestra:,
@@ -49,9 +51,29 @@ def metadata
 
       private
 
+      def login(email:, password:)
+        response = Faraday.post("https://www.el-recodo.com/connect?lang=en") do |req|
+          req.headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15"
+          req.headers["Content-Type"] = "application/x-www-form-urlencoded"
+          req.headers["Accept"] = "*/*"
+          req.headers["Connection"] = "keep-alive"
+
+          req.body = {
+            "wish" => "logged",
+            "email" => email,
+            "pwd" => password,
+            "autologin" => "1",
+            "backurl" => ""
+          }
+        end
+
+        response.headers["Set-Cookie"]
+      end
+
       def faraday
         @faraday ||= Faraday.new do |conn|
           conn.headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15"
+          conn.headers["Cookie"] = @cookie if @cookie
         end
       end
 
@@ -136,24 +158,21 @@ def safe_parse_date(date_string)
       end
 
       def parse_html
-        @parsed_html ||= Nokogiri::HTML(fetch_page.body)
+        page = faraday.get("https://www.el-recodo.com/music?id=#{@music_id}&lang=en")
+        @parsed_html ||= Nokogiri::HTML(page.body)
       end
 
-      def fetch_page
-        begin
-          response = faraday.get("https://www.el-recodo.com/music?id=#{@music_id}&lang=en")
-          if response.status == 429
-            Rails.logger.error("El Recodo Song Scraper: Too Many Requests")
-            raise TooManyRequestsError
-          elsif response.status != 200
-            Rails.logger.error("El Recodo Song Scraper: Page Not Found")
-            raise PageNotFoundError
-          end
-        rescue Faraday::Error => e
-          Rails.logger.error("El Recodo Song Scraper: #{e.message}")
-          raise
+      def extract_musicians
+        musicians = {}
+        instruments = ["PIANO", "DOUBLEBASS", "BANDONEON", "VIOLIN", "ARRANGER"]
+
+        instruments.each do |instrument|
+          instrument_data = parse_html.at_xpath("//text()[contains(.,'#{instrument}')]").parent
+          musician_links = instrument_data.css("a")
+          musicians[instrument.downcase.to_sym] = musician_links.map { |link| {name: link.text, url: link["href"]} }
         end
-        response
+
+        musicians
       end
     end
   end

api/config/initializers/faraday.rb

@@ -0,0 +1,3 @@
+Faraday.default_connection = Faraday.new do |conn|
+  conn.response :raise_error
+end

api/db/migrate/20240114234658_create_el_recodo_songs.rb

@@ -15,7 +15,10 @@ def change
       t.string :label
       t.jsonb :members, null: false, default: "{}"
       t.text :lyrics
+      t.integer :lyrics_year
       t.string :search_data
+      t.string :matrix
+      t.string :disk
       t.index :ert_number
       t.index :music_id, unique: true
       t.index :synced_at

api/db/migrate/20240114234659_create_el_recodo_musicians.rb

@@ -0,0 +1,17 @@
+class CreateElRecodoPersons < ActiveRecord::Migration[7.1]
+  def change
+    create_table :el_recodo_persons, id: :uuid do |t|
+      t.date :birth_date
+      t.date :death_date
+      t.string :real_name
+      t.string :nicknames, array: true
+      t.string :place_of_birth
+      t.string :url
+      t.string :image_url
+      t.datetime :synced_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+      t.datetime :page_updated_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+
+      t.timestamps
+    end
+  end
+end

api/spec/models/import/el_recodo/song_scraper_spec.rb

@@ -4,13 +4,13 @@
   describe "#metadata" do
     context "for normal songs" do
       before do
-        music_1_html = Rails.root.join("spec/fixtures/html/el_recodo_music_id_1.html")
-        stub_request(:get, "https://www.el-recodo.com/music?id=1&lang=en")
-          .to_return(status: 200, body: File.read(music_1_html))
+        # music_1_html = Rails.root.join("spec/fixtures/html/el_recodo_music_id_1.html")
+        # stub_request(:get, "https://www.el-recodo.com/music?id=1&lang=en")
+        #   .to_return(status: 200, body: File.read(music_1_html))
       end
 
       it "fetches and parses song metadata correctly" do
-        metadata = described_class.new(music_id: 1).metadata
+        metadata = Import::ElRecodo::SongScraper.new(email: "dogacozen87@gmail.com", password: "myNewPass123").fetch(music_id: 3573)
 
         expect(metadata.ert_number).to eq(1)
         expect(metadata.title).to eq("Te burlas tristeza")

api/spec/rails_helper.rb

@@ -25,9 +25,9 @@
   config.infer_spec_type_from_file_location!
   config.filter_rails_from_backtrace!
 
-  config.before(:suite) do
-    WebMock.disable_net_connect!(allow: ["elasticsearch", "localhost"])
-  end
+  # config.before(:suite) do
+  #   WebMock.disable_net_connect!(allow: ["elasticsearch", "localhost"])
+  # end
 
   config.before(:each) do
     ActiveStorage::Current.url_options = {host: "example.com"}