Merge pull request #16 from eivu/feature/overhaul-api-model

feat!: Overhauled api model

Gemfile

@@ -15,11 +15,11 @@ gem 'faker', '~> 2.18' # Faker, a port of Data::Faker from Perl, is used to easi
 gem 'id3tag', '~> 1.1' # Native Ruby ID3 tag reader that aims for 100% coverage of ID3v2.x and ID3v1.x standards
 gem 'itunes_parser', '~> 1.1', '>= 1.1.3' # Parses iTunes Library XML files
 gem 'mimemagic', '~> 0.4.3' # Fast mime detection by extension or content (Uses freedesktop.org.xml shared-mime-info database
-gem 'nokogiri', '~> 1.14' # xml parser
+gem 'nokogiri', '~> 1.16', '>= 1.16.2' # xml parser
 gem 'oj', '~> 3.16', '>= 3.16.1' # faster json parsing
 gem 'pry', '~> 0.14.1' # Debugger
 gem 'rest-client', '~> 2.1' # A simple HTTP and REST client for Ruby, inspired by the Sinatra microframework style of specifying actions
-gem 'rspec', '~> 3.11' # testing lib
+gem 'rspec', '~> 3.13' # testing lib
 gem 'semantic_logger', '~> 4.15' # feature rich logging framework, and replacement for existing Ruby & Rails loggers.
 gem 'rubocop', '~> 1.57', '>= 1.57.2' # code style checking and code formatting tool
 gem 'vcr', '~> 6.1' # VCR for testing

Gemfile.lock

@@ -64,7 +64,7 @@ GEM
     connection_pool (2.4.1)
     crack (0.4.5)
       rexml
-    diff-lcs (1.5.0)
+    diff-lcs (1.5.1)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     drb (2.2.0)
@@ -118,14 +118,12 @@ GEM
     mimemagic (0.4.3)
       nokogiri (~> 1)
       rake
-    mini_portile2 (2.8.1)
     minitest (5.20.0)
     mize (0.4.1)
       protocol (~> 2.0)
     mutex_m (0.2.0)
     netrc (0.11.0)
-    nokogiri (1.14.3)
-      mini_portile2 (~> 2.8.0)
+    nokogiri (1.16.2-arm64-darwin)
       racc (~> 1.4)
     nokogiri-plist (0.5.0)
       nokogiri
@@ -150,19 +148,19 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.5)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.0)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.0)
     rubocop (1.57.2)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
@@ -178,11 +176,12 @@ GEM
       parser (>= 3.2.1.0)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    ruby_parser (3.20.3)
+    ruby_parser (3.21.0)
+      racc (~> 1.5)
       sexp_processor (~> 4.16)
     semantic_logger (4.15.0)
       concurrent-ruby (~> 1.0)
-    sexp_processor (4.17.0)
+    sexp_processor (4.17.1)
     sync (0.5.0)
     tins (1.32.1)
       sync
@@ -216,11 +215,11 @@ DEPENDENCIES
   id3tag (~> 1.1)
   itunes_parser (~> 1.1, >= 1.1.3)
   mimemagic (~> 0.4.3)
-  nokogiri (~> 1.14)
+  nokogiri (~> 1.16, >= 1.16.2)
   oj (~> 3.16, >= 3.16.1)
   pry (~> 0.14.1)
   rest-client (~> 2.1)
-  rspec (~> 3.11)
+  rspec (~> 3.13)
   rubocop (~> 1.57, >= 1.57.2)
   semantic_logger (~> 4.15)
   vcr (~> 6.1)

README.md

@@ -18,4 +18,5 @@ Requirements
 - node
 - python
 - [yt-dlp](https://github.com/yt-dlp/yt-dlp)
+- Download the [fpcalc binary](https://acoustid.org/chromaprint)
 

lib/eivu/client.rb

@@ -21,6 +21,7 @@ def reset
         configuration.access_key_id   = nil
         configuration.secret_key      = nil
         configuration.bucket_name     = nil
+        configuration.bucket_uuid     = nil
         configuration.bucket_location = nil
         configuration.region          = nil
         configuration.endpoint        = nil
@@ -57,37 +58,24 @@ def initialize
       @status = { success: {}, failure: {} }
     end
 
-    def upload_file(path_to_file:, peepy: false, nsfw: false, metadata_list: [], override: {})
-      filesize      = File.size(path_to_file)
-      filename      = File.basename(path_to_file)
-
-      metadata_list += MetadataExtractor.extract(path_to_file)
-      metadata_list << { original_local_path_to_file: path_to_file } unless override[:skip_original_local_path_to_file]
-      asset         = Utils.sanitize(filename)
+    def upload_file(path_to_file:, peepy: false, nsfw: false, override: {}, metadata_list: [])
+      asset         = Utils.sanitize(File.basename(path_to_file))
       md5           = Eivu::Client::CloudFile.generate_md5(path_to_file)&.downcase
-      rating        = MetadataExtractor.extract_rating(filename)
-      year          = MetadataExtractor.extract_year(filename) || Utils.prune_from_metadata_list(metadata_list, 'eivu:year')
-      name          = override[:name] || Utils.prune_from_metadata_list(metadata_list, 'eivu:name')
-      artwork_md5   = Utils.prune_from_metadata_list(metadata_list, 'eivu:artwork_md5')
-      release_pos   = Utils.prune_from_metadata_list(metadata_list, 'eivu:release_pos')
-      duration      = Utils.prune_from_metadata_list(metadata_list, 'eivu:duration')
       log_tag       = "#{md5.first(5)}:#{asset}"
-      artist_name        = Utils.prune_from_metadata_list(metadata_list, 'eivu:artist_name')
-      release_name       = Utils.prune_from_metadata_list(metadata_list, 'eivu:release_name')
+      data_profile  = Utils.generate_data_profile(path_to_file:, override:, metadata_list:)
 
       Eivu::Logger.info 'Fetching/Reserving', tags: log_tag, label: Eivu::Client
-      cloud_file = CloudFile.reserve_or_fetch_by(bucket_name: configuration.bucket_name,
-                                                 provider: configuration.bucket_location, path_to_file:, peepy:, nsfw:)
-      remote_path_to_file = "#{cloud_file.s3_folder}/#{Utils.sanitize(filename)}"
+      cloud_file = CloudFile.reserve_or_fetch_by(bucket_uuid: configuration.bucket_uuid,
+                                                 path_to_file:, peepy:, nsfw:)
 
-      process_reservation_and_transfer(cloud_file:, path_to_file:, remote_path_to_file:, md5:, asset:, filesize:)
+      process_reservation_and_transfer(cloud_file:, path_to_file:, md5:, asset:)
 
       if cloud_file.transfered?
         Eivu::Logger.info 'Completing', tags: log_tag, label: Eivu::Client
-        cloud_file.complete!(artist_name:, release_name:, name:, year:, rating:, artwork_md5:, release_pos:, duration:, metadata_list:, matched_recording: nil)
+        cloud_file.complete!(data_profile)
       else
         Eivu::Logger.info 'Updating/Skipping', tags: log_tag, label: Eivu::Client
-        cloud_file.update_metadata!(artist_name:, release_name:, name:, year:, rating:, artwork_md5:, release_pos:, duration:, metadata_list:, matched_recording: nil)
+        cloud_file.update_metadata!(data_profile)
       end
 
       cloud_file
@@ -150,9 +138,12 @@ def validate_remote_md5!(remote_path_to_file:, path_to_file:, md5:)
 
     private
 
-    def process_reservation_and_transfer(cloud_file:, path_to_file:, remote_path_to_file:, md5:, asset:, filesize:)
+    def process_reservation_and_transfer(cloud_file:, path_to_file:, md5:, asset:)
       return unless cloud_file.reserved?
 
+      filesize      = File.size(path_to_file)
+      remote_path_to_file = "#{cloud_file.s3_folder}/#{Utils.sanitize(File.basename(path_to_file))}"
+
       log_tag = "#{md5.first(5)}:#{asset}"
       Eivu::Logger.info 'Writing to S3', tags: log_tag, label: Eivu::Client
 

lib/eivu/client/cloud_file.rb

@@ -49,42 +49,44 @@ class CloudFile < Dry::Struct
       attribute? :metadata, Types::JSON::Array.of(Types::JSON::Hash)
 
       class << self
-        def reserve_or_fetch_by(bucket_name:, provider:, path_to_file:, peepy: false, nsfw: false)
-          reserve(bucket_name:, provider:, path_to_file:, peepy:, nsfw:)
+        def reserve_or_fetch_by(path_to_file:, peepy: false, nsfw: false, bucket_uuid: Eivu::Client.configuration.bucket_uuid)
+          reserve(path_to_file:, peepy:, nsfw:, bucket_uuid:)
         rescue Errors::Server::InvalidCloudFileState
           md5 = generate_md5(path_to_file)
-          cloud_file = fetch(md5)
+          cloud_file = fetch(md5, bucket_uuid:)
           cloud_file.content_type = Client::Utils.detect_mime(path_to_file).type
           cloud_file
         end
 
-        def fetch(md5)
+        def fetch(md5, bucket_uuid: Eivu::Client.configuration.bucket_uuid)
           response = RestClient.get(
-            "#{Eivu::Client.configuration.host}/api/v1/cloud_files/#{md5}",
+            "#{Eivu::Client.configuration.host}/api/v1/buckets/#{bucket_uuid}/cloud_files/#{md5}",
             { 'Authorization' => "Token #{Eivu::Client.configuration.user_token}" }
           )
 
-          CloudFile.new Oj.load(response.body).symbolize_keys
+          cloud_file = CloudFile.new Oj.load(response.body).symbolize_keys
+          cloud_file.infer_state_history!
+          cloud_file
+        rescue RestClient::Forbidden
+          raise Errors::CloudStorage::MissingResource, "No bucket found with uuid: #{Eivu::Client.configuration.bucket_uuid}"
         rescue RestClient::NotFound
           raise Errors::CloudStorage::MissingResource, "Cloud file #{md5} not found"
         end
 
-        def post_request(action:, md5:, payload:)
+        def post_request(action:, md5:, payload:, bucket_uuid: Eivu::Client.configuration.bucket_uuid)
           response = RestClient.post(
-            "#{Eivu::Client.configuration.host}/api/v1/cloud_files/#{md5}/#{action}",
+            "#{Eivu::Client.configuration.host}/api/v1/buckets/#{bucket_uuid}/cloud_files/#{md5}/#{action}",
             payload,
             { 'Authorization' => "Token #{Eivu::Client.configuration.user_token}" }
           )
 
           raise Errors::Server::Connection, "Failed connection: #{response.code}" unless response.code == 200
 
           Oj.load(response.body).deep_symbolize_keys
-        rescue RestClient::Unauthorized
-          raise Errors::Server::Security, 'Bucket does is not owned by user'
+        rescue RestClient::Forbidden
+          raise Errors::CloudStorage::MissingResource, "No bucket found with uuid: #{Eivu::Client.configuration.bucket_uuid}"
         rescue RestClient::UnprocessableEntity
           raise Errors::Server::InvalidCloudFileState, "Failed to reserve file: #{md5}"
-        rescue RestClient::BadRequest
-          raise Errors::Server::Connection, 'Bucket does not exist'
         rescue Errno::ECONNREFUSED
           raise Errors::Server::Connection, "Failed to connect to eivu server: #{Eivu::Client.configuration.host}"
         end
@@ -93,12 +95,14 @@ def generate_md5(path_to_file)
           Digest::MD5.file(path_to_file).hexdigest.upcase
         end
 
-        def reserve(bucket_name:, provider:, path_to_file:, peepy: false, nsfw: false)
+        def reserve(path_to_file:, peepy: false, nsfw: false, bucket_uuid: Eivu::Client.configuration.bucket_uuid)
           md5          = generate_md5(path_to_file)
-          payload      = { bucket_name:, provider:, peepy:, nsfw:, fullpath: path_to_file }
-          parsed_body  = post_request(action: :reserve, md5:, payload:)
+          payload      = { peepy:, nsfw: }
+          parsed_body  = post_request(action: :reserve, md5:, payload:, bucket_uuid:)
           content_type = Client::Utils.detect_mime(path_to_file).type
-          CloudFile.new parsed_body.merge(state_history: [STATE_RESERVED], content_type:)
+          instance     = CloudFile.new parsed_body.merge(content_type:)
+          instance.state_history = [STATE_RESERVED]
+          instance
         end
       end
 
@@ -113,21 +117,20 @@ def transfer!(asset:, filesize:)
         self
       end
 
-      def update_data!(action: :complete, artist_name: nil, release_name: nil, year: nil, name: nil, rating: nil, release_pos: nil, duration: nil, metadata_list: [], matched_recording: nil, artwork_md5: nil)
-        matched_recording.nil? # trying to avoid rubocop error because it is not used yet
-        payload = { artist_name:, release_name:, name:, year:, rating:, release_pos:, duration:, metadata_list:, artwork_md5: }
+      def update_data!(data_profile, action: :complete)
+        payload = data_profile
         parsed_body = post_request(action:, payload:)
         assign_attributes(parsed_body)
         state_history << STATE_COMPLETED
         self
       end
 
-      def complete!(year: nil, artist_name: nil, release_name: nil, name: nil, rating: nil, release_pos: nil, duration: nil, metadata_list: [], matched_recording: nil, artwork_md5: nil)
-        update_data!(action: :complete, year:, artist_name:, release_name:, name:, rating:, release_pos:, duration:, metadata_list:, matched_recording:, artwork_md5:)
+      def complete!(data_profile)
+        update_data!(data_profile, action: :complete)
       end
 
-      def update_metadata!(year: nil, artist_name: nil, release_name: nil, name: nil, rating: nil, release_pos: nil, duration: nil, metadata_list: [], matched_recording: nil, artwork_md5: nil)
-        update_data!(action: :update_metadata, year:, artist_name:, release_name:, name:, rating:, release_pos:, duration:, metadata_list:, matched_recording:, artwork_md5:)
+      def update_metadata!(data_profile)
+        update_data!(data_profile, action: :update_metadata)
       end
 
       def visit
@@ -157,6 +160,18 @@ def completed?
         state == 'completed'
       end
 
+      def infer_state_history!
+        self.state_history =
+          case state
+          when 'reserved'
+            [STATE_RESERVED]
+          when 'transfered'
+            [STATE_RESERVED, STATE_TRANSFERED]
+          when 'completed'
+            [STATE_RESERVED, STATE_TRANSFERED, STATE_COMPLETED]
+          end
+      end
+
       private
 
       def post_request(action:, payload:)

lib/eivu/client/configuration.rb

@@ -3,7 +3,7 @@
 module Eivu
   class Client
     class Configuration
-      attr_writer :access_key_id, :secret_key, :bucket_name, :region, :user_token, :host, :endpoint, :bucket_location
+      attr_writer :access_key_id, :secret_key, :bucket_name, :bucket_uuid, :region, :user_token, :host, :endpoint, :bucket_location
 
       # Adds global configuration settings to the gem, including:
       #
@@ -58,6 +58,7 @@ def initialize
         @access_key_id   = ENV.fetch('EIVU_ACCESS_KEY_ID')
         @secret_key      = ENV.fetch('EIVU_SECRET_ACCESS_KEY')
         @bucket_name     = ENV.fetch('EIVU_BUCKET_NAME')
+        @bucket_uuid     = ENV.fetch('EIVU_BUCKET_UUID')
         @bucket_location = ENV.fetch('EIVU_BUCKET_LOCATION') || :aws
         @region          = ENV.fetch('EIVU_REGION')
         @user_token      = ENV.fetch('EIVU_USER_TOKEN')
@@ -68,7 +69,7 @@ def initialize
       def access_key_id
         unless @access_key_id
           raise Errors::Configuration,
-                'AWS access key id missing! See the documentation for configuration settings.'
+                'S3 access key id missing! See the documentation for configuration settings.'
         end
 
         @access_key_id
@@ -77,7 +78,7 @@ def access_key_id
       def secret_key
         unless @secret_key
           raise Errors::Configuration,
-                'AWS secret key missing! See the documentation for configuration settings.'
+                'S3 secret key missing! See the documentation for configuration settings.'
         end
 
         @secret_key
@@ -86,12 +87,21 @@ def secret_key
       def bucket_name
         unless @bucket_name
           raise Errors::Configuration,
-                'AWS bucket name missing! See the documentation for configuration settings.'
+                'S3 bucket name missing! See the documentation for configuration settings.'
         end
 
         @bucket_name
       end
 
+      def bucket_uuid
+        unless @bucket_uuid
+          raise Errors::Configuration,
+                'Eivu bucket uuid is missing! See the documentation for configuration settings.'
+        end
+
+        @bucket_uuid
+      end
+
       def user_token
         unless @user_token
           raise Errors::Configuration,

lib/eivu/client/metadata_extractor.rb

@@ -20,13 +20,14 @@ def extract(path_to_file)
           mime = Client::Utils.detect_mime(path_to_file)
           case mime.mediatype
           when 'audio'
-            from_audio_file(mime, path_to_file)
+            from_audio_file(path_to_file, mime:)
           else
             extract_metadata_list(File.basename(path_to_file))
           end
         end
 
-        def from_audio_file(mime, path_to_file)
+        def from_audio_file(path_to_file, mime: nil)
+          mime ||= Client::Utils.detect_mime(path_to_file)
           acoustid_client = Eivu::Fingerprinter::Acoustid.new
           acoustid_client.generate(path_to_file)
           metadata_hash =
@@ -72,10 +73,12 @@ def from_non_mp3_file(path_to_file)
         end
 
         def extract_year(string)
+          string = File.basename(string)
           string.scan(YEAR_REGEX)&.flatten&.first
         end
 
         def extract_metadata_list(string)
+          string = File.basename(string)
           # remove year from string
           temp_string = string.gsub(YEAR_REGEX, '')
           {
@@ -92,6 +95,7 @@ def extract_metadata_list(string)
         end
 
         def extract_rating(string)
+          string = File.basename(string)
           if string.starts_with?('__')
             5
           elsif string.starts_with?('_')