[Fix rubocop#12034] Fix an error when using an invalid encoding string

Fixes rubocop#12034.

This PR fixes invalid byte sequence in UTF-8 error when using an invalid encoding string.

changelog/fix_an_error_when_using_an_invalid_encoding_string.md

@@ -0,0 +1 @@
+* [#12034](https://github.com/rubocop/rubocop/issues/12034): Fix invalid byte sequence in UTF-8 error when using an invalid encoding string. ([@koic][])

lib/rubocop/cop/style/frozen_string_literal_comment.rb

@@ -142,7 +142,9 @@ def last_special_comment(processed_source)
           end
 
           next_token = processed_source.tokens[token_number]
-          token = next_token if Encoding::ENCODING_PATTERN.match?(next_token&.text)
+          if next_token&.text&.valid_encoding? && Encoding::ENCODING_PATTERN.match?(next_token.text)
+            token = next_token
+          end
 
           token
         end

spec/rubocop/cli_spec.rb

@@ -86,6 +86,24 @@
     end
   end
 
+  context 'when using an invalid encoding string' do
+    it 'does not crash and reports an offense' do
+      create_file('example.rb', <<~'RUBY')
+        "\xE3\xD3\x8B\xE3\x83\xBC\x83\xE3\x83\xE3\x82\xB3\xA3\x82\x99"
+      RUBY
+      result = cli.run(['--format', 'simple', 'example.rb'])
+      expect(result).to eq(1)
+      expect($stdout.string)
+        .to eq(<<~RESULT)
+          == example.rb ==
+          C:  1:  1: [Correctable] Style/FrozenStringLiteralComment: Missing frozen string literal comment.
+
+          1 file inspected, 1 offense detected, 1 offense autocorrectable
+      RESULT
+      expect($stderr.string).to eq ''
+    end
+  end
+
   context 'when checking a correct file' do
     it 'returns 0' do
       create_file('example.rb', <<~RUBY)