Handle escaping of index names

.travis.yml

@@ -27,8 +27,16 @@ matrix:
     env: LOGSTASH_BRANCH=6.5
   - rvm: jruby-1.7.27
     env: LOGSTASH_BRANCH=5.6
+  - rvm: jruby-1.7.27
+    env: DISTRIBUTION=legacy INTEGRATION=true ES_VERSION=5.6.13 LOGSTASH_BRANCH=5.6 TEST_DEBUG=true
+  - rvm: jruby-9.1.13.0
+    env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-6 LOGSTASH_BRANCH=6.6 TEST_DEBUG=true
+  - rvm: jruby-9.1.13.0
+    env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-7 LOGSTASH_BRANCH=master TEST_DEBUG=true
   allow_failures:
   - env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-7 TEST_DEBUG=true
+  - rvm: jruby-9.1.13.0
+    env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-7 LOGSTASH_BRANCH=master TEST_DEBUG=true
   fast_finish: true
 install: true
 script: ci/build.sh

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 9.3.1
+  - Fixed issue with escaping index names which was causing writing aliases for ILM to fail [#831](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/831)
+
 ## 9.3.0
   - Adds support for Index Lifecycle Management for Elasticsearch 6.6.0 and above, running with at least a Basic License(Beta) [#805](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/805)
 

lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -65,8 +65,8 @@ def perform_request(url, method, path, params={}, body=nil)
       end
 
       request_uri = format_url(url, path)
-
-      resp = @manticore.send(method.downcase, request_uri.to_s, params)
+      request_uri_as_string = remove_double_escaping(request_uri.to_s)
+      resp = @manticore.send(method.downcase, request_uri_as_string, params)
 
       # Manticore returns lazy responses by default
       # We want to block for our usage, this will wait for the repsonse
@@ -83,6 +83,7 @@ def perform_request(url, method, path, params={}, body=nil)
       resp
     end
 
+    # Returned urls from this method should be checked for double escaping.
     def format_url(url, path_and_query=nil)
       request_uri = url.clone
 
@@ -92,23 +93,33 @@ def format_url(url, path_and_query=nil)
       request_uri.password = nil
 
       return request_uri.to_s if path_and_query.nil?
-      
+
       parsed_path_and_query = java.net.URI.new(path_and_query)
-      
+
       query = request_uri.query
       parsed_query = parsed_path_and_query.query
-      
+
       new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
         part && !part.empty? # Skip empty nil and ""
       end
 
       request_uri.query = new_query_parts.join("&") unless new_query_parts.empty?
 
+      # use `raw_path`` as `path` will unescape any escaped '/' in the path
       request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.raw_path}".gsub(/\/{2,}/, "/")
-
       request_uri
     end
 
+    # Later versions of SafeURI will also escape the '%' sign in an already escaped URI.
+    # (If the path variable is used, it constructs a new java.net.URI object using the multi-arg constructor,
+    # which will escape any '%' characters in the path, as opposed to the single-arg constructor which requires illegal
+    # characters to be already escaped, and will throw otherwise)
+    # The URI needs to have been previously escaped, as it does not play nice with an escaped '/' in the
+    # middle of a URI, as required by date math, treating it as a path separator
+    def remove_double_escaping(url)
+      url.gsub(/%25([0-9A-F]{2})/i, '%\1')
+    end
+
     def close
       @manticore.close
     end

logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '9.3.0'
+  s.version         = '9.3.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -102,6 +102,7 @@
 
       it "should add the path correctly" do
         expect(formatted.path).to eq("#{url.path}special_bulk")
+        expect(subject.remove_double_escaping(formatted.path)).to eq("#{url.path}special_bulk")
       end 
 
       it "should add the query parameters correctly" do
@@ -117,6 +118,26 @@
         expect(formatted.userinfo).to be_nil
       end
     end
+
+    context 'when uri contains date math' do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+      let(:path) { CGI.escape("<logstash-{now/d}-0001>") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it 'should escape the uri correctly' do
+        expect(subject.remove_double_escaping(formatted.path)).to eq("/%3Clogstash-%7Bnow%2Fd%7D-0001%3E")
+      end
+    end
+
+    context 'when uri does not contain date math' do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+      let(:path) { CGI.escape("logstash-0001") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it 'should escape the uri correctly' do
+        expect(subject.remove_double_escaping(formatted.path)).to eq("/logstash-0001")
+      end
+    end
   end
 
   describe "integration specs", :integration => true do