update to centralized travis configuration (elastic#930)

same work as done in the logstash-plugins/logstash-filter-elasticsearch#126 and logstash-plugins/logstash-input-elasticsearch#120.

The only difference is that integration tests need to know the exact elasticsearch version being tested (but the input can be "7.x"). This PR modifies the .ci/logstash-run to fetch the version from elasticsearch itself.

Co-authored-by: Karol Bucek <kares@users.noreply.github.com>

.ci/Dockerfile

@@ -0,0 +1,23 @@
+ARG ELASTIC_STACK_VERSION
+# TODO: refactor this to be implicitly resolved by logstash-plugins/.ci/Dockerfile
+ARG DISTRIBUTION_SUFFIX
+FROM docker.elastic.co/logstash/logstash$DISTRIBUTION_SUFFIX:$ELASTIC_STACK_VERSION
+USER logstash
+COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile
+COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/
+RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml
+ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin"
+ENV LOGSTASH_SOURCE="1"
+ENV ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION
+# DISTRIBUTION="default" (by default) or "oss"
+ARG DISTRIBUTION
+ENV DISTRIBUTION=$DISTRIBUTION
+# INTEGRATION="true" while integration testing (false-y by default)
+ARG INTEGRATION
+ENV INTEGRATION=$INTEGRATION
+RUN gem install bundler -v '< 2'
+WORKDIR /usr/share/plugins/plugin
+RUN bundle install --with test ci
+COPY --chown=logstash:logstash . /usr/share/plugins/plugin
+RUN bundle exec rake vendor
+RUN .ci/setup.sh

.ci/Dockerfile.elasticsearch

@@ -0,0 +1,21 @@
+ARG ELASTIC_STACK_VERSION
+ARG DISTRIBUTION_SUFFIX
+FROM docker.elastic.co/elasticsearch/elasticsearch$DISTRIBUTION_SUFFIX:$ELASTIC_STACK_VERSION
+
+ARG plugin_path=/usr/share/plugins/plugin
+ARG es_path=/usr/share/elasticsearch
+ARG es_yml=$es_path/config/elasticsearch.yml
+ARG SECURE_INTEGRATION
+
+RUN rm -f $es_path/config/scripts
+
+COPY --chown=elasticsearch:elasticsearch spec/fixtures/scripts/groovy/* $es_path/config/scripts/
+COPY --chown=elasticsearch:elasticsearch spec/fixtures/test_certs/* $es_path/config/test_certs/
+COPY --chown=elasticsearch:elasticsearch .ci/elasticsearch-run.sh $es_path/
+
+RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.enabled: $SECURE_INTEGRATION" >> $es_yml; fi
+RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.key: $es_path/config/test_certs/test.key" >> $es_yml; fi
+RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate: $es_path/config/test_certs/test.crt" >> $es_yml; fi
+RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi
+RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser; fi
+RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser; fi

.ci/docker-compose.override.yml

@@ -0,0 +1,32 @@
+version: '3'
+
+services:
+
+  logstash:
+    command: /usr/share/plugins/plugin/.ci/logstash-run.sh
+    build:
+      args:
+        - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION
+        - DISTRIBUTION=${DISTRIBUTION:-default}
+        - DISTRIBUTION_SUFFIX=${DISTRIBUTION_SUFFIX}
+    environment:
+        - DISTRIBUTION=${DISTRIBUTION:-default}
+        - DISTRIBUTION_SUFFIX=${DISTRIBUTION_SUFFIX}
+        - INTEGRATION=${INTEGRATION:-false}
+        - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
+
+  elasticsearch:
+    build:
+      context: ../
+      dockerfile: .ci/Dockerfile.elasticsearch
+      args:
+        - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION
+        - INTEGRATION=${INTEGRATION:-false}
+        - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
+        - DISTRIBUTION_SUFFIX=${DISTRIBUTION_SUFFIX}
+    command: /usr/share/elasticsearch/elasticsearch-run.sh
+    tty: true
+    ports:
+      - "9200:9200"
+    user: elasticsearch
+

ci/docker-run.sh → .ci/docker-run.sh

@@ -2,8 +2,11 @@
 
 # This is intended to be run inside the docker container as the command of the docker-compose.
 set -ex
+
+cd .ci
+
 if [ "$INTEGRATION" == "true" ]; then
-    docker-compose -f ci/docker-compose.yml up --exit-code-from logstash
+    docker-compose up --exit-code-from logstash
 else
-    docker-compose -f ci/docker-compose.yml up --exit-code-from logstash logstash
+    docker-compose up --exit-code-from logstash logstash
 fi

ci/docker-setup.sh → .ci/docker-setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# This is intended to be run the plugin's root directory. `ci/docker-test.sh`
+# This is intended to be run the plugin's root directory. `.ci/docker-setup.sh`
 # Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable.
 set -e
 
@@ -67,12 +67,14 @@ if [ "$ELASTIC_STACK_VERSION" ]; then
         rm Gemfile.lock
     fi
 
+    cd .ci
+
     if [ "$INTEGRATION" == "true" ]; then
-        docker-compose -f ci/docker-compose.yml down
-        docker-compose -f ci/docker-compose.yml build
+        docker-compose down
+        docker-compose build
     else
-        docker-compose -f ci/docker-compose.yml down
-        docker-compose -f ci/docker-compose.yml build logstash 
+        docker-compose down
+        docker-compose build logstash
     fi
 else
     echo "Please set the ELASTIC_STACK_VERSION environment variable"

ci/logstash-run.sh → .ci/logstash-run.sh

@@ -3,21 +3,20 @@ set -ex
 
 export PATH=$BUILD_DIR/gradle/bin:$PATH
 
+if [[ "$SECURE_INTEGRATION" == "true" ]]; then
+  ES_URL="https://elasticsearch:9200 -k"
+else
+  ES_URL="http://elasticsearch:9200"
+fi
+
 wait_for_es() {
-  echo "Waiting for elasticsearch to respond..."
-  es_url="http://elasticsearch:9200"
-  if [[ "$SECURE_INTEGRATION" == "true" ]]; then
-    es_url="https://elasticsearch:9200 -k"
-  fi
   count=120
-  while ! curl --silent $es_url && [[ $count -ne 0 ]]; do
+  while ! curl -s $ES_URL >/dev/null && [[ $count -ne 0 ]]; do
     count=$(( $count - 1 ))
-    [[ $count -eq 0 ]] && return 1
+    [[ $count -eq 0 ]] && exit 1
     sleep 1
   done
-  echo "Elasticsearch is Up !"
-
-  return 0
+  echo $(curl -s $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
 }
 
 if [[ "$INTEGRATION" != "true" ]]; then
@@ -35,6 +34,8 @@ else
   elif [[ "$DISTRIBUTION" == "default" ]]; then
     extra_tag_args="$extra_tag_args --tag ~distribution:oss --tag distribution:xpack"
   fi
-  wait_for_es  
-  bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag update_tests:groovy --tag es_version:$ELASTIC_STACK_VERSION spec/integration
+  echo "Waiting for elasticsearch to respond..."
+  ES_VERSION=$(wait_for_es)
+  echo "Elasticsearch $ES_VERSION is Up!"
+  bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag update_tests:groovy --tag es_version:$ES_VERSION spec/integration
 fi

.travis.yml

@@ -1,19 +1,12 @@
----
-sudo: required
-services: docker
-addons:
-  apt:
-    packages:
-    - docker-ce
+import:
+- logstash-plugins/.ci:travis/travis.yml@1.x
+
 env:
-  - INTEGRATION=false ELASTIC_STACK_VERSION=5.x
-  - INTEGRATION=false ELASTIC_STACK_VERSION=6.x
-  - INTEGRATION=false ELASTIC_STACK_VERSION=7.x
+  - DISTRIBUTION=default INTEGRATION=false ELASTIC_STACK_VERSION=6.x
+  - DISTRIBUTION=default INTEGRATION=false ELASTIC_STACK_VERSION=7.x
   - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=6.x
   - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=7.x
   - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true
   - DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
   - DISTRIBUTION=default SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x
   - DISTRIBUTION=oss INTEGRATION=true ELASTIC_STACK_VERSION=7.x
-install: ci/docker-setup.sh
-script: ci/docker-run.sh