Merge pull request #777 from untergeek/unix_binaries

Add binary building support

.gitignore

@@ -17,3 +17,6 @@ index.html
 docs/asciidoc/html_docs
 wheelhouse
 Elastic.ico
+Vagrant/centos/6/.vagrant
+Vagrant/centos/7/.vagrant
+Vagrant/ubuntu/14.04/.vagrant

Vagrant/centos/6/Vagrantfile

@@ -0,0 +1,18 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+  config.vm.box = "elastic/centos-6-x86_64"
+
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo yum -y groupinstall "Development Tools"
+    sudo yum -y install python-devel
+  SHELL
+
+  config.vm.synced_folder "/curator_packages", "/curator_packages", create: true, owner: "vagrant", group: "vagrant"
+  config.vm.synced_folder "/curator_source", "/curator_source", create: true, owner: "vagrant", group: "vagrant"
+
+  config.vm.provider "virtualbox" do |v|
+    v.customize ["modifyvm", :id, "--nictype1", "virtio"]
+  end
+end

Vagrant/centos/7/Vagrantfile

@@ -0,0 +1,18 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+  config.vm.box = "elastic/centos-7-x86_64"
+
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo yum -y groupinstall "Development Tools"
+    sudo yum -y install python-devel
+  SHELL
+
+  config.vm.synced_folder "/curator_packages", "/curator_packages", create: true, owner: "vagrant", group: "vagrant"
+  config.vm.synced_folder "/curator_source", "/curator_source", create: true, owner: "vagrant", group: "vagrant"
+
+  config.vm.provider "virtualbox" do |v|
+    v.customize ["modifyvm", :id, "--nictype1", "virtio"]
+  end
+end

Vagrant/ubuntu/14.04/Vagrantfile

@@ -0,0 +1,19 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+  config.vm.box = "ubuntu/trusty64"
+
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo apt-get -y autoremove
+    sudo apt-get update
+    sudo apt-get install -y libxml2-dev zlib1g-dev pkg-config python-pip python-dev python3-dev python3-setuptools
+  SHELL
+
+  config.vm.synced_folder "/curator_packages", "/curator_packages", create: true, owner: "vagrant", group: "vagrant"
+  config.vm.synced_folder "/curator_source", "/curator_source", create: true, owner: "vagrant", group: "vagrant"
+
+  config.vm.provider "virtualbox" do |v|
+    v.customize ["modifyvm", :id, "--nictype1", "virtio"]
+  end
+end

docs/Changelog.rst

@@ -3,6 +3,28 @@
 Changelog
 =========
 
+4.1.2 (? ? ?)
+-------------
+
+**General**
+
+  * New Curator binary packages for common Linux systems!
+    These will be found in the same repositories that the python-based packages
+    are in, but have no dependencies.  All necessary libraries/modules are
+    bundled with the binary, so everything should work out of the box.
+    This feature doesn't change any other behavior, so it's not a major release.
+
+    These binaries have been tested in:
+      * CentOS 6 & 7
+      * Ubuntu 12.04, 14.04, 16.04
+      * Debian 8
+
+    They do not work in Debian 7 (library mismatch).  They may work in other
+    systems, but that is untested.
+
+    The script used is in the unix_packages directory.  The Vagrantfiles for
+    the various build systems are in the Vagrant directory.
+
 4.1.1 (27 September 2016)
 -------------------------
 

unix_packages/build_packages.sh

@@ -0,0 +1,250 @@
+#!/bin/bash
+
+BASEPATH=$(pwd)
+PKG_TARGET=/curator_packages
+WORKDIR=/tmp/curator
+PATCHFILE=cx_freeze.setup.py.patch
+CX_VER="4.3.4"
+CX_FILE="${CX_VER}.tar.gz"
+CX_PATH="anthony_tuininga-cx_freeze-f4b85046f841"
+INPUT_TYPE=python
+CATEGORY=python
+VENDOR=Elastic
+MAINTAINER="'Elastic Developers <info@elastic.co>'"
+PY_BIN="'/usr/bin/python'"
+PY_EASY_INSTALL="'/usr/bin/easy_install'"
+PY_BIN3="'/usr/bin/python3'"
+PY_EASY_INSTALL3="'/usr/bin/easy_install3'"
+VAF=${WORKDIR}/voluptuous_after_install.sh
+C_POST_INSTALL=${WORKDIR}/es_curator_after_install.sh
+C_PRE_REMOVE=${WORKDIR}/es_curator_before_removal.sh
+C_POST_REMOVE=${WORKDIR}/es_curator_after_removal.sh
+EXECUTOR=${WORKDIR}/execute_me.sh
+
+# Build our own package pre/post scripts
+sudo rm -rf ${WORKDIR} /opt/elasticsearch-curator
+mkdir -p ${WORKDIR}
+# Put the patchfile here before we do any cd
+cp ${PATCHFILE} ${WORKDIR}
+for file in ${VAF} ${C_POST_INSTALL} ${C_PRE_REMOVE} ${C_POST_REMOVE} ${EXECUTOR}; do
+  echo '#!/bin/bash' > ${file}
+  echo >> ${file}
+  chmod +x ${file}
+done
+
+echo "ln -s /opt/elasticsearch-curator/curator /usr/bin/curator" >> ${C_POST_INSTALL}
+echo "rm /usr/bin/curator" >> ${C_PRE_REMOVE}
+echo 'if [ -d "/opt/elasticsearch-curator" ]; then' >> ${C_POST_REMOVE}
+echo '  rmdir /opt/elasticsearch-curator' >> ${C_POST_REMOVE}
+echo 'fi' >> ${C_POST_REMOVE}
+
+ID=$(grep ^ID\= /etc/*release | awk -F\= '{print $2}' | tr -d \")
+VERSION_ID=$(grep ^VERSION_ID\= /etc/*release | awk -F\= '{print $2}' | tr -d \")
+if [ "${ID}x" == "x" ]; then
+  ID=$(cat /etc/*release | grep -v LSB | uniq | awk '{print $1}' | tr "[:upper:]" "[:lower:]" )
+  VERSION_ID=$(cat /etc/*release | grep -v LSB | uniq | awk '{print $3}' | awk -F\. '{print $1}')
+fi
+
+# build
+if [ "${1}x" == "x" ]; then
+  echo "Must provide version number"
+  exit 1
+else
+  FILE="v${1}.tar.gz"
+  cd ${WORKDIR}
+  wget https://github.com/elastic/curator/archive/${FILE}
+fi
+
+case "$ID" in
+  ubuntu|debian)
+	PKGTYPE=deb
+	PLATFORM=debian
+        PACKAGEDIR="${PKG_TARGET}/${1}/${PLATFORM}"
+	PYVER=2.7
+	DEPS="-d 'python:any << 3.0' -d 'python:any >= 2.7'"
+	DEPS3="-d 'python3:any << 4.0' -d 'python3:any >= 3.4'"
+	echo 'chmod o+r /usr/local/lib/python*/dist-packages/voluptuous-*.egg-info/*' >> ${VAF}
+	# Install patched version of cx_freeze
+	if [ "${CX_VER}" != "$(pip list | grep cx | awk '{print $2}' | tr -d '()')" ]; then
+	  rm -rf ${CX_PATH} ${CX_FILE}
+	  wget https://bitbucket.org/anthony_tuininga/cx_freeze/get/${CX_FILE}
+	  tar zxf ${CX_FILE}
+	  cd ${CX_PATH}
+	  patch < ${WORKDIR}/cx_freeze.setup.py.patch
+	  pip install -U --user .
+	  cd ${WORKDIR}
+	fi
+	;;
+  centos|rhel)
+	PKGTYPE=rpm
+        PLATFORM=centos
+	case "$VERSION_ID" in
+	  6)
+		PYVER=2.6
+		DEPS="-d 'python(abi) <= 2.7'"
+		echo 'chmod o+r /usr/lib/python2.6/site-packages/voluptuous-*-py2.6.egg-info/*' >> ${VAF}
+		;;
+          7)
+		PYVER=2.7
+		DEPS="-d 'python(abi) >= 2.7'"
+		echo 'chmod o+r /usr/lib/python2.7/site-packages/voluptuous-*-py2.7.egg-info/*' >> ${VAF}
+		;;
+ 	  *) echo "unknown system version: ${VERSION_ID}"; exit 1;;
+	esac
+        PACKAGEDIR="${PKG_TARGET}/${1}/${PLATFORM}/${VERSION_ID}"
+	pip install -U --user cx-Freeze
+	;;
+  *) echo "unknown system type: ${ID}"; exit 1;;
+esac
+
+if [ -e "/home/vagrant/.rvm/scripts/rvm" ]; then
+  source /home/vagrant/.rvm/scripts/rvm
+fi
+HAS_FPM=$(which fpm)
+if [ "${HAS_FPM}x" == "x" ]; then
+  gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+  curl -sSL https://get.rvm.io | bash -s stable
+  source /home/vagrant/.rvm/scripts/rvm
+  rvm install ruby
+  gem install fpm
+fi
+
+tar zxf ${FILE}
+
+
+mkdir -p ${PACKAGEDIR}
+cd curator-${1}
+pip install -U --user setuptools
+pip install -U --user requests_aws4auth
+pip install -U --user certifi
+pip install -U --user -r requirements.txt
+python setup.py build_exe
+sudo mv build/exe.linux-x86_64-${PYVER} /opt/elasticsearch-curator
+sudo chown -R root:root /opt/elasticsearch-curator
+cd ..
+fpm \
+ -s dir \
+ -t ${PKGTYPE} \
+ -n elasticsearch-curator \
+ -v ${1} \
+ --vendor ${VENDOR} \
+ --maintainer "${MAINTAINER}" \
+ --license 'Apache-2.0' \
+ --category tools \
+ --description 'Have indices in Elasticsearch? This is the tool for you!\n\nLike a museum curator manages the exhibits and collections on display, \nElasticsearch Curator helps you curate, or manage your indices.' \
+ --after-install ${C_POST_INSTALL} \
+ --before-remove ${C_PRE_REMOVE} \
+ --after-remove ${C_POST_REMOVE} \
+ --provides elasticsearch-curator \
+ --conflicts python-elasticsearch-curator \
+ --conflicts python3-elasticsearch-curator \
+/opt/elasticsearch-curator
+mv *.${PKGTYPE} ${PACKAGEDIR}
+
+if [ "${PLATFORM}" == "debian" ]; then
+  loop=2
+else
+  loop=1
+fi
+for ((i=0;i<loop;i++)); do
+  if [ $i -eq 0 ]; then
+    DEPENDENCIES="${DEPS}"
+    MY_PY=${PY_BIN}
+    MY_EASY=${PY_EASY_INSTALL}
+    PY_PREFIX=python
+  else
+    DEPENDENCIES="${DEPS3}"
+    MY_PY=${PY_BIN3}
+    MY_EASY=${PY_EASY_INSTALL3}
+    PY_PREFIX=python3
+  fi
+  cd ${PACKAGEDIR}
+
+  # We echo these out to file to guarantee proper argument recognition
+  # There are problems with quote encapsulation that aren't solvable in a
+  # single pass, unfortunately.
+  echo fpm \
+    -s ${INPUT_TYPE} -t ${PKGTYPE} --category ${CATEGORY} \
+    --vendor ${VENDOR} --maintainer ${MAINTAINER} ${DEPENDENCIES} \
+    --python-bin ${MY_PY} --python-easyinstall ${MY_EASY} \
+    --python-package-name-prefix ${PY_PREFIX} \
+    --license "'MPL-2.0'" \
+    --description "'Elastic build of certifi module'" \
+    certifi >> ${EXECUTOR}
+
+  echo fpm \
+    -s ${INPUT_TYPE} -t ${PKGTYPE} --category ${CATEGORY} \
+    --vendor ${VENDOR} --maintainer ${MAINTAINER} ${DEPENDENCIES} \
+    --python-bin ${MY_PY} --python-easyinstall ${MY_EASY} \
+    --python-package-name-prefix ${PY_PREFIX} \
+    --license "'BSD'" \
+    --description "'Elastic build of click module'" \
+    click >> ${EXECUTOR}
+
+  echo fpm \
+    -s ${INPUT_TYPE} -t ${PKGTYPE} --category ${CATEGORY} \
+    --vendor ${VENDOR} --maintainer ${MAINTAINER} ${DEPENDENCIES} \
+    --python-bin ${MY_PY} --python-easyinstall ${MY_EASY} \
+    --python-package-name-prefix ${PY_PREFIX} \
+    --license "'Apache-2.0'" \
+    --description "'Elastic build of elasticsearch module'" \
+    elasticsearch >> ${EXECUTOR}
+
+  echo fpm \
+    -s ${INPUT_TYPE} -t ${PKGTYPE} --category ${CATEGORY} \
+    --vendor ${VENDOR} --maintainer ${MAINTAINER} ${DEPENDENCIES} \
+    --python-bin ${MY_PY} --python-easyinstall ${MY_EASY} \
+    --python-package-name-prefix ${PY_PREFIX} \
+    --license "'PSF|ZPL'" \
+    --description "'Elastic build of setuptools module'" \
+    setuptools >> ${EXECUTOR}
+
+  echo fpm \
+    -s ${INPUT_TYPE} -t ${PKGTYPE} --category ${CATEGORY} \
+    --vendor ${VENDOR} --maintainer ${MAINTAINER} ${DEPENDENCIES} \
+    --python-bin ${MY_PY} --python-easyinstall ${MY_EASY} \
+    --python-package-name-prefix ${PY_PREFIX} \
+    --description "'Elastic build of PyYAML module'" \
+    pyyaml >> ${EXECUTOR}
+
+  echo fpm \
+    -s ${INPUT_TYPE} -t ${PKGTYPE} --category ${CATEGORY} \
+    --vendor ${VENDOR} --maintainer ${MAINTAINER} ${DEPENDENCIES} \
+    --python-bin ${MY_PY} --python-easyinstall ${MY_EASY} \
+    --after-install ${VAF} \
+    --python-package-name-prefix ${PY_PREFIX} \
+    --description "'Elastic build of voluptuous module'" \
+    voluptuous >> ${EXECUTOR}
+
+  echo fpm \
+    -s ${INPUT_TYPE} -t ${PKGTYPE} --category ${CATEGORY} \
+    --vendor ${VENDOR} --maintainer ${MAINTAINER} ${DEPENDENCIES} \
+    --python-bin ${MY_PY} --python-easyinstall ${MY_EASY} \
+    --python-package-name-prefix ${PY_PREFIX} \
+    --license "'MIT'" \
+    --description "'Elastic build of Urllib3 module'" \
+    urllib3 >> ${EXECUTOR}
+
+  echo cd ${WORKDIR}/curator-${1} >> ${EXECUTOR}
+
+  echo fpm \
+    -s ${INPUT_TYPE} -t ${PKGTYPE} --category ${CATEGORY} \
+    --vendor ${VENDOR} --maintainer ${MAINTAINER} ${DEPENDENCIES} \
+    --python-bin ${MY_PY} --python-easyinstall ${MY_EASY} \
+    --python-package-name-prefix ${PY_PREFIX} \
+    --conflicts elasticsearch-curator \
+    --license "'Apache-2.0'" \
+    --description "'Have indices in Elasticsearch? This is the tool for you!\n\nLike a museum curator manages the exhibits and collections on display, \nElasticsearch Curator helps you curate, or manage your indices.'" \
+    setup.py  >> ${EXECUTOR}
+
+  echo cd ${WORKDIR} >> ${EXECUTOR}
+done
+
+# Execute the file now that builds the packages.
+${EXECUTOR}
+# Copy the built packages from the curator-${1} directory to the package dir
+mv ${WORKDIR}/curator-${1}/*.${PKGTYPE} ${PACKAGEDIR}
+# cleanup
+rm ${VAF} ${C_POST_INSTALL} ${C_PRE_REMOVE} ${C_POST_REMOVE} ${EXECUTOR}
+# go back to where we started
+cd ${BASEPATH}

unix_packages/cx_freeze.setup.py.patch

@@ -0,0 +1,34 @@
+--- setup.py	2014-12-26 22:56:49.000000000 +0000
++++ fixed_setup.py	2016-10-04 17:28:03.182647980 +0000
+@@ -81,19 +81,18 @@
+                 extraArgs.append("-mwindows")
+         else:
+             vars = distutils.sysconfig.get_config_vars()
+-            if not vars.get("Py_ENABLE_SHARED", 0):
+-                libraryDirs.append(vars["LIBPL"])
+-                libraries.append("python%s.%s" % sys.version_info[:2])
+-                if vars["LINKFORSHARED"] and sys.platform != "darwin":
+-                    extraArgs.extend(vars["LINKFORSHARED"].split())
+-                if vars["LIBS"]:
+-                    extraArgs.extend(vars["LIBS"].split())
+-                if vars["LIBM"]:
+-                    extraArgs.append(vars["LIBM"])
+-                if vars["BASEMODLIBS"]:
+-                    extraArgs.extend(vars["BASEMODLIBS"].split())
+-                if vars["LOCALMODLIBS"]:
+-                    extraArgs.extend(vars["LOCALMODLIBS"].split())
++            libraryDirs.append(vars["LIBPL"])
++            libraries.append("python%s.%s" % sys.version_info[:2])
++            if vars["LINKFORSHARED"] and sys.platform != "darwin":
++                extraArgs.extend(vars["LINKFORSHARED"].split())
++            if vars["LIBS"]:
++                extraArgs.extend(vars["LIBS"].split())
++            if vars["LIBM"]:
++                extraArgs.append(vars["LIBM"])
++            if vars["BASEMODLIBS"]:
++                extraArgs.extend(vars["BASEMODLIBS"].split())
++            if vars["LOCALMODLIBS"]:
++                extraArgs.extend(vars["LOCALMODLIBS"].split())
+             extraArgs.append("-s")
+         self.compiler.link_executable(objects, fullName,
+                 libraries = libraries,