From bf4551ead1fc6e2b3611c074165bc75cfb57a5e7 Mon Sep 17 00:00:00 2001 From: Mark Feit Date: Fri, 24 Jul 2020 17:35:51 +0000 Subject: [PATCH] First cut at better build and install #106 --- .gitignore | 3 + Makefile | 33 +++++++++ Vagrantfile | 29 +------- build-esmond | 54 +++++++++++++++ rpm/esmond.spec | 61 ++++++----------- rpm/scripts/db/configure-pgsql95.sh | 2 +- rpm/scripts/db/upgrade-pgsql95.sh | 100 ---------------------------- 7 files changed, 111 insertions(+), 171 deletions(-) create mode 100644 Makefile create mode 100755 build-esmond delete mode 100755 rpm/scripts/db/upgrade-pgsql95.sh diff --git a/.gitignore b/.gitignore index de9ee07d..b3028358 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,6 @@ staticfiles # Build by-products /esmond-*.rpm + +# Editor droppings +*~ diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..72338ba9 --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +# +# Makefile for esmond +# +# (Doesn't really build anything, it's just here for the clean.) +# + +default: + @echo "Nothing to do here." + + +TO_CLEAN += \ + bin \ + configure_esmond \ + esmond.egg-info \ + include \ + lib \ + lib64 \ + pip-selfcheck.json \ + staticfiles \ + tsdb-data \ + vagrant-data \ + *.rpm + +FILE_MODE=644 +TO_CHMOD += \ + util/gen_django_secret_key.py \ + util/migrate_tastypie_keys.py \ + util/ps_remove_data.conf \ + util/ps_remove_data.py + +clean: + chmod $(FILE_MODE) $(TO_CHMOD) + rm -rf $(TO_CLEAN) *~ diff --git a/Vagrantfile b/Vagrantfile index 3a8eadc8..a575be3d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -206,34 +206,7 @@ EOF echo set -e - - # TODO: See if there's a way to get this location from Vagrant - cd /vagrant - - RPMBUILD="${HOME}/rpmbuild" - - yum -y install git rpm-build - - BRANCH=$(git rev-parse --abbrev-ref HEAD) - - RPMVERSION=$(rpmspec -q --qf "%{Version}\n" rpm/esmond.spec | head -1) - - for DIR in SOURCES SRPMS - do - mkdir -p "${RPMBUILD}/${DIR}" - done - - - git archive --format=tar --prefix="esmond-${RPMVERSION}/" "remotes/origin/${BRANCH}" \ - | gzip > "${RPMBUILD}/SOURCES/esmond-${RPMVERSION}.tar.gz" - - rpmbuild -bs rpm/esmond.spec - - yum-builddep -y ${RPMBUILD}/SRPMS/*.src.rpm - - rpmbuild -ba rpm/esmond.spec - - cp ${RPMBUILD}/RPMS/$(uname -m)/* . + /vagrant/build-esmond SHELL diff --git a/build-esmond b/build-esmond new file mode 100755 index 00000000..b7cb4673 --- /dev/null +++ b/build-esmond @@ -0,0 +1,54 @@ +#!/bin/sh + + +echo +echo Building Esmond +echo + +set -e + +yum -y install git rpm-build + +# TODO: See if there's a way to get this location from Vagrant +WORK_DIR=/tmp/esmond-build +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}" +rsync --archive --delete ./ "${WORK_DIR}" + +RPMBUILD="${HOME}/rpmbuild" +rm -rf "${RPMBUILD}" +for DIR in SOURCES SRPMS +do + mkdir -p "${RPMBUILD}/${DIR}" +done + +OLD_DIR="${PWD}" +cd "${WORK_DIR}" + +BRANCH=$(git rev-parse --abbrev-ref HEAD) + +RPMVERSION=$(rpmspec -q --qf "%{Version}\n" rpm/esmond.spec | head -1) + + +TARBALL="${RPMBUILD}/SOURCES/esmond-${RPMVERSION}.tar.gz" +rm -rf "${TARBALL}" + +TAR_BUILD=/tmp/esmond-tar-build +TAR_ESMOND="${TAR_BUILD}/esmond-${RPMVERSION}" +rm -rf "${TAR_BUILD}" +mkdir -p "${TAR_ESMOND}" +rsync \ + --exclude "*~" \ + --exclude "*.rpm" \ + --archive --delete ./ "${TAR_ESMOND}" +(cd "${TAR_BUILD}" && tar czf "${TARBALL}" .) +rm -rf "${TAR_BUILD}" + +rpmbuild -bs rpm/esmond.spec + +yum-builddep -y ${RPMBUILD}/SRPMS/*.src.rpm + +rpmbuild -ba rpm/esmond.spec + +cp ${RPMBUILD}/RPMS/$(uname -m)/* "${OLD_DIR}" + diff --git a/rpm/esmond.spec b/rpm/esmond.spec index 0fa46283..36555e88 100644 --- a/rpm/esmond.spec +++ b/rpm/esmond.spec @@ -68,13 +68,19 @@ Requires: policycoreutils-python Requires: cassandra20 Requires: httpd Requires: mod_ssl -Requires: esmond-database -Requires(post): esmond-database Requires: sqlite Requires: sqlite-devel Requires: memcached #java 1.7 needed for cassandra. dependency wrong in cassandra rpm. Requires: java-1.7.0-openjdk +Requires(post): drop-in + + +# Packages from older versions +Obsoletes: esmond-compat +Obsoletes: esmond-database + + %description @@ -83,48 +89,19 @@ uses a hybrid model for storing data using TSDB for time series data and an SQL database for everything else. All data is available via a REST style interface (as JSON) allowing for easy integration with other tools. -%package database-%{postgresql} -Summary: Esmond PostgreSQL %{postgresql_version} Database Plugin -Group: Development/Tools -Requires: %{postgresql} >= %{postgresql_version} -Requires: %{postgresql}-server >= %{postgresql_version} -Requires: %{postgresql}-devel >= %{postgresql_version} -Requires(post): %{postgresql} >= %{postgresql_version} -Requires(post): %{postgresql}-server >= %{postgresql_version} -Requires(post): %{postgresql}-devel >= %{postgresql_version} -Requires(post): drop-in -Provides: esmond-database - -%description database-%{postgresql} -Installs Postgresql 9.5 using one of the vendor RPMs. It will also try to migrate an -older version of the database to Postgresql 9.5 if it finds one present and there is not -already data . - -%package compat -Summary: Esmond Backward Compatibility -Group: Development/Tools -Requires: esmond >= 2.1 -Requires: esmond-database-%{postgresql} -Obsoletes: esmond < 2.1 - -%description compat -Transitions esmond instances prior to the split of database modules to new version %pre # Create the 'esmond' user /usr/sbin/groupadd -r esmond 2> /dev/null || : /usr/sbin/useradd -g esmond -r -s /sbin/nologin -c "Esmond User" -d /tmp esmond 2> /dev/null || : -%pre database-%{postgresql} -# Create the 'esmond' user -/usr/sbin/groupadd -r esmond 2> /dev/null || : -/usr/sbin/useradd -g esmond -r -s /sbin/nologin -c "Esmond User" -d /tmp esmond 2> /dev/null || : %prep %setup -q -n %{name}-%{version} %build + %install # Copy and build in place so that we know what the path in the various files # will be @@ -181,6 +158,14 @@ rm -rf rpms rm -f mkdevenv rm -f pylint.rc rm -f Vagrantfile +rm -rf vagrant +rm -rf .vagrant +rm -f build-esmond +rm -f Makefile +rm -rf debian +rm -rf .travis.yml + + # Install python libs so don't rely on pip connectivity during RPM install # NOTE: This part is why its not noarch @@ -188,7 +173,7 @@ rm -f Vagrantfile virtualenv-3.6 --prompt="(esmond)" . --system-site-packages --no-pip . bin/activate curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -python get-pip.py pip==18.1 +python3 get-pip.py pip==18.1 #Invoking pip using 'python -m pip' to avoid 128 char shebang line limit that pip can hit in build envs like Jenkins python3 -m pip install --install-option="--prefix=%{buildroot}%{install_base}" -r requirements.txt #not pretty but below is the best way I could find to remove references to buildroot @@ -253,11 +238,6 @@ if [ "$1" = "1" ]; then systemctl restart httpd fi -%post database-%{postgresql} -#try to update the database if this is a clean install -if [ "$1" = "1" ]; then - %{dbscript_base}/upgrade-pgsql95.sh -fi %postun if [ "$1" != "0" ]; then @@ -266,6 +246,7 @@ if [ "$1" != "0" ]; then fi %files +%defattr(-,esmond,esmond) %config(noreplace) %{config_base}/esmond.conf %attr(0755,esmond,esmond) %{install_base}/bin/* %attr(0755,esmond,esmond) %{install_base}/util/* @@ -302,12 +283,8 @@ fi /etc/httpd/conf.d/apache-esmond.conf %{_tmpfilesdir}/esmond.conf -%files database-%{postgresql} -%defattr(0644,esmond,esmond,0755) -%attr(0755,esmond,esmond) %{dbscript_base}/upgrade-pgsql95.sh %attr(0755,esmond,esmond) %{dbscript_base}/configure-pgsql95.sh -%files compat %changelog * Wed Mar 5 2014 Monte Goode .99-1 diff --git a/rpm/scripts/db/configure-pgsql95.sh b/rpm/scripts/db/configure-pgsql95.sh index d7709676..3db565f3 100755 --- a/rpm/scripts/db/configure-pgsql95.sh +++ b/rpm/scripts/db/configure-pgsql95.sh @@ -1,5 +1,5 @@ #!/bin/bash -PG_VERSION=9.5 +PG_VERSION=10 PG_BINDIR=/usr/pgsql-${PG_VERSION}/bin PG_DATADIR=/var/lib/pgsql/${PG_VERSION}/data PG_SERVICE_NAME="postgresql-${PG_VERSION}" diff --git a/rpm/scripts/db/upgrade-pgsql95.sh b/rpm/scripts/db/upgrade-pgsql95.sh deleted file mode 100755 index 8e48cf5b..00000000 --- a/rpm/scripts/db/upgrade-pgsql95.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -################################################################################ -# Script to upgrade an esmond postgresql 8.4 database to 9.5 -################################################################################ - -# Variables -OLD_DATADIR=/var/lib/pgsql/data -NEW_DATADIR=/var/lib/pgsql/9.5/data -OLD_BINDIR=/usr/bin -NEW_BINDIR=/usr/pgsql-9.5/bin - -# Make sure new data directory is clean -if [ "$(ls -A ${NEW_DATADIR})" ]; then - echo "Database directory at ${NEW_DATADIR} is not empty, so nothing to do. " - exit 0 -fi - -#Make sure there is something to migrate -if [ ! -d "$OLD_DATADIR" ] || [ ! "$(ls -A ${OLD_DATADIR})" ]; then - echo "No old data to migrate in ${OLD_DATADIR}, init ${NEW_DATADIR}. " - su -l postgres -c "${NEW_BINDIR}/initdb --pgdata='${NEW_DATADIR}' --auth='trust'" - if [ $? != 0 ]; then - exit 1 - fi - exit 0 -fi - -## -# Make sure both old and new are stopped -/etc/init.d/httpd stop 2> /dev/null -/etc/init.d/httpd24-httpd stop 2> /dev/null -sleep 1 -#make sure httpd is really dead -pkill -9 -f httpd -/etc/init.d/postgresql stop -/etc/init.d/postgresql-9.5 stop -sleep 5 -pkill -9 -f postgres - -## -# Temporarily update auth on old DB to allow upgrade to proceed -## -OLD_HBA_BAK=`mktemp` -cp ${OLD_DATADIR}/pg_hba.conf $OLD_HBA_BAK -if [ $? != 0 ]; then - exit 1 -fi -cat >${OLD_DATADIR}/pg_hba.conf <