From d5acd152aca178aca98bbc95d74aae7d5002df6b Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 1 Sep 2017 01:39:02 +0200 Subject: [PATCH 1/2] several changes to build binary+rpm in Docker and the os it is meant for --- Dockerfile | 10 +++-- Makefile | 103 ++++++++++++++++++++++++++++++++++++++++------- scripts/build.sh | 4 +- 3 files changed, 97 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6841dd3e..0fc63c8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,13 @@ FROM centos:centos7 -ARG RELEASE MAINTAINER Tim Vaillancourt + RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-4.noarch.rpm epel-release && \ yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all -RUN curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/$RELEASE/mongodb-consistent-backup.el7.centos.x86_64 && \ - chmod +x /usr/bin/mongodb-consistent-backup + +ADD build/rpm/RPMS/x86_64/mongodb_consistent_backup*.el*.centos.x86_64.rpm / +RUN yum localinstall -y /mongodb_consistent_backup*.el*.centos.x86_64.rpm && \ + yum clean all && rm -f /mongodb_consistent_backup*.el*.centos.x86_64.rpm + +USER mongodb_consistent_backup ENTRYPOINT ["mongodb-consistent-backup"] CMD ["--help"] diff --git a/Makefile b/Makefile index 7a2fe81b..f7bff4ab 100644 --- a/Makefile +++ b/Makefile @@ -2,22 +2,26 @@ # NAME=mongodb_consistent_backup +BIN_NAME?=mongodb-consistent-backup VERSION=$(shell cat VERSION | cut -d- -f1) +GIT_COMMIT?=$(shell git show 2>/dev/null | awk 'NR==1{print $$2}') PREFIX?=/usr/local +ARCH?=x86_64 BASEDIR?=$(DESTDIR)$(PREFIX) BINDIR?=$(BASEDIR)/bin SHAREDIR?=$(BASEDIR)/share DOCKER_TAG?="$(NAME):$(VERSION)" +DOCKER_BASE_IMAGE?=$(shell awk '/FROM/{print $$2}' Dockerfile) +MAKE_DIR=$(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +all: bin/$(BIN_NAME) -all: bin/mongodb-consistent-backup +bin/$(BIN_NAME): setup.py requirements.txt README.rst VERSION scripts/build.sh $(NAME)/*.py $(NAME)/*/*.py $(NAME)/*/*/*.py + BIN_NAME=$(BIN_NAME) GIT_COMMIT=$(GIT_COMMIT) PYTHON_BIN=$(PYTHON_BIN) VIRTUALENV_BIN=$(VIRTUALENV_BIN) bash scripts/build.sh -bin/mongodb-consistent-backup: setup.py requirements.txt README.rst VERSION scripts/build.sh $(NAME)/*.py $(NAME)/*/*.py $(NAME)/*/*/*.py - PYTHON_BIN=$(PYTHON_BIN) VIRTUALENV_BIN=$(VIRTUALENV_BIN) bash scripts/build.sh - -install: bin/mongodb-consistent-backup +install: bin/$(BIN_NAME) mkdir -p $(BINDIR) $(SHAREDIR)/$(NAME) || true - install -m 0755 bin/mongodb-consistent-backup $(BINDIR)/mongodb-consistent-backup + install -m 0755 bin/$(BIN_NAME) $(BINDIR)/mongodb-consistent-backup install -m 0644 conf/mongodb-consistent-backup.example.conf $(SHAREDIR)/$(NAME)/example.conf install -m 0644 LICENSE $(SHAREDIR)/$(NAME)/LICENSE install -m 0644 README.rst $(SHAREDIR)/$(NAME)/README.rst @@ -26,21 +30,90 @@ flake8: # Ignore long-lines and space-aligned = and : for now flake8 --ignore E221,E241,E501 $(PWD)/$(NAME) +rpm: bin/$(BIN_NAME) + mkdir -p $(MAKE_DIR)/build/rpm/SOURCES + cp -f $(MAKE_DIR)/{LICENSE,README.rst} build/rpm/SOURCES + cp -f $(MAKE_DIR)/bin/$(BIN_NAME) build/rpm/SOURCES/mongodb-consistent-backup + cp -f $(MAKE_DIR)/conf/mongodb-consistent-backup.example.conf build/rpm/SOURCES/mongodb-consistent-backup.conf + rpmbuild -D "_topdir $(MAKE_DIR)/build/rpm" -D "version $(VERSION)" -bb $(MAKE_DIR)/scripts/$(NAME).spec + uninstall: rm -f $(BINDIR)/mongodb-consistent-backup rm -rf $(SHAREDIR)/$(NAME) -rpm: bin/mongodb-consistent-backup - rm -rf build/rpm 2>/dev/null || true - mkdir -p build/rpm/SOURCES - cp -f $(PWD)/{LICENSE,README.rst} build/rpm/SOURCES - cp -f $(PWD)/bin/mongodb-consistent-backup build/rpm/SOURCES/mongodb-consistent-backup - cp -f $(PWD)/conf/mongodb-consistent-backup.example.conf build/rpm/SOURCES/mongodb-consistent-backup.conf - rpmbuild -D "_topdir $(PWD)/build/rpm" -D "version $(VERSION)" -bb scripts/$(NAME).spec +# Build CentOS7 RPM (in Docker) +build/rpm/RPMS/$(ARCH)/$(NAME)-$(VERSION)-1.el7.centos.$(ARCH).rpm: + mkdir -p $(MAKE_DIR)/build/rpm/RPMS/$(ARCH) + docker run --rm \ + -v "$(MAKE_DIR)/bin:/src/bin:Z" \ + -v "$(MAKE_DIR)/conf:/src/conf:Z" \ + -v "$(MAKE_DIR)/mongodb_consistent_backup:/src/mongodb_consistent_backup:Z" \ + -v "$(MAKE_DIR)/scripts:/src/scripts:Z" \ + -v "$(MAKE_DIR)/tmp/pip:/src/tmp/pip:Z" \ + -v "$(MAKE_DIR)/setup.py:/src/setup.py:Z" \ + -v "$(MAKE_DIR)/requirements.txt:/src/requirements.txt:Z" \ + -v "$(MAKE_DIR)/Makefile:/src/Makefile:Z" \ + -v "$(MAKE_DIR)/README.rst:/src/README.rst:Z" \ + -v "$(MAKE_DIR)/LICENSE:/src/LICENSE:Z" \ + -v "$(MAKE_DIR)/VERSION:/src/VERSION:Z" \ + -v "$(MAKE_DIR)/build/rpm/RPMS/$(ARCH):/src/build/rpm/RPMS/$(ARCH):Z" \ + -it centos:centos7 \ + /bin/bash -c "yum install -y python-devel python-virtualenv gcc make libffi-devel openssl-devel rpm-build && \ + make -C /src GIT_COMMIT=$(GIT_COMMIT) BIN_NAME=mongodb-consistent-backup.el7.centos.$(ARCH) rpm && \ + /src/bin/mongodb-consistent-backup.el7.centos.$(ARCH) --version" + +centos7: build/rpm/RPMS/$(ARCH)/$(NAME)-$(VERSION)-1.el7.centos.$(ARCH).rpm + +# Build Debian8 Binary (in Docker - .deb package soon!) +bin/mongodb-consistent-backup.debian8.$(ARCH): + docker run --rm \ + -v "$(MAKE_DIR)/bin:/src/bin:Z" \ + -v "$(MAKE_DIR)/conf:/src/conf:Z" \ + -v "$(MAKE_DIR)/mongodb_consistent_backup:/src/mongodb_consistent_backup:Z" \ + -v "$(MAKE_DIR)/scripts:/src/scripts:Z" \ + -v "$(MAKE_DIR)/tmp/pip:/src/tmp/pip:Z" \ + -v "$(MAKE_DIR)/setup.py:/src/setup.py:Z" \ + -v "$(MAKE_DIR)/requirements.txt:/src/requirements.txt:Z" \ + -v "$(MAKE_DIR)/Makefile:/src/Makefile:Z" \ + -v "$(MAKE_DIR)/README.rst:/src/README.rst:Z" \ + -v "$(MAKE_DIR)/LICENSE:/src/LICENSE:Z" \ + -v "$(MAKE_DIR)/VERSION:/src/VERSION:Z" \ + -it debian:jessie \ + /bin/bash -c "apt-get update && apt-get install -y python2.7-minimal python2.7-dev python-virtualenv gcc make libffi-dev libssl-dev && \ + make -C /src GIT_COMMIT=$(GIT_COMMIT) BIN_NAME=mongodb-consistent-backup.debian8.$(ARCH).tmp && \ + mv -vf /src/bin/mongodb-consistent-backup.debian8.$(ARCH).tmp /src/bin/mongodb-consistent-backup.debian8.$(ARCH) && \ + /src/bin/mongodb-consistent-backup.debian8.$(ARCH) --version" -docker: bin/mongodb-consistent-backup - docker build --no-cache --tag $(DOCKER_TAG) --build-arg "RELEASE=$(VERSION)" . +debian8: bin/mongodb-consistent-backup.debian8.$(ARCH) + +# Build Debian9 Binary (in Docker - .deb package soon!) +bin/mongodb-consistent-backup.debian9.$(ARCH): + docker run --rm \ + -v "$(MAKE_DIR)/bin:/src/bin:Z" \ + -v "$(MAKE_DIR)/conf:/src/conf:Z" \ + -v "$(MAKE_DIR)/mongodb_consistent_backup:/src/mongodb_consistent_backup:Z" \ + -v "$(MAKE_DIR)/scripts:/src/scripts:Z" \ + -v "$(MAKE_DIR)/tmp/pip:/src/tmp/pip:Z" \ + -v "$(MAKE_DIR)/setup.py:/src/setup.py:Z" \ + -v "$(MAKE_DIR)/requirements.txt:/src/requirements.txt:Z" \ + -v "$(MAKE_DIR)/Makefile:/src/Makefile:Z" \ + -v "$(MAKE_DIR)/README.rst:/src/README.rst:Z" \ + -v "$(MAKE_DIR)/LICENSE:/src/LICENSE:Z" \ + -v "$(MAKE_DIR)/VERSION:/src/VERSION:Z" \ + -it debian:stretch \ + /bin/bash -c "apt-get update && apt-get install -y python2.7-minimal python2.7-dev python-virtualenv gcc make libffi-dev libssl-dev && \ + make -C /src GIT_COMMIT=$(GIT_COMMIT) BIN_NAME=mongodb-consistent-backup.debian9.$(ARCH).tmp && \ + mv -vf /src/bin/mongodb-consistent-backup.debian9.$(ARCH).tmp /src/bin/mongodb-consistent-backup.debian9.$(ARCH) && \ + /src/bin/mongodb-consistent-backup.debian9.$(ARCH) --version" + +debian9: bin/mongodb-consistent-backup.debian9.$(ARCH) + +docker: build/rpm/RPMS/$(ARCH)/$(NAME)-$(VERSION)-1.el7.centos.$(ARCH).rpm + docker build --no-cache --tag $(DOCKER_TAG) . docker tag $(DOCKER_TAG) $(NAME):latest + docker run --rm -it $(DOCKER_TAG) --version + +release: centos7 debian8 debian9 docker clean: rm -rf bin build $(NAME).egg-info tmp 2>/dev/null diff --git a/scripts/build.sh b/scripts/build.sh index 2793a8e9..364cbec2 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -2,7 +2,7 @@ set -x -name=mongodb-consistent-backup +name=${BIN_NAME:-mongodb-consistent-backup} mod_name=mongodb_consistent_backup rootdir=$(readlink -f $(dirname $0)/..) srcdir=${rootdir}/${mod_name} @@ -15,6 +15,7 @@ venvdir=${builddir}/venv output_file=${bindir}/${name} require_file=${builddir}/requirements.txt version_file=${builddir}/VERSION +git_commit=${GIT_COMMIT:-unknown} python_bin=${PYTHON_BIN} if [ -z "$python_bin" ]; then @@ -73,7 +74,6 @@ if [ -d ${srcdir} ]; then exit 1 fi - git_commit=$(git show 2>/dev/null | awk 'NR==1{print $2}') if [ -z "$git_commit" ]; then echo "Warning: cannot find git commit hash!" else From 860001002c66afd7b4c2b1d0b98b8be10af02b42 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 1 Sep 2017 17:46:59 +0200 Subject: [PATCH 2/2] add-back Archive imports that should not have been removed in #198 flake8 cleanup --- mongodb_consistent_backup/Archive/Archive.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongodb_consistent_backup/Archive/Archive.py b/mongodb_consistent_backup/Archive/Archive.py index bd97812f..3f1bc013 100644 --- a/mongodb_consistent_backup/Archive/Archive.py +++ b/mongodb_consistent_backup/Archive/Archive.py @@ -1,3 +1,5 @@ +from mongodb_consistent_backup.Archive.Tar import Tar # NOQA +from mongodb_consistent_backup.Archive.Zbackup import Zbackup # NOQA from mongodb_consistent_backup.Pipeline import Stage