Skip to content

Commit

Permalink
Major overhaul of Jenkins build to fully integrate it into the
Browse files Browse the repository at this point in the history
existing build environment rather than trying to hack it in on
top.

A new variable USE_MOCK determines whether to use mock or not.  If
not, the old yroot/yinst build environment is now ifdef'd for the
non-USE_MOCK case.  This ifdef'ing will make it easy to eventually
retire the old system by cutting out the ifdef'd code.

I also added the ability to isolate the srpm build from the rpm
build.  They will use separate mock chroots.  That work is not
completely done yet though.
  • Loading branch information
Quentin Barnes committed Apr 19, 2015
1 parent 8c6456e commit 04fe515
Show file tree
Hide file tree
Showing 5 changed files with 167 additions and 131 deletions.
93 changes: 18 additions & 75 deletions linux/Makefile.jenkins
@@ -1,93 +1,36 @@
RPMARCH := $(shell rpm -E '%{_arch}')
RPMDIST := $(shell rpm -E '%{dist}')

DIST ?= $(patsubst .%,%,$(RPMDIST))
OSMAJNUM ?= $(patsubst el%,%,$(DIST))
PKGARCH ?= $(RPMARCH)
PKGNAME ?= iptables-daddr
PKGSPECFILE ?= $(PKGNAME).spec
MOCKPREFIX ?= l3dsr$(if $(JOB_NAME),-$(dir $(JOB_NAME)))
RESULT_DIR ?= results_$(MOCKPREFIX)/$(DIST)

MOCK = mock

define nl


endef

ifeq ($(DIST),el6)
kernel_vr = 2.6.32-71.el6
def_mock_args_extra += --define 'rhel_version 600'
endif

ifeq ($(DIST),el7)
kernel_vr = 3.10.0-123.el7
def_mock_args_extra += --define 'rhel_version 700'
endif

$(if ($(kernel_vr),),,$(error Can't determine $$kernel_vr.))

kernel_devel = kernel-devel-$(kernel_vr)
kernel_devel_rpm = $(kernel_devel).$(PKGARCH).rpm
kernel_rpm_repo_url = http://repo.linux.corp.yahoo.com/rh$(DIST)_kernels/$(PKGARCH)
# Start a build from Jenkins.
# Jenkins passes in BUILD_DIST, BUILD_ARCH, and BUILD_NUMBER.

def_make_args = \
NATIVE=1 \
CONFIG_ARGS='-r $(kernel_vr)' \
USE_MOCK=1 \
PLATFORMS='rh$(BUILD_DIST).$(BUILD_ARCH)'
BUILD_NUMBER='$(BUILD_NUMBER)'

def_mock_args = \
-r 'epel-$(OSMAJNUM)-$(PKGARCH)-yahoo' \
--uniqueext='$(MOCKPREFIX)' \
--resultdir '$(RESULT_DIR)/' \
--define 'kmod_kernel_version $(kernel_vr)'" \
--define 'kmodtool_list kmodtool.el5 kmodtool.el6'" \
$(def_mock_args_extra)

def_deploy_rpms_args = -f -T '$(RESULT_DIR)'
def_deploy_rpms_args = -f -T results_*

clean_files = $(kernel_devel_rpm)
clobber_files = $(clean_files) $(RESULT_DIR)
distclean_files = $(clobber_files) $(dir $(RESULT_DIR))
clean_files =
clobber_files = $(clean_files) \
$(wildcard results_*-build-src) \
$(wildcard results_*-build-$(BUILD_DIST)-$(BUILD_ARCH))
distclean_files = $(clobber_files)

commit-%: BUILD_NUMBER:=$(if $(BUILD_NUMBER),0.$(BUILD_NUMBER))
commit-%: %;
component-%: def_deploy_rpms_args += -R
component-%: %;

all: rpms

publish: all
./deploy-rpms $(def_deploy_rpms_args)

tarball:
all:
$(MAKE) $(def_make_args) $@

srpm: tarball mock_prep
$(MOCK) $(def_mock_args) \
--buildsrpm \
--arch noarch \
--spec '$(PKGSPECFILE)' \
--sources build-tarball

rpms: srpm mock_prep $(kernel_devel_rpm)
$(MOCK) $(def_mock_args) \
--yum-cmd install '$(kernel_devel_rpm)'
$(MOCK) $(def_mock_args) \
--no-clean \
--rebuild $(RESULT_DIR)/*.src.rpm

mock_prep:
$(MOCK) $(def_mock_args) --scrub=all

$(kernel_devel_rpm):
wget -N -q '$(kernel_rpm_repo_url)/$(kernel_devel_rpm)' || \
{ rm -f -- '$(kernel_devel_rpm)'; exit 1; }
publish: all
./deploy-rpms $(def_deploy_rpms_args) \
-T $(wildcard results_*-build-$(BUILD_DIST)-$(BUILD_ARCH))
./deploy-rpms $(def_deploy_rpms_args) \
-T $(wildcard results_*-build-src)

clean clobber distclean:
$(foreach f,$(wildcard $($@_files)),rm -rf -- '$f'$(nl))
$(MAKE) $(def_make_args) $@
$(foreach f,$(wildcard $($@_files)),rm -rf -- '$f'$(nl))

.DEFAULT: commit-all
.PHONY: all publish tarball srpm rpms mock_prep clean clobber distclean
.PHONY: all publish clean clobber distclean
51 changes: 40 additions & 11 deletions linux/mk/Makefile.sub
Expand Up @@ -18,39 +18,49 @@ tarball_mkvars = \
SPECFILE TARBALLFILE \
$(PACKAGE_SUB_EXTRA_VARS)


srpmfile = $(PKGNAME)$(DIST).src.rpm
rpmfile = $(PKGNAME)$(DIST).$(PKGARCH).rpm

tarball = $(TARBALLBUILDDIR)/$(TARBALLFILE)
starball = $(SRCBUILDDIR)/SOURCES/$(TARBALLFILE)
skmodtool_list = $(addprefix $(SRCBUILDDIR)/SOURCES/,$(KMODTOOL_LIST))
sspecfile = $(SRCBUILDDIR)/SPECS/$(SPECFILE)

ifeq ($(USE_MOCK),1)
ssrpm = $(SRCBUILDDIR)/$(srpmfile)
srpm = $(BINBUILDDIR)/$(srpmfile)
rpm = $(BINBUILDDIR)/$(rpmfile)
else
ssrpm = $(SRCBUILDDIR)/SRPMS/$(srpmfile)
srpm = $(BINBUILDDIR)/SRPMS/$(srpmfile)
rpm = $(BINBUILDDIR)/RPMS/$(PKGARCH)/$(rpmfile)
endif

rpmtmppath ?= /var/tmp/rpm-$(LOGNAME)

iptables_version_words = $(subst ., ,$(word 2,$(subst -, ,$(EXTENSIONSDIR))))
iptables_version_maj = $(word 1,$(iptables_version_words))
iptables_version_min = $(word 2,$(iptables_version_words))

rpmbuild_args = --define '_topdir $(topdir)' \
--define '_tmppath $(rpmtmppath)' \
--define 'kmod_name $(PACKAGE)' \
build_defs = --define 'kmod_name $(PACKAGE)' \
--define 'kmod_driver_version $(VERSION)' \
--define 'kmod_rpm_release $(RELEASE)' \
$(if $(KVARIANTS),--define 'kvariants $(subst ",\",$(KVARIANTS))') \
$(if $(OSMACRO),--define '$(OSMACRO) $(OSMACROVER)') \
$(if $(URL),--define 'url $(URL)') \
$(if $(KVERREL),--define 'kmod_kernel_version $(KVERREL)') \
$(if $(KMODTOOL_LIST),--define 'kmodtool_list $(KMODTOOL_LIST)') \
$(rpmbuild_extras)
$(if $(KMODTOOL_LIST),--define 'kmodtool_list $(KMODTOOL_LIST)')

rpm_build_args = --define '_topdir $(topdir)' \
--define '_tmppath $(rpmtmppath)' \
$(build_defs) \
$(rpm_build_extras)

clean_files = $(rpm) $(srpm) $(ssrpm) \
$(tarball) $(starball) \
$(sspecfile) $(skmodtool_list)
mock_build_args = $(build_args) $(mock_build_extras)

clean_files = $(rpm) $(srpm) $(ssrpm) \
$(tarball) $(starball) \
$(sspecfile) $(skmodtool_list)
clobber_files = $(BINBUILDDIR) $(SRCBUILDDIR) $(TARBALLBUILDDIR)
distclean_files = $(clobber_files)

Expand All @@ -66,24 +76,43 @@ $(rpm): topdir = $(PWD)/$(BINBUILDDIR)
$(rpm): $(srpm)
$(call mkdirs_cmd_call,$(BINBUILDDIR))
$(call mkrpmdirs_cmd_call,$(BINBUILDDIR))
ifneq ($(USE_MOCK),)
$(MOCK) $(MOCK_RPM_ARGS) \
--define 'dist $(if $(DIST),$(DIST),%{nil})' \
--target='$(PKGARCH)' \
$(mock_build_args) \
$(without_kabichk) \
--no-clean \
--rebuild $(ssrpm)
else
rpm -i --define '_topdir $(topdir)' '$(topdir)/SRPMS/$(<F)'
rpmbuild -bb \
--define 'dist $(if $(DIST),$(DIST),%{nil})' \
--target='$(PKGARCH)' \
$(rpmbuild_args) \
$(rpm_build_args) \
$(without_kabichk) \
'$(topdir)/SPECS/$(SPECFILE)'
endif

$(srpm): $(ssrpm)
$(call copy_file_cmd_call,$<,$@)

$(ssrpm): topdir = $(PWD)/$(SRCBUILDDIR)
$(ssrpm): $(starball) $(sspecfile) $(skmodtool_list)
$(call mkdirs_cmd_call,$(@D))
ifeq ($(USE_MOCK),1)
$(MOCK) $(MOCK_SRPM_ARGS) \
--target='noarch' \
$(mock_build_args) \
--buildsrpm \
--spec '$(sspecfile)' \
--sources '$(SRCBUILDDIR)/SOURCES'
else
rpmbuild -bs \
--target='noarch' \
$(rpmbuild_args) \
$(rpm_build_args) \
'$(topdir)/SPECS/$(SPECFILE)'
endif

$(starball): $(tarball)
$(call copy_file_cmd_call,$<,$@)
Expand Down
72 changes: 51 additions & 21 deletions linux/mk/Makefile.ybuild
Expand Up @@ -7,51 +7,65 @@ include mk/macros.mk
$(call varchklist_call,\
PKGARCH \
OSMACRO OSMACROVER OSDIST \
ROOTIMG ROOT \
KVARIANTS KVERREL \
BUILDRUNCMD)

ifneq ($(USE_MOCK),)
$(call varchklist_call,\
MOCK \
MOCK_SRPM_ARGS MOCK_RPM_ARGS)
else
$(call varchklist_call,\
ROOTIMG ROOT)
endif

ifeq ($(PKGARCH),i686)
rootarch = --i386
endif

yinstcmd ?= /usr/local/bin/yinst
rootcmd ?= /home/y/bin/yroot

BINBUILDRUNCMD = $(BUILDRUNCMD) '$(ROOT)'

rootsh = \
ifneq ($(USE_MOCK),)
rootsh = \
$(pound)! /bin/bash\
\neval \"\$$@\"
else
yinstcmd ?= /usr/local/bin/yinst
rootcmd ?= /home/y/bin/yroot

rootsh = \
$(pound)! /bin/bash\
\nrootname=\"\$$1\"\
\nshift\
\nexec $(rootcmd) \"\$$rootname\" --cmd \"\$$@\"

rootyinst = $(BINBUILDRUNCMD) \
rootyinst = $(BINBUILDRUNCMD) \
"$(yinstcmd) ls '$(1)' > /dev/null || \
$(yinstcmd) i '$(1)'"

rootkrpmbase = $(BINBUILDRUNCMD) \
rootkrpmbase = $(BINBUILDRUNCMD) \
"rpm -q '$(1)' > /dev/null || \
sudo rpm -i --oldpackage '$(2)/$(1).rpm'"

rootyum = $(BINBUILDRUNCMD) \
rootyum = $(BINBUILDRUNCMD) \
"rpm -q '$(1)' > /dev/null || \
sudo yum install -y $(2) '$(1)'"

rootyumu = $(call rootyum,$(1),--enablerepo=updates,y-updates)
rootyumu = $(call rootyum,$(1),--enablerepo=updates,y-updates)

ifeq ($(OSMACRO),rhel_version)
OSDISTMAJVER=$(word 1,$(subst 0, ,$(OSMACROVER)))
ifneq (,$(findstring YAHOO,$(OSDIST)))
kloc = $(OSDISTMAJVER).y-updates$(patsubst %,-%,$(subst i686,,$(PKGARCH)))
else
ifeq ($(OSDISTMAJVER),6)
rootkrpm = $(call rootkrpmbase,$(1),http://repo.linux.corp.yahoo.com/rhel6_kernels/x86_64)
ifeq ($(OSMACRO),rhel_version)
OSDISTMAJVER=$(word 1,$(subst 0, ,$(OSMACROVER)))
ifneq (,$(findstring YAHOO,$(OSDIST)))
kloc = $(OSDISTMAJVER).y-updates$(patsubst %,-%,$(subst i686,,$(PKGARCH)))
else
kloc = $(OSDISTMAJVER).updates$(patsubst %,-%,$(subst i686,,$(PKGARCH)))
krepourl := $(shell $(yinstcmd) which-dist)/yum/properties/ylinux/$(kloc)
rootkrpm = $(call rootkrpmbase,$(1),$(krepourl))
ifeq ($(OSDISTMAJVER),6)
rootkrpm = $(call rootkrpmbase,\
$(1),http://repo.linux.corp.yahoo.com/rhel6_kernels/x86_64)
else
kloc = $(OSDISTMAJVER).updates$(patsubst %,-%,$(subst i686,,$(PKGARCH)))
krepourl := $(shell $(yinstcmd) which-dist)/yum/properties/ylinux/$(kloc)
rootkrpm = $(call rootkrpmbase,$(1),$(krepourl))
endif
endif
endif
endif
Expand All @@ -62,7 +76,7 @@ kpkgs = $(if $(findstring "",$(KVARIANTS)),\
$(foreach k,$(filter-out "",$(KVARIANTS)),\
kernel-$k-devel-$(KVERREL).$(PKGARCH))

yumpkgs = rpm-build iptables-devel
yumpkgs = module-init-tools rpm-build iptables-devel
yumpkgsopt =

ifeq ($(OSMACRO),rhel_version)
Expand All @@ -71,7 +85,11 @@ ifeq ($(OSMACRO),rhel_version)
endif
ifeq ($(OSMACROVER),600)
yumpkgs += redhat-rpm-config
yumpkgsopt += kabi-whitelists
yumpkgsopt += kernel-abi-whitelists
endif
ifeq ($(OSMACROVER),700)
yumpkgs += redhat-rpm-config
yumpkgsopt += kernel-abi-whitelists
endif
endif

Expand All @@ -83,15 +101,27 @@ distclean_files = $(clobber_files)
all: create_buildenv prep_buildenv

prep_buildenv: $(BUILDRUNCMD)
ifneq ($(USE_MOCK),)
$(MOCK) $(MOCK_SRPM_ARGS) --init
$(MOCK) $(MOCK_RPM_ARGS) --init
$(MOCK) $(MOCK_RPM_ARGS) --install $(yumpkgs) $(kpkgs)
-$(MOCK) $(MOCK_RPM_ARGS) --install $(yumpkgsopt)
else
$(call rootyinst,yumbase)
$(foreach r,$(yumpkgs),$(call rootyum,$r) || \
{ echo >&2 "Build requires '$r' rpm."; exit 1; }$(nl))
$(foreach r,$(yumpkgsopt),$(call rootyum,$r) || exit 0$(nl))
$(foreach r,$(kpkgs),$(call rootkrpm,$r)$(nl))
endif

create_buildenv:
ifneq ($(USE_MOCK),)
$(rootcmd) --start '$(ROOT)' > /dev/null 2>&1 || \
$(rootcmd) --create '$(ROOT)' $(rootarch) $(ROOTIMG)
else
$(MOCK) $(MOCK_SRPM_ARGS) --scrub=all
$(MOCK) $(MOCK_RPM_ARGS) --scrub=all
endif

$(BUILDRUNCMD):
rm -f -- "$@"
Expand Down

0 comments on commit 04fe515

Please sign in to comment.