PR #385 from @j-ogas: RPM build improvements (progress on #384)

doc-src/dev.rst

@@ -502,7 +502,9 @@ Building RPMs
 We maintain :code:`.spec` files and infrastructure for building RPMs in the
 Charliecloud source code. This is for two purposes:
 
-  1. We maintain our own Fedora RPMs.
+  1. We maintain our own Fedora RPMs (see `packaging guidelines
+     <https://docs.fedoraproject.org/en-US/packaging-guidelines/>`_).
+
   2. We want to be able to build an RPM of any commit.
 
 Item 2 is tested; i.e., if you break the RPM build, the test suite will fail.
@@ -590,11 +592,12 @@ commit, and it's not possible to create an RPM of uncommitted source code.
 Changelog maintenance
 ~~~~~~~~~~~~~~~~~~~~~
 
-The spec file changelog contains manually maintained release notes for all
-Charliecloud released versions and corresponding RPM releases. For released
-versions, :code:`build` verifies that the most recent changelog entry matches
-the given :code:`VERSION` argument. The timestamp is not automatically
-verified.
+The spec file contains a manually maintained changelog. Add a new entry for
+each new RPM release; do not include the Charliecloud release notes.
+
+For released versions, :code:`build` verifies that the most recent changelog
+entry matches the given :code:`VERSION` argument. The timestamp is not
+automatically verified.
 
 For other Charliecloud versions, :code:`build` adds a generic changelog entry
 with the appropriate version stating that it's a pre-release RPM.

doc-src/test.rst

@@ -16,7 +16,7 @@ Getting started
 
 Charliecloud's tests are based in the directory :code:`test`, which is either
 at the top level of the source code or installed at
-:code:`$PREFIX/share/doc/charliecloud`. To run them, go there::
+:code:`$PREFIX/libexec/charliecloud`. To run them, go there::
 
   $ cd test
 

packaging/fedora/build

@@ -14,11 +14,12 @@ import re
 import shutil
 import socket
 import subprocess
+import sys
 import time
 
 CH_BASE = os.path.abspath(os.path.dirname(__file__) + "/../..")
 CH_RUN  = [CH_BASE + "/bin/ch-run"]
-PACKAGES = ["charliecloud", "charliecloud-debuginfo", "charliecloud-doc"]
+PACKAGES = ["charliecloud", "charliecloud-debuginfo", "charliecloud-test"]
 ARCH = platform.machine()
 
 def main():
@@ -53,7 +54,7 @@ def main():
       rpm_release = "0"
    else:
       m           = re.search(r"([0-9.]+)-([0-9]+)", args.version)
-      branch      = "v" + m.group(1)
+      commit      = "v" + m.group(1)
       rpm_release = m.group(2)
 
    # Create rpmbuild root
@@ -96,7 +97,12 @@ def main():
 
    # Copy and patch spec file.
    rpm_vr = "%s-%s" % (ch_version, rpm_release)
-   spec   = "charliecloud-%s.spec" % rpm_vr
+   # Fedora requires no version in spec file. Add a version for pre-release
+   # specs to make it hard to upload one to Fedora by mistake.
+   if ("~pre" not in ch_version):
+      spec = "charliecloud.spec"
+   else:
+      spec = "charliecloud-%s.spec" % rpm_vr
    with open("%s/packaging/fedora/charliecloud.spec" % CH_BASE, "rt") as in_, \
         open("%s/%s" % (rpm_specs, spec), "wt") as out:
       print("# writing %s" % out.name)
@@ -115,8 +121,11 @@ def main():
 - Pre-release package. See Git history for what is going on.
 """ % (timestamp, name, moniker, domain, rpm_vr))
       else:
-         # Last changelog entry must match version.
-         assert False, "unimplemented"
+         # Verify requested version matches changelog.
+         m = re.search(r"%changelog\n.+?([0-9.-]+)\n", t)
+         if (m.group(1) != rpm_vr):
+            print("requested version %s != changelog %s" % (rpm_vr, m.group(1)))
+            sys.exit(1)
       out.write(t)
 
    # Prepare build and rpmlint arguments.

packaging/fedora/charliecloud.rpmlintrc

@@ -19,18 +19,23 @@ addFilter(r'missing-call-to-chdir-with-chroot')
 # The only files under /usr/lib are those placed there by rpmbuild.
 addFilter(r'only-non-binary-in-usr-lib')
 
-# charliecloud-doc
-
-# Charliecloud is a container runtime. The libsotest objects are test suite
-# resources that are injected into a container (guest), these objects are not
-# used on the host.
+# charliecloud-test
+
+# Charliecloud is a container runtime. These shared objects are never used in 
+# the host environment; rather, they are compiled by the test suite (both 
+# running and examination of which serve as end-user documentation) and injected
+# into the container (guest) via utility script 'ch-fromhost'. The ldconfig 
+# links are generated inside the container runtime environment. For more 
+# information, see the test file: test/run/ch-fromhost.bats (line 108).
 addFilter(r'no-ldconfig-symlink')
 addFilter(r'library-without-ldconfig-postin')
 addFilter(r'library-without-ldconfig-postun')
 
-# The example/*.c and test/*.c files are example files that illustrate various
-# details of the charliecloud runtime. For all intents and purposes, they are
-# part of our documentation.
+# The test suite has a few C files, e.g. userns.c, pivot_root.c, 
+# chroot-escape.c, sotest.c, setgroups.c, mknods.c, setuid.c, etc., that 
+# document -- line-by-line in some cases -- various components of the open source
+# runtime. These C files serve to show end users how containers work; some of
+# them are used explicitly during test suite runtime.
 addFilter(r'devel-file-in-non-devel-package')
 
 # The symlink to /usr/bin is created and does exist.

packaging/fedora/charliecloud.spec

@@ -1,19 +1,40 @@
+# Charliecloud fedora package spec file
+#
+# Contributors:
+#    Dave Love           @loveshack
+#    Michael Jennings    @mej
+#    Jordan Ogas         @jogas
+#    Reid Priedhorksy    @reidpr
+
+# Don't try to compile python files with /usr/bin/python
+%{?el7:%global __python %__python3}
+
+# Fedora does not allow SUSE conditionals, thus we define libexecdir to ensure
+# consistency.
+%define _libexecdir %{_prefix}/libexec
+
+# Specify python version of a given file
+%define versionize_script() (sed -i 's,/env python,/env %1,g' %2)
+
+%{!?build_cflags:%global build_cflags $RPM_OPT_FLAGS}
+%{!?build_ldflags:%global build_ldflags %nil}
+
 Name:           charliecloud
 Version:        @VERSION@
 Release:        @RELEASE@%{?dist}
 Summary:        Lightweight user-defined software stacks for high-performance computing
 License:        ASL 2.0
 URL:            https://hpc.github.io/%{name}/
-Source0:        https://github.com/hpc/%{name}/archive/v%{version}/%{name}-%{version}.tar.gz
-BuildRequires:  gcc  >= 4.8.5
-BuildRequires:  make >= 3.82
+Source0:        https://github.com/hpc/%{name}/releases/download/v%{version}/%{name}-%{version}.tar.gz
+BuildRequires:  gcc
 
-%package doc
-Summary:        Charliecloud examples and test suite
-Requires:       %{name} = %{version}
-Requires:       bats >= 0.4.0
-Requires:       bash >= 4.2.46
-Requires:       wget >= 1.14
+%package   test
+Summary:   Charliecloud examples and test suite
+Requires:  %{name}%{?_isa} = %{version}-%{release}
+Requires:  bats
+Requires:  bash
+Requires:  wget
+Requires:  /usr/bin/python3
 
 %description
 Charliecloud uses Linux user namespaces to run containers with no privileged
@@ -26,57 +47,59 @@ a standard Linux filesystem tree.
 
 For more information: https://hpc.github.io/charliecloud/
 
-%description doc
-Charliecloud test suite and examples.
-
-# Voodoo to stop our python scripts from being byte compiled on Centos7, which
-# otherwise results in rpmbuild failing to build the package. see:
-# https://github.com/scylladb/scylla/issues/2235
-%global __os_install_post    \
-     /usr/lib/rpm/redhat/brp-compress \
-     %{!?__debug_package:\
-     /usr/lib/rpm/redhat/brp-strip %{__strip} \
-     /usr/lib/rpm/redhat/brp-strip-comment-note %{__strip} %{__objdump} \
-     } \
-     /usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \
-     %{!?__jar_repack:/usr/lib/rpm/redhat/brp-java-repack-jars} \
-%{nil}
+%description test
+Charliecloud test suite and examples. The test suite takes advantage of
+container image builders such as Docker, Skopeo, and Buildah.
 
 %prep
 %setup -q
 
+%{versionize_script python3 test/make-auto}
+%{versionize_script python3 test/make-perms-test}
+
 %build
-%{__make} %{?mflags}
+%make_build CFLAGS="%build_cflags -std=c11 -pthread" LDFLAGS="%build_ldflags"
 
 %install
-%{__make} %{?mflags_install} install PREFIX=%{_prefix} DESTDIR=%{buildroot}
+%make_install PREFIX=%{_prefix}
+
+cat > README.EL7 <<EOF
+For RHEL7 you must increase the number of available user namespaces to a non-
+zero number (note the number below is taken from the default for RHEL8):
+
+  echo user.max_user_namespaces=3171 >/etc/sysctl.d/51-userns.conf
+  systemctl -p
+
+Note for versions below RHEL7.6, you will also need to enable user namespaces:
+
+  grubby --args=namespace.unpriv_enable=1 --update-kernel=ALL
+  systemctl -p
+EOF
+
+cat > README.TESTS <<EOF
+Charliecloud comes with a fairly comprehensive Bats test suite. For testing
+instructions visit: https://hpc.github.io/charliecloud/test.html
+EOF
+
+%check
 
 %files
-# Documentation
-%doc %{_datadir}/doc/%{name}/LICENSE
-%doc %{_datadir}/doc/%{name}/README.rst
-%doc %{_mandir}/man1/ch*
+%license LICENSE
+%doc README.rst %{?el7:README.EL7}
+%{_mandir}/man1/ch*
+%exclude %{_datadir}/doc/%{name}
 
-# Helper scripts
+# Binaries and helper scripts
 %{_libexecdir}/%{name}/base.sh
 %{_libexecdir}/%{name}/version.sh
-%{_bindir}/ch-build
-%{_bindir}/ch-build2dir
-%{_bindir}/ch-docker2tar
-%{_bindir}/ch-fromhost
-%{_bindir}/ch-pull2dir
-%{_bindir}/ch-pull2tar
-%{_bindir}/ch-tar2dir
-
-# Binaries
-%{_bindir}/ch-run
-%{_bindir}/ch-ssh
-
-%files doc
-%doc %{_datadir}/doc/%{name}/LICENSE
-%doc %{_datadir}/doc/%{name}/README.rst
-%doc %{_datadir}/doc/%{name}/html
+%{_bindir}/ch-*
+
+%files test
+%doc README.TESTS
 %{_libexecdir}/%{name}/examples
 %{_libexecdir}/%{name}/test
+%exclude %{_datadir}/doc/%{name}
 
 %changelog
+* Thu Mar 14 2019 <jogas@lanl.gov> @VERSION@-@RELEASE@
+- Add initial Fedora/EPEL package.

test/Dockerfile.centos7

@@ -9,6 +9,7 @@ RUN yum -y install \
            bats \
            gcc \
            make \
+           python36 \
            rpm-build \
            rpmlint \
            wget

test/README.md

@@ -0,0 +1,2 @@
+Charliecloud comes with a fairly comprehensive Bats test suite. For testing
+instructions visit: https://hpc.github.io/charliecloud/test.html

test/run/build-rpms.bats

@@ -17,7 +17,7 @@ load ../common
     [[ $status -eq 0 ]]
     [[ $output = *'charliecloud-'* ]]
     [[ $output = *'charliecloud-debuginfo-'* ]]
-    [[ $output = *'charliecloud-doc-'* ]]
+    [[ $output = *'charliecloud-test-'* ]]
     run ch-run "$img" -- rpm -ql "charliecloud"
     echo "$output"
     [[ $status -eq 0 ]]
@@ -29,22 +29,22 @@ load ../common
     [[ $status -eq 0 ]]
     [[ $output = *'/usr/lib/debug/usr/bin/ch-run.debug'* ]]
     [[ $output = *'/usr/lib/debug/usr/libexec/charliecloud/test/sotest/lib/libsotest.so.1.0.debug'* ]]
-    run ch-run "$img" -- rpm -ql "charliecloud-doc"
+    run ch-run "$img" -- rpm -ql "charliecloud-test"
     echo "$output"
     [[ $status -eq 0 ]]
     [[ $output = *'/usr/libexec/charliecloud/examples/mpi/lammps/Dockerfile'* ]]
     [[ $output = *'/usr/libexec/charliecloud/test/Build.centos7xz'* ]]
     [[ $output = *'/usr/libexec/charliecloud/test/sotest/lib/libsotest.so.1.0'* ]]
 
     # Uninstall to avoid interfering with the rest of the test suite.
-    run ch-run -w "$img" -- rpm -v --erase charliecloud-doc \
+    run ch-run -w "$img" -- rpm -v --erase charliecloud-test \
                                            charliecloud-debuginfo \
                                            charliecloud
     echo "$output"
     [[ $status -eq 0 ]]
     [[ $output = *'charliecloud-'* ]]
     [[ $output = *'charliecloud-debuginfo-'* ]]
-    [[ $output = *'charliecloud-doc-'* ]]
+    [[ $output = *'charliecloud-test-'* ]]
 
     # All gone?
     run ch-run "$img" -- rpm -qa "charliecloud*"