make: Refactor flavor BASEOS_ vars to DISTRO vars

It isn't very straightforward for a user that the BASEOS_REPO and
BASEOS_TAG specified in the flavor spec (`<ceph version>[point
release],<baseos repo>, <baseos tag>`) refer to the ceph-container
source directories. Fix this by changing BASEOS_REPO to DISTRO and
BASEOS_TAG to DISTRO_VERSION in the makefile. We still need BASEOS_REPO
and BASEOS_TAG, because we are planning to make these overrideable.

Signed-off-by: Blaine Gardner <blaine.gardner@suse.com>

Makefile

@@ -17,7 +17,6 @@
 # Build tunables
 
 # When updating these defaults, be sure to check that ALL_BUILDABLE_FLAVORS is updated
-# CEPH_VERSION,BASEOS_REPO,BASEOS_TAG
 FLAVORS ?= \
 	luminous,ubuntu,16.04 \
 	jewel,ubuntu,16.04 \
@@ -44,7 +43,6 @@ BASE_IMAGE ?= ""
 include maint-lib/makelib.mk
 
 # All flavor options that can be passed to FLAVORS
-# CEPH_VERSION,BASEOS_REPO,BASEOS_TAG
 ALL_BUILDABLE_FLAVORS := \
 	luminous,ubuntu,16.04 \
 	jewel,ubuntu,16.04 \
@@ -160,17 +158,16 @@ help:
 	@echo 'OPTIONS:'
 	@echo ''
 	@echo '  FLAVORS - ceph-container images to operate on in the form'
-	@echo '    <CEPH_VERSION>[CEPH_POINT_RELEASE],<BASEOS_REPO>,<BASEOS_TAG>'
+	@echo '    <CEPH_VERSION>[CEPH_POINT_RELEASE],<DISTRO>,<DISTRO_VERSION>'
 	@echo '    and multiple forms may be separated by spaces.'
-	@echo '      CEPH_VERSION - named ceph version (e.g., luminous, mimic)'
+	@echo '      CEPH_VERSION - Ceph named version part of the ceph-releases source path (e.g., luminous, mimic)'
 	@echo '      CEPH_POINT_RELEASE - Optional field to select a particular version of Ceph'
-	@echo '                           Regarding the package manager the version separator may vary :'
+	@echo '                           Regarding the package manager the version separator may vary:'
 	@echo '                             yum/dnf/zypper are using dash (e.g -12.2.2)'
 	@echo '                             apt is using an equal (e.g =12.2.2)'
-	@echo '      BASEOS_REPO  - The base image to use for the daemon-base container. This is also'
-	@echo '                      the distro path sourced from ceph-container (e.g., ubuntu, centos)'
-	@echo '      BASEOS_TAG   - Tagged version of the base repo to use. Also the distro version'
-	@echo '                      sourced from ceph-container (e.g., ubuntu:"16.04", centos:"7")'
+	@echo '      DISTRO - Distro part of the ceph-releases source path (e.g., ubuntu, centos)'
+	@echo '      DISTRO_VERSION - Distro version part of the ceph-releases source path'
+	@echo '                       (e.g., ubuntu/"16.04", centos/"7")'
 	@echo '    e.g., FLAVORS="luminous,ubuntu,16.04 jewel,ubuntu,14.04"'
 	@echo ''
 	@echo '  TAG_REGISTRY - The name of the registry to tag images with and to push images to.'

maint-lib/makelib.mk

@@ -5,7 +5,7 @@ HOST_ARCH ?= $(shell uname --machine)
 
 
 # Export all relevant environment variables from a flavor spec in the format:
-#   HOST_ARCH,CEPH_VERSION_SPEC,BASEOS_REPO,BASEOS_TAG
+#   HOST_ARCH,CEPH_VERSION_SPEC,DISTRO,DISTRO_VERSION
 # Note that CEPH_VERSION_SPEC is split into CEPH_VERSION and CEPH_POINT_RELEASE
 # Note that this format is the same as the FLAVORS spec format with HOST_ARCH prepended; it's
 #   necessary to pass HOST_ARCH in so we can do parallel cross-builds of different arches later.
@@ -17,12 +17,14 @@ $(shell bash -c 'set -eu ; \
 	ceph_version_spec="$(word 2, $(subst $(comma), ,$(1)))" ; \
 	set_var CEPH_VERSION       "$$(bash maint-lib/ceph_version.sh "$$ceph_version_spec" CEPH_VERSION)" ; \
 	set_var CEPH_POINT_RELEASE "$$(bash maint-lib/ceph_version.sh "$$ceph_version_spec" CEPH_POINT_RELEASE)" ; \
-	set_var BASEOS_REPO        "$(word 3, $(subst $(comma), , $(1)))" ; \
-	set_var BASEOS_TAG         "$(word 4, $(subst $(comma), , $(1)))" ; \
+	set_var DISTRO             "$(word 3, $(subst $(comma), , $(1)))" ; \
+	set_var DISTRO_VERSION     "$(word 4, $(subst $(comma), , $(1)))" ; \
 	set_var BASEOS_REGISTRY    "_" ; \
+	set_var BASEOS_REPO        "$$DISTRO" ; \
+	set_var BASEOS_TAG         "$$DISTRO_VERSION" ; \
 	set_var IMAGES_TO_BUILD    "$(IMAGES_TO_BUILD)" ; \
-
 	set_var STAGING_DIR       "staging/$$CEPH_VERSION$$CEPH_POINT_RELEASE-$$BASEOS_REPO-$$BASEOS_TAG-$$HOST_ARCH" ; \
+	\
 	base_img="$$BASEOS_REGISTRY/$$BASEOS_REPO:$$BASEOS_TAG" ; \
 	if [ -n "$(BASE_IMAGE)" ] ; then base_img="$(BASE_IMAGE)" ; fi ; \
 	set_var BASE_IMAGE        "$${base_img#_/}" ; \

maint-lib/stage.py

@@ -63,20 +63,20 @@ def main(CORE_FILES_DIR, CEPH_RELEASES_DIR):
     logging.info('GO_ARCH: {}'.format(getEnvVar('GO_ARCH')))
 
     CEPH_VERSION = getEnvVar('CEPH_VERSION')
-    BASEOS_REPO = getEnvVar('BASEOS_REPO')
-    BASEOS_TAG = getEnvVar('BASEOS_TAG')
+    DISTRO = getEnvVar('DISTRO')
+    DISTRO_VERSION = getEnvVar('DISTRO_VERSION')
     IMAGES_TO_BUILD = getEnvVar('IMAGES_TO_BUILD').split(' ')
     # STAGING_DIR is gotten globally
 
     # Search from least specfic to most specific
     path_search_order = [
         "{}".format(CORE_FILES_DIR),
         os.path.join(CEPH_RELEASES_DIR, 'ALL'),
-        os.path.join(CEPH_RELEASES_DIR, 'ALL', BASEOS_REPO),
-        os.path.join(CEPH_RELEASES_DIR, 'ALL', BASEOS_REPO, BASEOS_TAG),
+        os.path.join(CEPH_RELEASES_DIR, 'ALL', DISTRO),
+        os.path.join(CEPH_RELEASES_DIR, 'ALL', DISTRO, DISTRO_VERSION),
         os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION),
-        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, BASEOS_REPO),
-        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, BASEOS_REPO, BASEOS_TAG),
+        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, DISTRO),
+        os.path.join(CEPH_RELEASES_DIR, CEPH_VERSION, DISTRO, DISTRO_VERSION),
     ]
     logging.debug('Path search order: {}'.format(path_search_order))
 

maint-lib/stagelib/envglobals.py

@@ -14,11 +14,15 @@
 # Add new required variables simply by adding a new tuple to this setup
 # Allow lines to extend beyond 99 char limit by 6 chars to support output formatting to 100 cols
 REQUIRED_ENV_VARS = OrderedDict([
-    ('CEPH_VERSION',       'Ceph named version being built (e.g., luminous, mimic)'),  # noqa: E241
-    ('CEPH_POINT_RELEASE', 'Points to specific version of ceph (e.g -12.2.0) or empty'),  # noqa: E241,E501
+    ('CEPH_VERSION',       'Ceph named version part of the ceph-releases source path' +  # noqa: E241,E501
+                            ALIGNED_NEWLINE + '(e.g., luminous, mimic)'),  # noqa: E241
+    ('CEPH_POINT_RELEASE', 'Points to specific version of Ceph (e.g -12.2.0) or empty'),  # noqa: E241,E501
+    ('DISTRO',             'Distro part of the ceph-releases source path (e.g., ubuntu, centos)'),  # noqa: E241,E501
+    ('DISTRO_VERSION',     'Distro version part of the ceph-releases source path' +  # noqa: E241,E501
+                            ALIGNED_NEWLINE + '(e.g. in quotes, ubuntu/"16.04", centos/"7")'),
     ('HOST_ARCH',          'Architecture of binaries being built (e.g., amd64, arm32, arm64)'),  # noqa: E241,E501
     ('BASEOS_REGISTRY',    'Registry for the container base image (e.g., _ (x86_64), arm64v8 (aarch64))' +  # noqa: E241,E501
-                           ALIGNED_NEWLINE + 'There is a relation between HOST_ARCH and this value'),  # noqa: E241,E501
+                            ALIGNED_NEWLINE + 'There is a relation between HOST_ARCH and this value'),  # noqa: E241,E501
     ('BASEOS_REPO',        'Repository for the container base image (e.g., ubuntu, opensuse)'),  # noqa: E241,E501
     ('BASEOS_TAG',         'Tagged version of BASEOS_REPO container (e.g., 16.04, 42.3 respectively)'),  # noqa: E241,E501
     ('IMAGES_TO_BUILD',    'Container images to be built (usually should be "dockerfile daemon")'),  # noqa: E241,E501

src/STAGING_ENV_VARS.md

@@ -3,6 +3,8 @@ See `ceph-container/maint-lib/stagelib/envglobals.py` for most updated list and
 
 ## Required as input to staging
  - CEPH_VERSION
+ - DISTRO
+ - DISTRO_VERSION
  - CEPH_POINT_RELEASE
  - HOST_ARCH
  - BASEOS_REGISTRY

tests/stage-test/src/required-env-vars

@@ -0,0 +1,13 @@
+CEPH_VERSION: __ENV_[CEPH_VERSION]__
+CEPH_POINT_RELEASE: __ENV_[CEPH_POINT_RELEASE]__
+DISTRO: __ENV_[DISTRO]__
+DISTRO_VERSION: __ENV_[DISTRO_VERSION]__
+HOST_ARCH: __ENV_[HOST_ARCH]__
+BASEOS_REGISTRY: __ENV_[BASEOS_REGISTRY]__
+BASEOS_REPO: __ENV_[BASEOS_REPO]__
+BASEOS_TAG: __ENV_[BASEOS_TAG]__
+IMAGES_TO_BUILD: __ENV_[IMAGES_TO_BUILD]__
+STAGING_DIR: __ENV_[STAGING_DIR]__
+RELEASE: __ENV_[RELEASE]__
+DAEMON_BASE_IMAGE: __ENV_[DAEMON_BASE_IMAGE]__
+DAEMON_IMAGE: __ENV_[DAEMON_IMAGE]__

tests/stage-test/stage-key/daemon-base/required-env-vars

@@ -0,0 +1,13 @@
+CEPH_VERSION: luminous
+CEPH_POINT_RELEASE: -12.2.1-0
+DISTRO: ubuntu
+DISTRO_VERSION: 16.04
+HOST_ARCH: x86_64
+BASEOS_REGISTRY: _
+BASEOS_REPO: ubuntu.repo
+BASEOS_TAG: ubuntu.tag
+IMAGES_TO_BUILD: daemon-base daemon
+STAGING_DIR: tests/stage-test/staging/luminous-12.2.1-0-ubuntu.repo-ubuntu.tag-x86_64
+RELEASE: test-release
+DAEMON_BASE_IMAGE: test-reg/daemon-base:test-release-1
+DAEMON_IMAGE: test-reg/daemon:test-release-1

tests/stage-test/stage-key/daemon/required-env-vars

@@ -0,0 +1,13 @@
+CEPH_VERSION: luminous
+CEPH_POINT_RELEASE: -12.2.1-0
+DISTRO: ubuntu
+DISTRO_VERSION: 16.04
+HOST_ARCH: x86_64
+BASEOS_REGISTRY: _
+BASEOS_REPO: ubuntu.repo
+BASEOS_TAG: ubuntu.tag
+IMAGES_TO_BUILD: daemon-base daemon
+STAGING_DIR: tests/stage-test/staging/luminous-12.2.1-0-ubuntu.repo-ubuntu.tag-x86_64
+RELEASE: test-release
+DAEMON_BASE_IMAGE: test-reg/daemon-base:test-release-1
+DAEMON_IMAGE: test-reg/daemon:test-release-1

tests/stage-test/stage-key/files-sources

@@ -14,6 +14,7 @@ daemon-base/lu-daemon-base-test-file
 daemon-base/lu-test-file                                                          <-   tests/stage-test/ceph-releases/luminous/ubuntu/lu-test-file
 daemon-base/lum-daemon-base-test-file                                             <-   tests/stage-test/ceph-releases/luminous/ubuntu/16.04/daemon-base/lum-daemon-base-test-file
 daemon-base/lum-test-file                                                         <-   tests/stage-test/ceph-releases/luminous/ubuntu/16.04/lum-test-file
+daemon-base/required-env-vars                                                     <-   tests/stage-test/src/required-env-vars
 daemon-base/src-daemon-base-test-file                                             <-   tests/stage-test/src/daemon-base/src-daemon-base-test-file
 daemon-base/ubuntu-mimic-override                                                 <-   tests/stage-test/ceph-releases/luminous/ubuntu/16.04/ubuntu-mimic-override
 daemon/__DO_STUFF__                                                               <-   tests/stage-test/ceph-releases/ALL/ubuntu/16.04/daemon/__DO_STUFF__
@@ -29,6 +30,7 @@ daemon/lu-test-file
 daemon/lum-daemon-test-dir/lum-daemon-test-dir-test-file                          <-   tests/stage-test/ceph-releases/luminous/ubuntu/16.04/daemon/lum-daemon-test-dir/lum-daemon-test-dir-test-file
 daemon/lum-daemon-test-file                                                       <-   tests/stage-test/ceph-releases/luminous/ubuntu/16.04/daemon/lum-daemon-test-file
 daemon/lum-test-file                                                              <-   tests/stage-test/ceph-releases/luminous/ubuntu/16.04/lum-test-file
+daemon/required-env-vars                                                          <-   tests/stage-test/src/required-env-vars
 daemon/src-daemon-test-dir/src-daemon-test-dir-test-file                          <-   tests/stage-test/src/daemon/src-daemon-test-dir/src-daemon-test-dir-test-file
 daemon/src-daemon-test-file                                                       <- * tests/stage-test/src/daemon/src-daemon-test-file
 daemon/ubuntu-mimic-override                                                      <-   tests/stage-test/ceph-releases/luminous/ubuntu/16.04/ubuntu-mimic-override

tests/stage-test/stage-key/staging_output.txt

@@ -1,12 +1,14 @@
 
   CEPH_VERSION      : luminous
   CEPH_POINT_RELEASE: -12.2.1-0
+  DISTRO            : ubuntu
+  DISTRO_VERSION    : 16.04
   HOST_ARCH         : x86_64
   BASEOS_REGISTRY   : _
-  BASEOS_REPO       : ubuntu
-  BASEOS_TAG        : 16.04
+  BASEOS_REPO       : ubuntu.repo
+  BASEOS_TAG        : ubuntu.tag
   IMAGES_TO_BUILD   : daemon-base daemon
-  STAGING_DIR       : tests/stage-test/staging/luminous-12.2.1-0-ubuntu-16.04-x86_64
+  STAGING_DIR       : tests/stage-test/staging/luminous-12.2.1-0-ubuntu.repo-ubuntu.tag-x86_64
   RELEASE           : test-release
   DAEMON_BASE_IMAGE : test-reg/daemon-base:test-release-1
   DAEMON_IMAGE      : test-reg/daemon:test-release-1

tests/stage-test/test_staging.sh

@@ -8,10 +8,12 @@ CEPH_VERSION=$(maint-lib/ceph_version.sh "${CEPH_VERSION_SPEC}" "CEPH_VERSION")
 export CEPH_VERSION
 CEPH_POINT_RELEASE=$(maint-lib/ceph_version.sh "${CEPH_VERSION_SPEC}" "CEPH_POINT_RELEASE")
 export CEPH_POINT_RELEASE
+export DISTRO=ubuntu
+export DISTRO_VERSION=16.04
 export HOST_ARCH=x86_64
 export BASEOS_REGISTRY=_
-export BASEOS_REPO=ubuntu
-export BASEOS_TAG=16.04
+export BASEOS_REPO=ubuntu.repo
+export BASEOS_TAG=ubuntu.tag
 export STAGING_DIR=tests/stage-test/staging/${CEPH_VERSION}${CEPH_POINT_RELEASE}-${BASEOS_REPO}-${BASEOS_TAG}-${HOST_ARCH}
 export IMAGES_TO_BUILD="daemon-base daemon"
 export RELEASE='test-release'