Skip to content

Commit

Permalink
ODCS compose enhancements
Browse files Browse the repository at this point in the history 
Fixes a few things:

1. Makes it possible to request ODCS composes for local builder
   other than Docker
2. Makes it possible to request ODCS compose using community service

Fixes cekit#491
  • Loading branch information
@goldmann
goldmann committed Apr 24, 2019
1 parent 84247e0 commit 94f824d
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 83 deletions.
69 changes: 63 additions & 6 deletions cekit/generator/base.py
View file
Expand Up @@ -2,8 +2,12 @@

import logging
import os
import platform
import subprocess
import shutil

import yaml

from jinja2 import Environment, FileSystemLoader

from cekit import tools
Expand All @@ -25,11 +29,11 @@ class Generator(object):
Args:
descriptor_path - path to an image descriptor
target - path to target directory
builder - builder type
overrides - path to overrides file (can be None)
params - dictionary of builder specific parameters
"""

ODCS_HIDDEN_REPOS_FLAG = 'include_unpublished_pulp_repos'

def __init__(self, descriptor_path, target, overrides):
self._descriptor_path = descriptor_path
self._overrides = []
Expand Down Expand Up @@ -152,7 +156,6 @@ def copy_modules(self):
# write out the module with any overrides
module.write(os.path.join(dest, "module.yaml"))


def get_tech_preview_overrides(self):
class TechPreviewOverrides(Overrides):
def __init__(self, generator):
Expand Down Expand Up @@ -296,6 +299,63 @@ def prepare_repositories(self, builder):
else:
self.image['packages']['set_url'] = injected_repos

def _prepare_content_sets(self, content_sets):
if not content_sets:
return False

arch = platform.machine()

if arch not in content_sets:
raise CekitError("There are no content_sets defined for platform '{}'!".format(arch))

repos = ' '.join(content_sets[arch])

try:
# ideally this will be API for ODCS, but there is no python3 package for ODCS
cmd = ['/usr/bin/odcs']
odcs_service_type = "Fedora"

if CONFIG.get('common', 'redhat'):
odcs_service_type = "Red Hat"
cmd.append('--redhat')

LOGGER.info("Using {} ODCS service to created composes".format(odcs_service_type))

cmd.append('create')

compose = self.image.get('osbs', {}).get(
'configuration', {}).get('container', {}).get('compose', {})

if compose.get(Generator.ODCS_HIDDEN_REPOS_FLAG, False):
cmd.extend(['--flag', Generator.ODCS_HIDDEN_REPOS_FLAG])

cmd.extend(['pulp', repos])

LOGGER.debug("Creating ODCS content set via '%s'" % " ".join(cmd))

output = subprocess.check_output(cmd).decode()
normalized_output = '\n'.join(output.replace(" u'", " '")
.replace(' u"', ' "')
.split('\n')[1:])

odcs_result = yaml.safe_load(normalized_output)

if odcs_result['state'] != 2:
raise CekitError("Cannot create content set: '%s'"
% odcs_result['state_reason'])

repo_url = odcs_result['result_repofile']
return repo_url

except CekitError as ex:
raise ex
except OSError as ex:
raise CekitError("ODCS is not installed, please install 'odcs-client' package")
except subprocess.CalledProcessError as ex:
raise CekitError("Cannot create content set: '%s'" % ex.output)
except Exception as ex:
raise CekitError('Cannot create content set!', ex)

def _handle_repository(self, repo):
"""Process and prepares all v2 repositories.
Expand Down Expand Up @@ -326,9 +386,6 @@ def _handle_repository(self, repo):

return False

def _prepare_content_sets(self, content_sets):
raise NotImplementedError("Content sets repository injection not implemented!")

def _prepare_repository_rpm(self, repo):
raise NotImplementedError("RPM repository injection was not implemented!")

Expand Down
62 changes: 0 additions & 62 deletions cekit/generator/docker.py
View file
@@ -1,9 +1,5 @@
import logging
import os
import platform
import subprocess

import yaml

from cekit.config import Config
from cekit.errors import CekitError
Expand All @@ -15,8 +11,6 @@

class DockerGenerator(Generator):

ODCS_HIDDEN_REPOS_FLAG = 'include_unpublished_pulp_repos'

def __init__(self, descriptor_path, target, overrides):
super(DockerGenerator, self).__init__(descriptor_path, target, overrides)
self._fetch_repos = True
Expand All @@ -38,62 +32,6 @@ def dependencies():

return deps

def _prepare_content_sets(self, content_sets):
if not content_sets:
return False

if not config.get('common', 'redhat'):
return False

arch = platform.machine()

if arch not in content_sets:
raise CekitError("There are no content_sets defined for platform '{}'!".format(arch))

repos = ' '.join(content_sets[arch])

try:
# ideally this will be API for ODCS, but there is no python3 package for ODCS
cmd = ['/usr/bin/odcs']

if config.get('common', 'redhat'):
cmd.append('--redhat')

cmd.append('create')

compose = self.image.get('osbs', {}).get(
'configuration', {}).get('container', {}).get('compose', {})

if compose.get(DockerGenerator.ODCS_HIDDEN_REPOS_FLAG, False):
cmd.extend(['--flag', DockerGenerator.ODCS_HIDDEN_REPOS_FLAG])

cmd.extend(['pulp', repos])

logger.debug("Creating ODCS content set via '%s'" % " ".join(cmd))

output = subprocess.check_output(cmd).decode()
normalized_output = '\n'.join(output.replace(" u'", " '")
.replace(' u"', ' "')
.split('\n')[1:])

odcs_result = yaml.safe_load(normalized_output)

if odcs_result['state'] != 2:
raise CekitError("Cannot create content set: '%s'"
% odcs_result['state_reason'])

repo_url = odcs_result['result_repofile']
return repo_url

except CekitError as ex:
raise ex
except OSError as ex:
raise CekitError("ODCS is not installed, please install 'odcs-client' package")
except subprocess.CalledProcessError as ex:
raise CekitError("Cannot create content set: '%s'" % ex.output)
except Exception as ex:
raise CekitError('Cannot create content set!', ex)

def _prepare_repository_rpm(self, repo):
# no special handling is needed here, everything is in template
pass
Expand Down
24 changes: 9 additions & 15 deletions tests/test_unit_generator_docker.py
View file
Expand Up @@ -74,12 +74,6 @@ def setup_function():
Config.cfg = {'common': {}}


def test_prepare_content_sets_should_return_early_when_redhat_is_not_set(tmpdir):
with docker_generator(tmpdir) as generator:
with cekit_config(redhat=False):
assert generator._prepare_content_sets({'something': []}) is False


def test_prepare_content_sets_should_not_fail_when_cs_is_none(tmpdir):
with docker_generator(tmpdir) as generator:
with cekit_config(redhat=True):
Expand All @@ -93,7 +87,7 @@ def test_prepare_content_sets_should_not_fail_when_cs_is_empty(tmpdir):


def test_prepare_content_sets_should_fail_when_cs_are_note_defined_for_current_platform(tmpdir, mocker):
mocker.patch('cekit.generator.docker.platform.machine',
mocker.patch('cekit.generator.base.platform.machine',
return_value="current_platform")

with docker_generator(tmpdir) as generator:
Expand All @@ -104,10 +98,10 @@ def test_prepare_content_sets_should_fail_when_cs_are_note_defined_for_current_p


def test_prepare_content_sets_should_request_odcs(tmpdir, mocker):
mocker.patch('cekit.generator.docker.platform.machine',
mocker.patch('cekit.generator.base.platform.machine',
return_value="current_platform")
mock_odcs = mocker.patch(
'cekit.generator.docker.subprocess.check_output', return_value=odcs_fake_resp)
'cekit.generator.base.subprocess.check_output', return_value=odcs_fake_resp)

with docker_generator(tmpdir) as generator:
generator.image = {'osbs': {'configuration': {'container': {'compose': {}}}}}
Expand All @@ -120,10 +114,10 @@ def test_prepare_content_sets_should_request_odcs(tmpdir, mocker):


def test_prepare_content_sets_should_request_odcs_with_hidden_repos_flag(tmpdir, mocker):
mocker.patch('cekit.generator.docker.platform.machine',
mocker.patch('cekit.generator.base.platform.machine',
return_value="current_platform")
mock_odcs = mocker.patch(
'cekit.generator.docker.subprocess.check_output', return_value=odcs_fake_resp)
'cekit.generator.base.subprocess.check_output', return_value=odcs_fake_resp)

with docker_generator(tmpdir) as generator:
generator.image = {'osbs': {'configuration': {'container': {
Expand All @@ -138,10 +132,10 @@ def test_prepare_content_sets_should_request_odcs_with_hidden_repos_flag(tmpdir,


def test_prepare_content_sets_should_handle_incorrect_state(tmpdir, mocker):
mocker.patch('cekit.generator.docker.platform.machine',
mocker.patch('cekit.generator.base.platform.machine',
return_value="current_platform")
mock_odcs = mocker.patch(
'cekit.generator.docker.subprocess.check_output', return_value=odcs_fake_invalid_resp)
'cekit.generator.base.subprocess.check_output', return_value=odcs_fake_invalid_resp)

with docker_generator(tmpdir) as generator:
generator.image = {'osbs': {'configuration': {'container': {
Expand All @@ -156,10 +150,10 @@ def test_prepare_content_sets_should_handle_incorrect_state(tmpdir, mocker):


def test_prepare_content_sets_should_handle_no_odcs_command(tmpdir, mocker):
mocker.patch('cekit.generator.docker.platform.machine',
mocker.patch('cekit.generator.base.platform.machine',
return_value="current_platform")
mock_odcs = mocker.patch(
'cekit.generator.docker.subprocess.check_output', side_effect=OSError)
'cekit.generator.base.subprocess.check_output', side_effect=OSError)

with docker_generator(tmpdir) as generator:
generator.image = {'osbs': {'configuration': {'container': {
Expand Down

0 comments on commit 94f824d

Please sign in to comment.