From f13226ec4351fff09dc5e656353568c407ee7fce Mon Sep 17 00:00:00 2001 From: Brian Bouterse Date: Mon, 15 Jun 2015 14:36:41 -0400 Subject: [PATCH] Converts puppet Module to mongoengine. The following are done: * model definition * upload importer * forge importer * directory importer * copy importer * puppet_distributor * puppet_install_distributor * puppet_file_distributor * wholerepo profiler * migrations * forge API * adds a migration to drop specific because they were redundant * adds 2.8.0 release notes * adds a release note about the index dropping https://pulp.plan.io/issues/875 closes #875 --- devel/pulp_puppet/__init__.py | 13 - devel/pulp_puppet/devel/__init__.py | 12 - devel/pulp_puppet/devel/base_cli.py | 13 - docs/user-guide/release-notes/2.8.x.rst | 21 ++ docs/user-guide/release-notes/index.rst | 1 + manage_setup_pys.sh | 12 - pulp-dev.py | 2 - pulp-puppet.spec | 4 - .../pulp_puppet/common/__init__.py | 12 - .../pulp_puppet/common/constants.py | 23 +- .../pulp_puppet/common/model.py | 264 -------------- .../pulp_puppet/common/publish_progress.py | 95 ++--- .../pulp_puppet/common/reporting.py | 13 - .../pulp_puppet/common/sync_progress.py | 97 +++--- .../extensions/admin/consumer/bind.py | 13 - .../pulp_puppet/extensions/admin/pulp_cli.py | 13 - .../extensions/admin/repo/__init__.py | 12 - .../extensions/admin/repo/copy_modules.py | 13 - .../pulp_puppet/extensions/admin/repo/cudl.py | 11 - .../extensions/admin/repo/modules.py | 13 - .../admin/repo/publish_schedules.py | 11 - .../extensions/admin/repo/remove.py | 13 - .../extensions/admin/repo/status.py | 13 - .../extensions/admin/repo/sync_schedules.py | 11 - .../extensions/admin/repo/units_display.py | 12 - .../extensions/admin/repo/upload.py | 15 - .../pulp_puppet/extensions/admin/structure.py | 11 - .../admin/repo/test_copy_modules.py | 13 - .../unit/extensions/admin/repo/test_remove.py | 12 - .../admin/repo/test_units_display.py | 12 - .../test/unit/test_binding.py | 13 - .../test/unit/test_consumer_content.py | 13 - .../test/unit/test_extension_modules.py | 13 - .../test/unit/test_extension_publish.py | 12 - .../test/unit/test_extension_status.py | 13 - .../test/unit/test_extension_structure.py | 13 - .../unit/test_extension_sync_schedules.py | 11 - .../test/unit/test_extension_upload.py | 14 - .../pulp_puppet/extensions/consumer/bind.py | 13 - .../extensions/consumer/pulp_cli.py | 13 - .../extensions/consumer/structure.py | 13 - .../test/unit/consumer_base_cli.py | 11 - .../test/unit/test_binding.py | 13 - .../test/unit/test_extension_structure.py | 13 - .../pulp_puppet/handlers/puppet.py | 27 +- .../test/unit/test_bind_handler.py | 13 - .../test/unit/test_module_handler.py | 13 - .../etc/httpd/conf.d/pulp_puppet.conf | 13 - .../pulp_puppet/forge/releases.py | 26 +- pulp_puppet_plugins/pulp_puppet/forge/unit.py | 18 +- pulp_puppet_plugins/pulp_puppet/forge/urls.py | 2 +- .../pulp_puppet/forge/views/releases.py | 3 +- .../pulp_puppet/plugins/db/__init__.py | 0 .../pulp_puppet/plugins/db/models.py | 199 +++++++++++ .../plugins/distributors/configuration.py | 19 +- .../plugins/distributors/distributor.py | 24 +- .../plugins/distributors/filedistributor.py | 66 ++-- .../distributors/installdistributor.py | 213 ++++++------ .../plugins/distributors/publish.py | 177 +++++----- .../pulp_puppet/plugins/importers/__init__.py | 1 - .../plugins/importers/configuration.py | 33 +- .../pulp_puppet/plugins/importers/copier.py | 13 +- .../plugins/importers/directory.py | 119 +++---- .../plugins/importers/downloaders/__init__.py | 12 - .../plugins/importers/downloaders/base.py | 32 +- .../importers/downloaders/exceptions.py | 15 - .../plugins/importers/downloaders/factory.py | 18 +- .../plugins/importers/downloaders/local.py | 16 +- .../plugins/importers/downloaders/web.py | 107 +++--- .../pulp_puppet/plugins/importers/forge.py | 186 +++++----- .../pulp_puppet/plugins/importers/importer.py | 26 +- .../pulp_puppet/plugins/importers/metadata.py | 24 +- .../pulp_puppet/plugins/importers/upload.py | 58 ++-- .../0001_puppet_module_unit_checksum.py | 16 +- ...0002_puppet_publishing_directory_change.py | 16 +- .../0003_puppet_drop_module_indexes.py | 30 ++ .../plugins/migrations/__init__.py | 10 - .../plugins/profilers/wholerepo.py | 43 +-- .../pulp_puppet/plugins/types/puppet.json | 9 - pulp_puppet_plugins/setup.py | 3 + .../test/integration/test_import_directory.py | 11 - .../integration/test_live_http_downloader.py | 14 +- .../test/unit/forge/test_unit.py | 13 - .../test/unit/plugins/db/__init__.py | 0 .../test/unit/plugins/db/test_models.py | 100 +----- .../distributors/test_filedistributor.py | 49 +-- .../importer/downloaders/base_downloader.py | 17 +- .../plugins/importer/downloaders/test_base.py | 13 - .../importer/downloaders/test_factory.py | 13 - .../importer/downloaders/test_local.py | 31 +- .../plugins/importer/downloaders/test_web.py | 36 +- .../plugins/importer/test_configuration.py | 12 - .../test/unit/plugins/importer/test_copier.py | 22 -- .../unit/plugins/importer/test_directory.py | 257 ++------------ .../test/unit/plugins/importer/test_forge.py | 138 +------- .../unit/plugins/importer/test_importer.py | 15 +- .../unit/plugins/importer/test_metadata.py | 117 +------ .../test/unit/plugins/importer/test_upload.py | 33 +- .../test_0001_puppet_module_unit_checksum.py | 14 - ...0002_puppet_publishing_directory_change.py | 16 +- .../test_0003_puppet_drop_module_indexes.py | 24 ++ .../test/unit/test_distributor.py | 11 - .../unit/test_distributor_configuration.py | 11 - .../test/unit/test_distributor_publish.py | 327 ------------------ .../test/unit/test_install_distributor.py | 249 +------------ .../test/unit/test_profiler_whole_repo.py | 30 -- .../tools/puppet_module_builder.py | 11 - .../test/unit/test_puppet_module_builder.py | 11 - rel-eng/lib/pulptagger.py | 12 +- 109 files changed, 1017 insertions(+), 3148 deletions(-) create mode 100644 docs/user-guide/release-notes/2.8.x.rst delete mode 100644 pulp_puppet_common/pulp_puppet/common/model.py create mode 100644 pulp_puppet_plugins/pulp_puppet/plugins/db/__init__.py create mode 100644 pulp_puppet_plugins/pulp_puppet/plugins/db/models.py create mode 100644 pulp_puppet_plugins/pulp_puppet/plugins/migrations/0003_puppet_drop_module_indexes.py delete mode 100644 pulp_puppet_plugins/pulp_puppet/plugins/types/puppet.json create mode 100644 pulp_puppet_plugins/test/unit/plugins/db/__init__.py rename pulp_puppet_common/test/unit/test_common_model.py => pulp_puppet_plugins/test/unit/plugins/db/test_models.py (53%) create mode 100644 pulp_puppet_plugins/test/unit/plugins/migrations/test_0003_puppet_drop_module_indexes.py delete mode 100644 pulp_puppet_plugins/test/unit/test_distributor_publish.py diff --git a/devel/pulp_puppet/__init__.py b/devel/pulp_puppet/__init__.py index 6a91acd5..3ad9513f 100644 --- a/devel/pulp_puppet/__init__.py +++ b/devel/pulp_puppet/__init__.py @@ -1,15 +1,2 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2014 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from pkgutil import extend_path __path__ = extend_path(__path__, __name__) diff --git a/devel/pulp_puppet/devel/__init__.py b/devel/pulp_puppet/devel/__init__.py index cc2191cf..e69de29b 100644 --- a/devel/pulp_puppet/devel/__init__.py +++ b/devel/pulp_puppet/devel/__init__.py @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2014 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/devel/pulp_puppet/devel/base_cli.py b/devel/pulp_puppet/devel/base_cli.py index 1a8a6260..d8bd414d 100644 --- a/devel/pulp_puppet/devel/base_cli.py +++ b/devel/pulp_puppet/devel/base_cli.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import copy import logging import os.path diff --git a/docs/user-guide/release-notes/2.8.x.rst b/docs/user-guide/release-notes/2.8.x.rst new file mode 100644 index 00000000..66027f0e --- /dev/null +++ b/docs/user-guide/release-notes/2.8.x.rst @@ -0,0 +1,21 @@ +============================= +Pulp Puppet 2.8 Release Notes +============================= + +Pulp Puppet 2.8.0 +================= + +New Features +------------ + +- This release drops the following three indexes using a database migration which runs as part of + the `pulp-manage-db` command: + - `tag_list` + - `name_1_version_1_author_1` + - `author` +- This release also adds an index for `author_1_name_1_version_1` + +Bugs Fixed +---------- + +You can see the :fixedbugs:`list of bugs fixed<2.8.0>`. diff --git a/docs/user-guide/release-notes/index.rst b/docs/user-guide/release-notes/index.rst index b2f9c1e4..cef07c6c 100644 --- a/docs/user-guide/release-notes/index.rst +++ b/docs/user-guide/release-notes/index.rst @@ -6,6 +6,7 @@ Contents: .. toctree:: :maxdepth: 2 + 2.8.x 2.7.x 2.6.x 2.5.x diff --git a/manage_setup_pys.sh b/manage_setup_pys.sh index 284dfbe7..e8aed662 100755 --- a/manage_setup_pys.sh +++ b/manage_setup_pys.sh @@ -1,16 +1,4 @@ #!/usr/bin/env bash -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. # Use this bash script to run argument 1 on each setup.py in this project diff --git a/pulp-dev.py b/pulp-dev.py index 7de54121..e35cf878 100755 --- a/pulp-dev.py +++ b/pulp-dev.py @@ -33,8 +33,6 @@ ('pulp_puppet_plugins/etc/httpd/conf.d/pulp_puppet.conf', '/etc/httpd/conf.d/pulp_puppet.conf'), ('pulp_puppet_plugins/etc/pulp/vhosts80/puppet.conf', '/etc/pulp/vhosts80/puppet.conf'), ('pulp_puppet_plugins/srv/pulp/puppet_forge.wsgi', '/srv/pulp/puppet_forge.wsgi'), - # Puppet Support Plugins - ('pulp_puppet_plugins/pulp_puppet/plugins/types/puppet.json', DIR_PLUGINS + '/types/puppet.json'), # Puppet Support Admin Extensions ('pulp_puppet_extensions_admin/etc/pulp/admin/conf.d/puppet.conf', '/etc/pulp/admin/conf.d/puppet.conf'), # handlers diff --git a/pulp-puppet.spec b/pulp-puppet.spec index 12bcbf0d..90db34ba 100644 --- a/pulp-puppet.spec +++ b/pulp-puppet.spec @@ -95,7 +95,6 @@ popd mkdir -p %{buildroot}/%{_sysconfdir}/pulp/vhosts80 mkdir -p %{buildroot}/srv/pulp -mkdir -p %{buildroot}/%{_usr}/lib/pulp/plugins/types mkdir -p %{buildroot}/%{_var}/lib/pulp/published/puppet/http mkdir -p %{buildroot}/%{_var}/lib/pulp/published/puppet/https @@ -103,8 +102,6 @@ cp -R pulp_puppet_plugins/etc/httpd %{buildroot}/%{_sysconfdir} cp pulp_puppet_plugins/etc/pulp/vhosts80/puppet.conf %{buildroot}/%{_sysconfdir}/pulp/vhosts80/ # WSGI app cp -R pulp_puppet_plugins/srv/pulp/puppet_forge.wsgi %{buildroot}/srv/pulp/ -# Types -cp -R pulp_puppet_plugins/pulp_puppet/plugins/types/* %{buildroot}/%{_usr}/lib/pulp/plugins/types/ %endif # End pulp_server if block pushd pulp_puppet_handlers @@ -181,7 +178,6 @@ to provide Puppet specific support. %{python_sitelib}/pulp_puppet/forge/ %{python_sitelib}/pulp_puppet/plugins/ %config(noreplace) %{_sysconfdir}/httpd/conf.d/pulp_puppet.conf -%{_usr}/lib/pulp/plugins/types/puppet.json %{python_sitelib}/pulp_puppet_plugins*.egg-info /srv/pulp/puppet_forge.wsgi diff --git a/pulp_puppet_common/pulp_puppet/common/__init__.py b/pulp_puppet_common/pulp_puppet/common/__init__.py index 0be8f97a..e69de29b 100644 --- a/pulp_puppet_common/pulp_puppet/common/__init__.py +++ b/pulp_puppet_common/pulp_puppet/common/__init__.py @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/pulp_puppet_common/pulp_puppet/common/constants.py b/pulp_puppet_common/pulp_puppet/common/constants.py index 036ea4dc..d0dffb31 100644 --- a/pulp_puppet_common/pulp_puppet/common/constants.py +++ b/pulp_puppet_common/pulp_puppet/common/constants.py @@ -1,27 +1,14 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - """ Contains constants that are global across the entire puppet plugin. Eventually, this will be pulled into a common dependency across all of the puppet support plugins (importers, distributors, extensions). """ -# -- ids ---------------------------------------------------------------------- - -# ID used to refer to the puppet importer import os + +# -- ids ---------------------------------------------------------------------- + IMPORTER_TYPE_ID = 'puppet_importer' # ID used to refer to the puppet importer instance on a repository @@ -49,7 +36,7 @@ TYPE_PUPPET_MODULE = 'puppet_module' # Used as a note on a repository to indicate it is a Puppet repository -REPO_NOTE_KEY = '_repo-type' # needs to be standard across extensions +REPO_NOTE_KEY = '_repo-type' # needs to be standard across extensions REPO_NOTE_PUPPET = 'puppet-repo' # -- storage and hosting ------------------------------------------------------ @@ -162,7 +149,7 @@ # as its value that should be used for the request SKIP_DEP_OPTION = 'skip_dep' -# Option key passed to an "install", "update" or uninstall consumer request with a modulepath +# Option key passed to an "install", "update" or uninstall consumer request with a modulepath # as its value that should be used for the request MODULEPATH_OPTION = 'module_path' diff --git a/pulp_puppet_common/pulp_puppet/common/model.py b/pulp_puppet_common/pulp_puppet/common/model.py deleted file mode 100644 index 749b6ead..00000000 --- a/pulp_puppet_common/pulp_puppet/common/model.py +++ /dev/null @@ -1,264 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -""" -Contains data structures representing puppet repository and module concepts -and methods to serialize and deserialize them. -""" - -import copy - -from pulp.common.compat import json - -from pulp_puppet.common import constants - - -class RepositoryMetadata(object): - - def __init__(self): - self.modules = [] - - def update_from_json(self, metadata_json): - """ - Updates this metadata instance with modules found in the given JSON - document. This can be called multiple times to merge multiple - repository metadata JSON documents into this instance. - - :return: object representing the repository and all of its modules - :rtype: RepositoryMetadata - """ - - parsed = json.loads(metadata_json) - - # The contents of the metadata document is a list of dictionaries, - # each represnting a single module. - for module_dict in parsed: - module = Module.from_dict(module_dict) - self.modules.append(module) - - def to_json(self): - """ - Serializes the repository metadata into its JSON representation. - """ - - # Assemble all of the modules in dictionary form - module_dicts = [m.to_dict() for m in self.modules] - - # For each module, we only want a small subset of data that goes in the - # repo metadata document - limited_module_dicts = [] - included_fields = ('name', 'author', 'version', 'tag_list') - for m in module_dicts: - clean_module = dict([(k, v) for k, v in m.items() if k in included_fields]) - limited_module_dicts.append(clean_module) - - # Serialize the dictionaries into a single JSON document - serialized = json.dumps(limited_module_dicts) - - return serialized - - -class Module(object): - - UNIT_KEY_NAMES = ('name', 'version', 'author') - - @classmethod - def from_dict(cls, module_dict): - """ - Parses the given snippet of module metadata into an object - representation. This call assumes the JSON has already been parsed - and the dict representation is provided. - - :return: object representation of the given module - :rtype: Module - """ - - # The unique identifier fields are all required and should be present - name = module_dict.get('name') - version = module_dict.get('version') - author = module_dict.get('author') - - module = Module(name, version, author) - module.update_from_dict(module_dict) - - return module - - @classmethod - def from_unit(cls, pulp_unit): - """ - Converts a Pulp unit into a Module representation. - - :param pulp_unit: unit returned from the Pulp conduit - :type pulp_unit: pulp.plugins.model.Unit - - :return: object representation of the given module - :rtype: Module - """ - unit_as_dict = copy.copy(pulp_unit.unit_key) - unit_as_dict.update(pulp_unit.metadata) - - return cls.from_dict(unit_as_dict) - - @classmethod - def from_json(cls, module_json): - """ - Converts a module's metadata.json to a Module representation. This - method does not use the 'author' field of metadata.json. The author - is retrieved from the 'name' field, which is expected to be in the - format 'authorname-modulename' or 'authorname/modulename'. - - :param module_json: dict with values from metadata.json - :type module_json: dict - - :return: object representation of the given module - :rtype: Module - """ - # The unique identifier fields are all required and should be present - try: - author, name = module_json.get('name').split("-", 1) - except ValueError: - # This is the old naming format, but Puppet still allows it - author, name = module_json.get('name').split("/", 1) - - version = module_json.get('version') - - module = cls(name, version, author) - module.update_from_dict(module_json) - - return module - - @staticmethod - def generate_unit_key(name, version, author): - """ - Formats the module unique pieces into the Pulp unit key. - :rtype: dict - """ - return { - 'name' : name, - 'version' : version, - 'author' : author, - } - - def __init__(self, name, version, author): - - # Unit Key Fields - self.name = name - self.version = version - self.author = author - - # From Repository Metadata - self.tag_list = None - - # From Module Metadata - self.source = None - self.license = None - self.summary = None - self.description = None - self.project_page = None - self.types = None # list of something I don't know yet :) - self.dependencies = None # list of dicts of name to version_requirement - self.checksums = None # dict of file name (with relative path) to checksum - self.checksum = None # checksum for the .tgz of the unit itself - self.checksum_type = constants.DEFAULT_HASHLIB - - def to_dict(self): - """ - Returns a dict view on the module in the same format as was parsed from - update_from_dict. - - :return: dict view on the module - :rtype: dict - """ - d = self.unit_key() - d.update(self.unit_metadata()) - return d - - def update_from_json(self, metadata_json): - """ - Takes the module's metadata in JSON format and merges it into this - instance. - - :param metadata_json: module metadata in JSON - :type metadata_json: str - """ - parsed = json.loads(metadata_json) - self.update_from_dict(parsed) - - def update_from_dict(self, module_dict): - """ - Updates the instance variables with the values in the given dict. - """ - - # Not all calls into this will contain the tag list - if 'tag_list' in module_dict: - self.tag_list = module_dict['tag_list'] - - # Found in the module metadata itself - self.source = module_dict.get('source', None) - self.license = module_dict.get('license', None) - self.summary = module_dict.get('summary', None) - self.description = module_dict.get('description', None) - self.project_page = module_dict.get('project_page', None) - self.types = module_dict.get('types', []) - self.dependencies = module_dict.get('dependencies', []) - self.checksums = module_dict.get('checksums', {}) - self.checksum = module_dict.get('checksum', None) - self.checksum_type = module_dict.get('checksum_type', constants.DEFAULT_HASHLIB) - - # Special handling of the DB-safe checksum to rebuild it - if isinstance(self.checksums, list): - self.checksums = dict([ (c[0], c[1]) for c in self.checksums]) - - def unit_key(self): - """ - Returns the unit key for this module that will uniquely identify - it in Pulp. This is the unique key for the inventoried module in Pulp. - """ - return self.generate_unit_key(self.name, self.version, self.author) - - def unit_metadata(self): - """ - Returns all non-unit key metadata that should be stored in Pulp - for this module. This is how the module will be inventoried in Pulp. - """ - metadata = { - 'description': self.description, - 'tag_list': self.tag_list, - 'source': self.source, - 'license': self.license, - 'summary': self.summary, - 'project_page': self.project_page, - 'types': self.types, - 'dependencies': self.dependencies, - 'checksum': self.checksum, - 'checksum_type': self.checksum_type - } - - # Checksums is expressed as a dict of file to checksum. This causes - # a problem in mongo since keys can't have periods in them, but file - # names clearly will. Translate to a list of tuples to get around this - - clean_checksums = [ (k, v) for k, v in self.checksums.items()] - metadata['checksums'] = clean_checksums - - return metadata - - def filename(self): - """ - Generates the filename for the given module. - - :return: puppet standard filename for this module - :rtype: str - """ - f = constants.MODULE_FILENAME % (self.author, self.name, self.version) - return f diff --git a/pulp_puppet_common/pulp_puppet/common/publish_progress.py b/pulp_puppet_common/pulp_puppet/common/publish_progress.py index e6b6b115..f131f20f 100644 --- a/pulp_puppet_common/pulp_puppet/common/publish_progress.py +++ b/pulp_puppet_common/pulp_puppet/common/publish_progress.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - """ Contains classes and functions related to tracking the progress of a puppet distributor. @@ -27,8 +14,36 @@ class PublishProgressReport(object): the update_progress call. Once the publish is finished, this object should be used to produce the final report to return to Pulp to describe the result of the operation. + + :ivar conduit: The repository conduit used by the publisher. + :type conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit """ + def __init__(self, conduit): + self.conduit = conduit + + # Modules symlink step + self.modules_state = STATE_NOT_STARTED + self.modules_execution_time = None + self.modules_total_count = None + self.modules_finished_count = None + self.modules_error_count = None + self.modules_individual_errors = None # mapping of module to its error + self.modules_error_message = None # overall execution error + self.modules_exception = None + self.modules_traceback = None + + # Metadata generation + self.metadata_state = STATE_NOT_STARTED + self.metadata_execution_time = None + self.metadata_error_message = None + self.metadata_exception = None + self.metadata_traceback = None + + # Publishing + self.publish_http = STATE_NOT_STARTED + self.publish_https = STATE_NOT_STARTED + @classmethod def from_progress_dict(cls, report): """ @@ -43,6 +58,7 @@ def from_progress_dict(cls, report): :param report: progress report retrieved from the server's task :type report: dict + :return: instance populated with the state in the report :rtype: PublishProgressReport """ @@ -73,31 +89,6 @@ def from_progress_dict(cls, report): return r - def __init__(self, conduit): - self.conduit = conduit - - # Modules symlink step - self.modules_state = STATE_NOT_STARTED - self.modules_execution_time = None - self.modules_total_count = None - self.modules_finished_count = None - self.modules_error_count = None - self.modules_individual_errors = None # mapping of module to its error - self.modules_error_message = None # overall execution error - self.modules_exception = None - self.modules_traceback = None - - # Metadata generation - self.metadata_state = STATE_NOT_STARTED - self.metadata_execution_time = None - self.metadata_error_message = None - self.metadata_exception = None - self.metadata_traceback = None - - # Publishing - self.publish_http = STATE_NOT_STARTED - self.publish_https = STATE_NOT_STARTED - def update_progress(self): """ Sends the current state of the progress report to Pulp. @@ -156,17 +147,25 @@ def add_failed_module(self, unit, traceback): Updates the progress report that a module failed to be built to the repository. - :param unit: Pulp representation of the module - :type unit: pulp.plugins.model.AssociatedUnit + :param unit: puppet module + :type unit: pulp_puppet.plugins.db.models.Module + :param traceback: the traceback associated with the module failure + :type traceback: traceback """ self.modules_error_count += 1 self.modules_individual_errors = self.modules_individual_errors or {} - error_key = '%s-%s-%s' % (unit.unit_key['name'], unit.unit_key['version'], unit.unit_key['author']) + error_key = '%s-%s-%s' % (unit.name, unit.version, unit.author) self.modules_individual_errors[error_key] = reporting.format_traceback(traceback) # -- report creation methods ---------------------------------------------- def _modules_section(self): + """ + Builds the "modules" section of the progress report. + + :return: The "modules" section of the progress report. + :rtype: dict + """ modules_report = { 'state' : self.modules_state, 'execution_time' : self.modules_execution_time, @@ -181,6 +180,12 @@ def _modules_section(self): return modules_report def _metadata_section(self): + """ + Builds the "metadata" section of the progress report. + + :return: The "metadata" section of the progress report. + :rtype: dict + """ metadata_report = { 'state' : self.metadata_state, 'execution_time' : self.metadata_execution_time, @@ -191,8 +196,14 @@ def _metadata_section(self): return metadata_report def _publishing_section(self): + """ + Builds the "publishing" section of the progress report. + + :return: The "publishing" section of the progress report. + :rtype: dict + """ publishing_report = { 'http' : self.publish_http, 'https' : self.publish_https, } - return publishing_report \ No newline at end of file + return publishing_report diff --git a/pulp_puppet_common/pulp_puppet/common/reporting.py b/pulp_puppet_common/pulp_puppet/common/reporting.py index 8601f03a..e19ce231 100644 --- a/pulp_puppet_common/pulp_puppet/common/reporting.py +++ b/pulp_puppet_common/pulp_puppet/common/reporting.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - """ Contains methods related to formatting the progress reports sent back to Pulp by all of the puppet plugins. diff --git a/pulp_puppet_common/pulp_puppet/common/sync_progress.py b/pulp_puppet_common/pulp_puppet/common/sync_progress.py index d7fc5d20..df7ee6bc 100644 --- a/pulp_puppet_common/pulp_puppet/common/sync_progress.py +++ b/pulp_puppet_common/pulp_puppet/common/sync_progress.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - """ Contains classes and functions related to tracking the progress of the puppet importer. @@ -22,13 +9,42 @@ class SyncProgressReport(object): """ - Used to carry the state of the sync run as it proceeds. This object is used - to update the on going progress in Pulp at appropriate intervals through - the update_progress call. Once the sync is finished, this object should - be used to produce the final report to return to Pulp to describe the - sync. + Stores the state of the sync run as it proceeds. + + This object is used to update the on going progress in Pulp at appropriate intervals through + the update_progress call. Once the sync is finished, this object should be used to produce + the final report to return to Pulp describing the sync. + + :ivar conduit: The repository conduit used by the sync. + :type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit """ + def __init__(self, conduit): + self.conduit = conduit + + # Metadata download & parsing + self.metadata_state = STATE_NOT_STARTED + self.metadata_query_finished_count = None + self.metadata_query_total_count = None + self.metadata_current_query = None + self.metadata_execution_time = None + self.metadata_error_message = None + self.metadata_exception = None + self.metadata_traceback = None + + # Module download + self.modules_state = STATE_NOT_STARTED + self.modules_execution_time = None + self.modules_total_count = None + self.modules_finished_count = None + self.modules_error_count = None + # list of dictionaries describing module failures. The keys are module, author, exception, + # and traceback. + self.modules_individual_errors = [] + self.modules_error_message = None # overall execution error + self.modules_exception = None + self.modules_traceback = None + @classmethod def from_progress_dict(cls, report): """ @@ -43,8 +59,9 @@ def from_progress_dict(cls, report): :param report: progress report retrieved from the server's task :type report: dict + :return: instance populated with the state in the report - :rtype: SyncProgressReport + :rtype: cls """ r = cls(None) @@ -72,32 +89,6 @@ def from_progress_dict(cls, report): return r - def __init__(self, conduit): - self.conduit = conduit - - # Metadata download & parsing - self.metadata_state = STATE_NOT_STARTED - self.metadata_query_finished_count = None - self.metadata_query_total_count = None - self.metadata_current_query = None - self.metadata_execution_time = None - self.metadata_error_message = None - self.metadata_exception = None - self.metadata_traceback = None - - # Module download - self.modules_state = STATE_NOT_STARTED - self.modules_execution_time = None - self.modules_total_count = None - self.modules_finished_count = None - self.modules_error_count = None - # list of dictionaries describing module failures. The keys are module, author, exception, - # and traceback. - self.modules_individual_errors = [] - self.modules_error_message = None # overall execution error - self.modules_exception = None - self.modules_traceback = None - def update_progress(self): """ Sends the current state of the progress report to Pulp. @@ -111,8 +102,10 @@ def build_final_report(self): The conduit will include information that it has tracked over the course of its usage, therefore this call should only be invoked when it is time to return the report. - """ + :return: Returns an object representing the final report at the end of a sync. + :rtype: pulp.plugins.model.SyncReport + """ # Report fields total_execution_time = -1 if self.metadata_execution_time is not None and self.modules_execution_time is not None: @@ -160,14 +153,22 @@ def build_progress_report(self): def add_failed_module(self, module, exception, traceback): """ Updates the progress report that a module failed to be imported. + + :param module: The module being processed when the failure occurred + :type module: pulp_puppet.plugins.db.models.Module + :param exception: The exception related to the module failure + :type exception: exception + :param traceback: The traceback corresponding with the exception + :type traceback: traceback """ self.modules_error_count += 1 - self.modules_individual_errors.append({ + error_dict = { 'module': '%s-%s' % (module.name, module.version), 'author': module.author, 'exception': reporting.format_exception(exception), 'traceback': reporting.format_traceback(traceback), - }) + } + self.modules_individual_errors.append(error_dict) def _metadata_section(self): metadata_report = { diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/consumer/bind.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/consumer/bind.py index 1d80d6df..748b1a74 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/consumer/bind.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/consumer/bind.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from pulp.client.commands.consumer import bind from pulp_puppet.common import constants diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/pulp_cli.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/pulp_cli.py index 814c4565..aa7fcc8d 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/pulp_cli.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/pulp_cli.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os from pulp.client.commands.repo import cudl, sync_publish, upload diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/__init__.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/__init__.py index 0be8f97a..e69de29b 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/__init__.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/__init__.py @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/copy_modules.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/copy_modules.py index cfaf4d0f..0d7a7f48 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/copy_modules.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/copy_modules.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ from pulp.client.commands.unit import UnitCopyCommand diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/cudl.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/cudl.py index 0ffd931a..96d3f15d 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/cudl.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/cudl.py @@ -1,14 +1,3 @@ -# Copyright (C) 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ from pulp.client import arg_utils, parsers diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/modules.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/modules.py index 416f68a0..12665415 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/modules.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/modules.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - """ Commands related to searching for modules in a Puppet repository. """ diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/publish_schedules.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/publish_schedules.py index 83536166..40ffc1cf 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/publish_schedules.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/publish_schedules.py @@ -1,14 +1,3 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ from pulp.client.commands.schedule import (\ diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/remove.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/remove.py index fa55a44e..2e3fbdec 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/remove.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/remove.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ from pulp.client.commands.unit import UnitRemoveCommand diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/status.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/status.py index b56ea55d..b03fb137 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/status.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/status.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ import traceback diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/sync_schedules.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/sync_schedules.py index b109abb1..04c0d501 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/sync_schedules.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/sync_schedules.py @@ -1,14 +1,3 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ from pulp.client.commands.schedule import ( diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/units_display.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/units_display.py index 5f100bd4..2db9d0a4 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/units_display.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/units_display.py @@ -1,15 +1,3 @@ -# -*- coding: utf-8 -*- -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - """ Methods to handle the rendering of a unit list returned from either the copy or remove units commands. diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/upload.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/upload.py index 9236c541..62a93858 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/upload.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/repo/upload.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import copy from gettext import gettext as _ import os @@ -19,9 +6,7 @@ from pulp.client.commands.repo import upload as upload_commands from pulp_puppet.common import constants -from pulp_puppet.common.model import Module -# -- commands ----------------------------------------------------------------- DESC_FILE = _('path to a file to upload; may be specified multiple times ' 'for multiple files. File name format must be ' diff --git a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/structure.py b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/structure.py index 015d037f..c38e636f 100644 --- a/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/structure.py +++ b/pulp_puppet_extensions_admin/pulp_puppet/extensions/admin/structure.py @@ -1,14 +1,3 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - """ Contains methods related to the creation and navigation of the structure of the Puppet branch of the CLI. This module should be used in place of the extensions diff --git a/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_copy_modules.py b/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_copy_modules.py index 87eece7c..67095a52 100644 --- a/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_copy_modules.py +++ b/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_copy_modules.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import mock from pulp.bindings.exceptions import BadRequestException diff --git a/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_remove.py b/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_remove.py index cf7e6df5..44199530 100644 --- a/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_remove.py +++ b/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_remove.py @@ -1,15 +1,3 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import mock from pulp.client.commands.unit import UnitRemoveCommand diff --git a/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_units_display.py b/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_units_display.py index d3bcdaa5..8e87989f 100644 --- a/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_units_display.py +++ b/pulp_puppet_extensions_admin/test/unit/extensions/admin/repo/test_units_display.py @@ -1,15 +1,3 @@ -# -*- coding: utf-8 -*- -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest from pulp_puppet.common.constants import TYPE_PUPPET_MODULE diff --git a/pulp_puppet_extensions_admin/test/unit/test_binding.py b/pulp_puppet_extensions_admin/test/unit/test_binding.py index fe0e94cc..9dc760cb 100644 --- a/pulp_puppet_extensions_admin/test/unit/test_binding.py +++ b/pulp_puppet_extensions_admin/test/unit/test_binding.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest import mock diff --git a/pulp_puppet_extensions_admin/test/unit/test_consumer_content.py b/pulp_puppet_extensions_admin/test/unit/test_consumer_content.py index 4e3f84f0..7162ccca 100644 --- a/pulp_puppet_extensions_admin/test/unit/test_consumer_content.py +++ b/pulp_puppet_extensions_admin/test/unit/test_consumer_content.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest import mock diff --git a/pulp_puppet_extensions_admin/test/unit/test_extension_modules.py b/pulp_puppet_extensions_admin/test/unit/test_extension_modules.py index 9966169b..0f0e8b0d 100644 --- a/pulp_puppet_extensions_admin/test/unit/test_extension_modules.py +++ b/pulp_puppet_extensions_admin/test/unit/test_extension_modules.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from pulp.client.commands import options from pulp.client.commands.criteria import DisplayUnitAssociationsCommand diff --git a/pulp_puppet_extensions_admin/test/unit/test_extension_publish.py b/pulp_puppet_extensions_admin/test/unit/test_extension_publish.py index 0f5d59da..8e95be7f 100644 --- a/pulp_puppet_extensions_admin/test/unit/test_extension_publish.py +++ b/pulp_puppet_extensions_admin/test/unit/test_extension_publish.py @@ -1,15 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. from pulp.client.commands.options import OPTION_REPO_ID from pulp.client.commands.schedule import ListScheduleCommand, CreateScheduleCommand, DeleteScheduleCommand, UpdateScheduleCommand, NextRunCommand diff --git a/pulp_puppet_extensions_admin/test/unit/test_extension_status.py b/pulp_puppet_extensions_admin/test/unit/test_extension_status.py index 8108777a..82d6754e 100644 --- a/pulp_puppet_extensions_admin/test/unit/test_extension_status.py +++ b/pulp_puppet_extensions_admin/test/unit/test_extension_status.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import sys import traceback diff --git a/pulp_puppet_extensions_admin/test/unit/test_extension_structure.py b/pulp_puppet_extensions_admin/test/unit/test_extension_structure.py index 264f2c0c..77ff5fab 100644 --- a/pulp_puppet_extensions_admin/test/unit/test_extension_structure.py +++ b/pulp_puppet_extensions_admin/test/unit/test_extension_structure.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from pulp_puppet.devel import base_cli from pulp_puppet.extensions.admin import structure diff --git a/pulp_puppet_extensions_admin/test/unit/test_extension_sync_schedules.py b/pulp_puppet_extensions_admin/test/unit/test_extension_sync_schedules.py index eddf1e87..b30bb3b2 100644 --- a/pulp_puppet_extensions_admin/test/unit/test_extension_sync_schedules.py +++ b/pulp_puppet_extensions_admin/test/unit/test_extension_sync_schedules.py @@ -1,14 +1,3 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import mock from pulp.client.commands.options import OPTION_REPO_ID diff --git a/pulp_puppet_extensions_admin/test/unit/test_extension_upload.py b/pulp_puppet_extensions_admin/test/unit/test_extension_upload.py index 34f991d0..5fc52df6 100644 --- a/pulp_puppet_extensions_admin/test/unit/test_extension_upload.py +++ b/pulp_puppet_extensions_admin/test/unit/test_extension_upload.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os import unittest @@ -18,7 +5,6 @@ from pulp.client.commands.repo.upload import UploadCommand, OPTION_FILE from pulp_puppet.common import constants -from pulp_puppet.common.model import Module from pulp_puppet.devel import base_cli from pulp_puppet.extensions.admin.repo import upload diff --git a/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/bind.py b/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/bind.py index ee9bac35..e903f351 100644 --- a/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/bind.py +++ b/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/bind.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from pulp.client.commands.consumer import bind from pulp_puppet.common import constants diff --git a/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/pulp_cli.py b/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/pulp_cli.py index 90e88d5b..d5427791 100644 --- a/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/pulp_cli.py +++ b/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/pulp_cli.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from pulp.client.commands.repo.query import RepoSearchCommand from pulp.client.extensions.decorator import priority diff --git a/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/structure.py b/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/structure.py index 36d9ece1..06b8692b 100644 --- a/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/structure.py +++ b/pulp_puppet_extensions_consumer/pulp_puppet/extensions/consumer/structure.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ SECTION_ROOT = 'puppet' diff --git a/pulp_puppet_extensions_consumer/test/unit/consumer_base_cli.py b/pulp_puppet_extensions_consumer/test/unit/consumer_base_cli.py index d80f429a..e0ff8f76 100644 --- a/pulp_puppet_extensions_consumer/test/unit/consumer_base_cli.py +++ b/pulp_puppet_extensions_consumer/test/unit/consumer_base_cli.py @@ -1,14 +1,3 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import copy import logging import os diff --git a/pulp_puppet_extensions_consumer/test/unit/test_binding.py b/pulp_puppet_extensions_consumer/test/unit/test_binding.py index c1cefcb8..0e9a39bf 100644 --- a/pulp_puppet_extensions_consumer/test/unit/test_binding.py +++ b/pulp_puppet_extensions_consumer/test/unit/test_binding.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest import mock diff --git a/pulp_puppet_extensions_consumer/test/unit/test_extension_structure.py b/pulp_puppet_extensions_consumer/test/unit/test_extension_structure.py index 3b0cd032..4ba5d6a2 100644 --- a/pulp_puppet_extensions_consumer/test/unit/test_extension_structure.py +++ b/pulp_puppet_extensions_consumer/test/unit/test_extension_structure.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import consumer_base_cli from pulp_puppet.extensions.consumer import bind, pulp_cli, structure diff --git a/pulp_puppet_handlers/pulp_puppet/handlers/puppet.py b/pulp_puppet_handlers/pulp_puppet/handlers/puppet.py index 7ad26320..0acf27c2 100644 --- a/pulp_puppet_handlers/pulp_puppet/handlers/puppet.py +++ b/pulp_puppet_handlers/pulp_puppet/handlers/puppet.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ import logging import urlparse @@ -264,7 +251,7 @@ def _perform_operation(cls, operation, units, forge_url=None, skip_dep=None, mod try: popen = subprocess.Popen(args, stdout=subprocess.PIPE) except OSError: - logger.error('"puppet module" tool not found') + logger.error(_('"puppet module" tool not found')) errors[full_name] = {'error': '"puppet module" tool not found'} break @@ -275,7 +262,9 @@ def _perform_operation(cls, operation, units, forge_url=None, skip_dep=None, mod if operation_report.get('result') == 'success': num_changes += 1 if popen.returncode == 0 and 'error' not in operation_report: - logger.info('%s of module %s' % (operation, full_name)) + msg_dict = {'operation': operation, 'module': full_name} + msg = _('%(operation)s of module %(module)s') + logger.info(msg, msg_dict) successes[full_name] = operation_report else: errors[full_name] = operation_report @@ -309,7 +298,9 @@ def _interpret_operation_report(output, operation, full_name): # if there was any error trying to parse puppet's JSON output, make # an empty report except (IndexError, ValueError): - logger.warning('failed to parse JSON output from %s of %s' % (operation, full_name)) + msg_dict = {'operation': operation, 'module': full_name} + msg = _('failed to parse JSON output from %(operation)s of %(module)s') + logger.warning(msg, msg_dict) operation_report = {} return operation_report @@ -428,7 +419,9 @@ def bind(conduit, binding, options): :rtype: BindReport """ repo_id = binding['repo_id'] - logger.info('binding to repo %s' % repo_id) + msg = _('binding to repo %(repo_id)s') + msg_dict = {'repo_id': repo_id} + logger.info(msg, msg_dict) report = BindReport(repo_id) report.set_succeeded() diff --git a/pulp_puppet_handlers/test/unit/test_bind_handler.py b/pulp_puppet_handlers/test/unit/test_bind_handler.py index c8853fcc..bd4a2094 100644 --- a/pulp_puppet_handlers/test/unit/test_bind_handler.py +++ b/pulp_puppet_handlers/test/unit/test_bind_handler.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest import mock diff --git a/pulp_puppet_handlers/test/unit/test_module_handler.py b/pulp_puppet_handlers/test/unit/test_module_handler.py index a24b2fa0..122cf9e8 100644 --- a/pulp_puppet_handlers/test/unit/test_module_handler.py +++ b/pulp_puppet_handlers/test/unit/test_module_handler.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import subprocess import unittest diff --git a/pulp_puppet_plugins/etc/httpd/conf.d/pulp_puppet.conf b/pulp_puppet_plugins/etc/httpd/conf.d/pulp_puppet.conf index c5210a61..566d94e5 100644 --- a/pulp_puppet_plugins/etc/httpd/conf.d/pulp_puppet.conf +++ b/pulp_puppet_plugins/etc/httpd/conf.d/pulp_puppet.conf @@ -1,16 +1,3 @@ -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public License, -# version 2 (GPLv2). There is NO WARRANTY for this software, express or -# implied, including the implied warranties of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 -# along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. -# -# Red Hat trademarks are not licensed under GPLv2. No permission is -# granted to use or replicate Red Hat trademarks that are incorporated -# in this software or its documentation. - # # Apache configuration file for Pulp's Puppet support # diff --git a/pulp_puppet_plugins/pulp_puppet/forge/releases.py b/pulp_puppet_plugins/pulp_puppet/forge/releases.py index 85f814cd..a0834d18 100644 --- a/pulp_puppet_plugins/pulp_puppet/forge/releases.py +++ b/pulp_puppet_plugins/pulp_puppet/forge/releases.py @@ -1,17 +1,5 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import gdbm +from gettext import gettext as _ import json import logging import os.path @@ -23,6 +11,7 @@ from pulp_puppet.common import constants from pulp_puppet.forge.unit import Unit + _LOGGER = logging.getLogger(__name__) @@ -36,6 +25,9 @@ def unit_generator(dbs, module_name, hostname): :type module_name: str :param hostname: The hostname of server serving modules :type hostname: str + + :return: A generator of pulp_puppet.forge.unit.Unit objects + :rtype: generator """ for repo_id, data in dbs.iteritems(): protocol = data['protocol'] @@ -43,7 +35,9 @@ def unit_generator(dbs, module_name, hostname): try: json_data = db[module_name] except KeyError: - _LOGGER.debug('module %s not found in repo %s' % (module_name, repo_id)) + msg_dict = {'module': module_name, 'repo_id': repo_id} + msg = _('module %(module)s not found in repo %(repo_id)s') + _LOGGER.debug(msg, msg_dict) continue units = json.loads(json_data) for unit in units: @@ -167,8 +161,8 @@ def get_repo_data(repo_ids): try: ret[repo_id] = {'db': gdbm.open(db_path, 'r'), 'protocol': publish_protocol} except gdbm.error: - _LOGGER.error('failed to find dependency database for repo %s. re-publish to fix.' % - repo_id) + _LOGGER.error(_('failed to find dependency database for repo %s. re-publish to fix.' % + repo_id)) return ret diff --git a/pulp_puppet_plugins/pulp_puppet/forge/unit.py b/pulp_puppet_plugins/pulp_puppet/forge/unit.py index 28e2a38a..b6233dcb 100644 --- a/pulp_puppet_plugins/pulp_puppet/forge/unit.py +++ b/pulp_puppet_plugins/pulp_puppet/forge/unit.py @@ -1,16 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - +from gettext import gettext as _ import json import logging @@ -186,7 +174,9 @@ def units_from_json(cls, name, db, repo_id, host, protocol): try: json_data = db[name] except KeyError: - _LOGGER.debug('module %s not found in repo %s' % (name, repo_id)) + msg = _('module %(name)s not found in repo %(repo_id)s') + msg_dict = {'name': name, 'repo_id': repo_id} + _LOGGER.debug(msg, msg_dict) return [] units = json.loads(json_data) return [ diff --git a/pulp_puppet_plugins/pulp_puppet/forge/urls.py b/pulp_puppet_plugins/pulp_puppet/forge/urls.py index caa7d727..4d66e377 100644 --- a/pulp_puppet_plugins/pulp_puppet/forge/urls.py +++ b/pulp_puppet_plugins/pulp_puppet/forge/urls.py @@ -2,7 +2,7 @@ from pulp_puppet.forge.views.releases import ReleasesView, ReleasesPost36View from pulp.server.db import connection -# This is all that is required to start using Manager classes + connection.initialize() urlpatterns = patterns('', diff --git a/pulp_puppet_plugins/pulp_puppet/forge/views/releases.py b/pulp_puppet_plugins/pulp_puppet/forge/views/releases.py index 7129905a..dd338d1f 100644 --- a/pulp_puppet_plugins/pulp_puppet/forge/views/releases.py +++ b/pulp_puppet_plugins/pulp_puppet/forge/views/releases.py @@ -4,9 +4,10 @@ from django.http import HttpResponseNotFound, HttpResponse, HttpResponseBadRequest from django.views.generic import View +from pulp.server.webservices.views.util import generate_json_response from pulp_puppet.forge import releases -from pulp.server.webservices.views.util import generate_json_response + MODULE_PATTERN = re.compile('(^[a-zA-Z0-9]+)(/|-)([a-zA-Z0-9_]+)$') diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/db/__init__.py b/pulp_puppet_plugins/pulp_puppet/plugins/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/db/models.py b/pulp_puppet_plugins/pulp_puppet/plugins/db/models.py new file mode 100644 index 00000000..4d694090 --- /dev/null +++ b/pulp_puppet_plugins/pulp_puppet/plugins/db/models.py @@ -0,0 +1,199 @@ +from mongoengine import ListField, StringField +from pulp.common.compat import json +from pulp.server.db.model import FileContentUnit + +from pulp_puppet.common import constants +from pulp_puppet.plugins.importers import metadata as metadata_parser + + +class RepositoryMetadata(object): + """ + An object that stores and produces Puppet Repository metadata + + :ivar modules: the list of modules in the repository + :type modules: list + """ + + def __init__(self): + self.modules = [] + + def update_from_json(self, metadata_json): + """ + Updates this metadata instance with modules found in the given JSON + document. This can be called multiple times to merge multiple + repository metadata JSON documents into this instance. + + :return: object representing the repository and all of its modules + :rtype: RepositoryMetadata + """ + + parsed = json.loads(metadata_json) + + # The contents of the metadata document is a list of dictionaries, + # each representing a single module. + for module_dict in parsed: + module = Module.from_metadata(module_dict) + self.modules.append(module) + + def to_json(self): + """ + Return the repository metadata as a JSON representation. + + :return: The repository metadata as json. + :rtype: str + """ + repo_metadata_dict = [] + for module in self.modules: + module_metadata = {'name': module.name, 'author': module.author, + 'version': module.version, 'tag_list': module.tag_list} + repo_metadata_dict.append(module_metadata) + + # Serialize metadata of all modules in the repo into a single JSON document + return json.dumps(repo_metadata_dict) + + +class Module(FileContentUnit): + """ + The mongoengine representation of a Puppet Module. + + The fields stored by this model correspond with attributes defined in Puppet Module metadata + documentation. Refer to the Puppet documentation for more information about these fields. + """ + + name = StringField(required=True) + version = StringField(required=True) + author = StringField(required=True) + + # From Repository Metadata + tag_list = ListField() + + # Generated at the file level + checksum = StringField(required=True) + checksum_type = StringField(default=constants.DEFAULT_HASHLIB) + + # From Module Metadata + source = StringField() + license = StringField() + summary = StringField() + description = StringField() + project_page = StringField() + types = ListField() + dependencies = ListField() + checksums = ListField() + + # For backward compatibility + _ns = StringField(default='units_puppet_module') + unit_type_id = StringField(db_field='_content_type_id', required=True, + default=constants.TYPE_PUPPET_MODULE) + + unit_key_fields = ('author', 'name', 'version') + + meta = { + 'allow_inheritance': False, + 'collection': 'units_puppet_module', + 'indexes': [ + { + 'fields': unit_key_fields, + 'unique': True + }, + ], + } + + @classmethod + def pre_save_signal(cls, sender, document, **kwargs): + """ + The signal that is triggered before a unit is saved. This is used to automatically + calculate the checksum field. + + :param sender: sender class + :type sender: object + + :param document: Document that sent the signal + :type document: FileContentUnit + """ + super(Module, cls).pre_save_signal(sender, document, **kwargs) + if document.checksum is None: + document.checksum = metadata_parser.calculate_checksum(document.storage_path) + + # Checksums is expressed as a dict of files to checksum. This causes a problem in mongo + # since keys can't have periods in them, but file names clearly will. Translate this to a + # list of tuples to get around this. + if isinstance(document.checksums, dict): + document.checksums = [(k, v) for k, v in document.checksums.items()] + + def __str__(self): + """ Backwards compatible with __str__ from pulp.plugins.model.AssociatedUnit """ + return 'Unit [key=%s] [type=%s] [id=%s]' % (self.unit_key, self.unit_type_id, self.id) + + def __repr__(self): + """ Backwards compatible with __repr__ from pulp.plugins.model.AssociatedUnit """ + return str(self) + + @staticmethod + def split_filename(filename): + """ + Splits a module's filename into two parts 'author' and 'name' and returns them as a dict. + + Split the filename of a module into into two parts and return it as a dict with the keys + 'author' and 'name'. The module filenamename is expected to be in the format 'author-name' + or 'author/name'. + + :param filename: The module's filename to be split into author and name. + :type filename: basestring + + :return: A dictionary with 'author' and 'name' containing the author and name respectively. + :rtype: A dict of strings. + """ + try: + author, name = filename.split("-", 1) + except ValueError: + # This is the forge format, but Puppet still allows it + author, name = filename.split("/", 1) + return {'author': author, 'name': name} + + @classmethod + def from_metadata(cls, metadata): + """ + Returns a cls instantiated from a dict of metadata. + + Not all metadata will be stored. Metadata is stripped out to only storable fields using + the whitelist_fields method. + + :param metadata: A dictionary of Puppet module metadata. + :type metdata: dict + + :return: Returns an instantiated cls created from whitelisted metdata + :rtype: cls + """ + whitelist_fields = cls.whitelist_fields(metadata) + return cls(**whitelist_fields) + + @classmethod + def whitelist_fields(cls, metadata): + """ + Returns a dict containing only keys that can only be stored in the db as fields. + + The Puppet metadata specification contains more keys than Pulp stores. This function takes + a dict of Puppet metadata and returns a dict with all non-storable keys removed. + + :param metadata: A dictionary of Puppet module metadata. + :type metadata: dict + + :return: A dictionary containing only keys that can be stored in the database. + :rtype: dict + """ + whitelist_metadata = {} + for k, v in metadata.iteritems(): + if k in cls._fields: + whitelist_metadata[k] = v + return whitelist_metadata + + def puppet_standard_filename(self): + """ + Returns the Puppet standard filename for this module. + + :return: Puppet standard filename for this module + :rtype: str + """ + return constants.MODULE_FILENAME % (self.author, self.name, self.version) + diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/configuration.py b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/configuration.py index 6f6a4219..797b70ef 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/configuration.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/configuration.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ from pulp_puppet.common import constants @@ -58,7 +45,8 @@ def _validate_http(config): """ parsed = config.get_boolean(constants.CONFIG_SERVE_HTTP) if parsed is None: - return False, _('The value for <%(k)s> must be either "true" or "false"') % {'k' : constants.CONFIG_SERVE_HTTP} + msg_dict = {'k' : constants.CONFIG_SERVE_HTTP} + return False, _('The value for <%(k)s> must be either "true" or "false"') % msg_dict return True, None @@ -69,7 +57,8 @@ def _validate_https(config): """ parsed = config.get_boolean(constants.CONFIG_SERVE_HTTPS) if parsed is None: - return False, _('The value for <%(k)s> must be either "true" or "false"') % {'k' : constants.CONFIG_SERVE_HTTPS} + msg_dict = {'k' : constants.CONFIG_SERVE_HTTPS} + return False, _('The value for <%(k)s> must be either "true" or "false"') % msg_dict return True, None diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/distributor.py b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/distributor.py index 6abe6176..c19c621c 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/distributor.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/distributor.py @@ -1,19 +1,7 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ from pulp.plugins.distributor import Distributor +from pulp.server.db.model import Repository from pulp_puppet.common import constants from pulp_puppet.plugins.distributors import configuration, publish @@ -49,11 +37,12 @@ def distributor_removed(self, repo, config): config.default_config = configuration.DEFAULT_CONFIG publish.unpublish_repo(repo, config) - def publish_repo(self, repo, publish_conduit, config): + def publish_repo(self, repo_transfer, publish_conduit, config): + repo = Repository.objects.get_repo_or_missing_resource(repo_transfer.id) self.publish_cancelled = False config.default_config = configuration.DEFAULT_CONFIG - publish_runner = publish.PuppetModulePublishRun(repo, publish_conduit, config, - self.is_publish_cancelled) + publish_runner = publish.PuppetModulePublishRun(repo, repo_transfer, publish_conduit, + config, self.is_publish_cancelled) report = publish_runner.perform_publish() return report @@ -65,8 +54,7 @@ def cancel_publish_repo(self): def is_publish_cancelled(self): """ - Hook back into this plugin to check if a cancel request has been issued - for a publish operation. + Hook into this plugin to check if a cancel request has been issued for a publish operation. :return: true if the sync should stop running; false otherwise :rtype: bool diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/filedistributor.py b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/filedistributor.py index eba6f965..48ee35fd 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/filedistributor.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/filedistributor.py @@ -1,20 +1,7 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -import os from gettext import gettext as _ +import os -from pulp.plugins.file.distributor import FileDistributor +from pulp.plugins.file.model_distributor import FileDistributor from pulp_puppet.common import constants from pulp_puppet.plugins.distributors import configuration @@ -34,13 +21,14 @@ class PuppetFileDistributor(FileDistributor): """ Distribute Puppet Module File """ + @classmethod def metadata(cls): """ - Advertise the capabilities of the mighty PuppetFileDistributor. + Advertise the capabilities of the PuppetFileDistributor. - :return: The description of the impressive PuppetFileDistributor's capabilities. - :rtype: dict + :return: The description of PuppetFileDistributor's capabilities. + :rtype: dict """ return { 'id': constants.DISTRIBUTOR_FILE_TYPE_ID, @@ -50,23 +38,19 @@ def metadata(cls): def validate_config(self, repo, config, config_conduit): """ - Validate the configuration information for the puppet file distributor - Ensures that the https directory where the files are going to be served - from is valid. + Validate the configuration information for the puppet file distributor. - :param repo: metadata describing the repository to which the - configuration applies - :type repo: pulp.plugins.model.Repository + Ensures that the https directory where the files are going to be served from is valid. - :param config: plugin configuration instance; the proposed repo - configuration is found within - :type config: pulp.plugins.config.PluginCallConfiguration - - :param config_conduit: Configuration Conduit; - :type config_conduit: pulp.plugins.conduits.repo_config.RepoConfigConduit + :param repo: metadata describing the repository to which the configuration applies + :type repo: pulp.plugins.model.Repository + :param config: plugin configuration instance; contains the proposed repo configuration + :type config: pulp.plugins.config.PluginCallConfiguration + :param config_conduit: Configuration Conduit + :type config_conduit: pulp.plugins.conduits.repo_config.RepoConfigConduit - :return: tuple of (bool, str) to describe the result - :rtype: tuple (bool, str) + :return: A tuple of validation results + :rtype: tuple of length two. Either (False, str) or (True, None) """ config.default_config = configuration.DEFAULT_CONFIG https_dir = config.get(constants.CONFIG_FILE_HTTPS_DIR) @@ -79,14 +63,15 @@ def validate_config(self, repo, config, config_conduit): def publish_metadata_for_unit(self, unit): """ Publish the metadata for a single unit. + This should be writing to open file handles from the initialize_metadata call - :param unit: the unit for which metadata needs to be generated - :type unit: pulp.plugins.model.AssociatedUnit + :param unit: the unit for which metadata needs to be written + :type unit: pulp_puppet.plugins.db.models.Module """ self.metadata_csv_writer.writerow([os.path.basename(unit.storage_path), - unit.metadata['checksum'], - unit.metadata['checksum_type']]) + unit.checksum, + unit.checksum_type]) def get_hosting_locations(self, repo, config): """ @@ -94,11 +79,11 @@ def get_hosting_locations(self, repo, config): :param repo: The repository that is going to be hosted :type repo: pulp.plugins.model.Repository - :param config: plugin configuration - :type config: pulp.plugins.config.PluginConfiguration + :param config: plugin configuration + :type config: pulp.plugins.config.PluginConfiguration """ config.default_config = configuration.DEFAULT_CONFIG - hosting_dir = os.path.join(config.get(constants.CONFIG_FILE_HTTPS_DIR), repo.id) + hosting_dir = os.path.join(config.get(constants.CONFIG_FILE_HTTPS_DIR), repo.repo_id) return [hosting_dir] def get_paths_for_unit(self, unit): @@ -106,7 +91,8 @@ def get_paths_for_unit(self, unit): Get the paths within a target directory where this unit should be linked to. :param unit: The unit for which we want to return target paths - :type unit: pulp.plugins.model.AssociatedUnit + :type unit: pulp_puppet.plugins.db.models.Module + :return: a list of paths the unit should be linked to :rtype: list of str """ diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/installdistributor.py b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/installdistributor.py index 0af44386..424661fb 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/installdistributor.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/installdistributor.py @@ -1,29 +1,17 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - +import errno from gettext import gettext as _ import logging import os import shutil import tarfile import tempfile -import errno from pulp.plugins.distributor import Distributor -from pulp.server.db.model.criteria import UnitAssociationCriteria +from pulp.server.controllers.repository import find_repo_content_units from pulp.plugins.util.misc import get_parent_directory, mkdir from pulp_puppet.common import constants +from pulp_puppet.plugins.db.models import Module ERROR_MESSAGE_PATH = 'one or more units contains a path outside its base extraction path' _LOGGER = logging.getLogger(__name__) @@ -41,12 +29,19 @@ def entry_point(): class PuppetModuleInstallDistributor(Distributor): + def __init__(self): super(PuppetModuleInstallDistributor, self).__init__() self.detail_report = DetailReport() @classmethod def metadata(cls): + """ + Advertise the capabilities of the PuppetModuleInstallDistributor. + + :return: The description of PuppetModuleInstallDistributor's capabilities. + :rtype: dict + """ return { 'id': constants.INSTALL_DISTRIBUTOR_TYPE_ID, 'display_name': _('Puppet Install Distributor'), @@ -55,17 +50,17 @@ def metadata(cls): def validate_config(self, repo, config, config_conduit): """ - :param repo: metadata describing the repository to which the - configuration applies - :type repo: pulp.plugins.model.Repository - - :param config: plugin configuration instance; the proposed repo - configuration is found within - :type config: pulp.plugins.config.PluginCallConfiguration - :param config_conduit: Configuration Conduit; - :type config_conduit: pulp.plugins.conduits.repo_config.RepoConfigConduit - :return: tuple of (bool, str) to describe the result - :rtype: tuple + Validate the configuration of the install distributor. + + :param repo: metadata describing the repository to which the configuration applies + :type repo: pulp.plugins.model.Repository + :param config: plugin configuration instance; contains the proposed repo configuration + :type config: pulp.plugins.config.PluginCallConfiguration + :param config_conduit: Configuration conduit + :type config_conduit: pulp.plugins.conduits.repo_config.RepoConfigConduit + + :return: A tuple of validation results + :rtype: tuple of length two. Either (False, str) or (True, None) """ path = config.get(constants.CONFIG_INSTALL_PATH) if not isinstance(path, basestring): @@ -81,30 +76,30 @@ def publish_repo(self, repo, publish_conduit, config): destination directory. This effectively means extracting each module's tarball in that directory. - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository + :param repo: plugin repository object + :type repo: pulp.plugins.model.Repository :param publish_conduit: provides access to relevant Pulp functionality - :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit - :param config: plugin configuration - :type config: pulp.plugins.config.PluginConfiguration + :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit + :param config: plugin configuration + :type config: pulp.plugins.config.PluginConfiguration :return: report describing the publish run - :rtype: pulp.plugins.model.PublishReport + :rtype: pulp.plugins.model.PublishReport """ # get dir from config destination = config.get(constants.CONFIG_INSTALL_PATH) if not destination: - return publish_conduit.build_failure_report('install path not provided', + return publish_conduit.build_failure_report(_('install path not provided'), self.detail_report.report) - - units = publish_conduit.get_units(UnitAssociationCriteria([constants.TYPE_PUPPET_MODULE])) + units = list(find_repo_content_units(repo.repo_obj, yield_content_unit=True)) duplicate_units = self._find_duplicate_names(units) if duplicate_units: for unit in duplicate_units: self.detail_report.error(unit.unit_key, 'another unit in this repo also has this name') - return publish_conduit.build_failure_report('duplicate unit names', self.detail_report.report) + return publish_conduit.build_failure_report(_('duplicate unit names'), + self.detail_report.report) # check for unsafe paths in tarballs, and fail early if problems are found self._check_for_unsafe_archive_paths(units, destination) @@ -113,12 +108,11 @@ def publish_repo(self, repo, publish_conduit, config): # ensure the destination directory exists try: - self._ensure_destination_dir(destination) + mkdir(destination) temporarydestination = self._create_temporary_destination_directory(destination) except OSError, e: return publish_conduit.build_failure_report( - 'failed to create destination directory: %s' % str(e), - self.detail_report.report) + _('failed to create destination directory: %s') % str(e), self.detail_report.report) # actually publish for unit in units: @@ -134,7 +128,7 @@ def publish_repo(self, repo, publish_conduit, config): self.detail_report.error(unit.unit_key, str(e)) if self.detail_report.has_errors: - return publish_conduit.build_failure_report('failed publishing units', + return publish_conduit.build_failure_report(_('failed publishing units'), self.detail_report.report) # remove old directory if exists @@ -142,7 +136,7 @@ def publish_repo(self, repo, publish_conduit, config): self._clear_destination_directory(destination) except (IOError, OSError), e: return publish_conduit.build_failure_report( - 'failed to clear destination directory: %s' % str(e), + _('failed to clear destination directory: %s') % str(e), self.detail_report.report) # move the subdirs of the temporary dir to the destination dir @@ -150,32 +144,35 @@ def publish_repo(self, repo, publish_conduit, config): self._move_to_destination_directory(temporarydestination, destination) except (IOError, OSError), e: return publish_conduit.build_failure_report( - 'failed to move temporary destination to destination directory: %s' % str(e), + _('failed to move temporary destination to destination directory: %s') % str(e), self.detail_report.report) # return some kind of report if self.detail_report.has_errors: - return publish_conduit.build_failure_report('failed', self.detail_report.report) + return publish_conduit.build_failure_report(_('failed'), self.detail_report.report) else: - return publish_conduit.build_success_report('success', self.detail_report.report) + return publish_conduit.build_success_report(_('success'), self.detail_report.report) def distributor_removed(self, repo, config): """ Remove the environment that is configured for use. - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration + :param repo: metadata describing the repository + :type repo: pulp.plugins.model.Repository + :param config: plugin configuration + :type config: pulp.plugins.config.PluginCallConfiguration """ destination = config.get(constants.CONFIG_INSTALL_PATH) if destination: - _LOGGER.info(_('removing installed modules from environment at %(directory)s' % - {'directory': destination})) + msg = _('removing installed modules from environment at %(directory)s') + msg_dict = {'directory': destination} + _LOGGER.info(msg, msg_dict) try: shutil.rmtree(destination) except Exception, e: - _LOGGER.error(_('error removing environment: %(e)s' % {'e': e})) + msg = _('error removing environment: %(exc)s') + msg_dict = {'exc': e} + _LOGGER.error(msg, msg_dict) raise @staticmethod @@ -187,15 +184,15 @@ def _find_duplicate_names(units): and renamed to just the "name" portion of their unit key. Multiple units with the name name will conflict on the filesystem. - :param units: iterable of all units being published - :type units: iterable + :param units: iterable of all units being published + :type units: iterable - :return: list of units that have conflicting names - :rtype: list + :return: list of units that have conflicting names + :rtype: list """ names = {} for unit in units: - name = unit.unit_key['name'] + name = unit.name if name not in names: names[name] = 1 else: @@ -205,7 +202,7 @@ def _find_duplicate_names(units): for name, count in names.iteritems(): if count > 1: duplicates.add(name) - return [unit for unit in units if unit.unit_key['name'] in duplicates] + return [unit for unit in units if unit.name in duplicates] @staticmethod def _rename_directory(unit, destination, names): @@ -214,39 +211,31 @@ def _rename_directory(unit, destination, names): out the name of the directory that was extracted, and then move it to the name that puppet expects. - :param unit: unit whose tarball was extracted at the destination - :type unit: pulp.plugins.model.AssociatedUnit - :param destination: absolute path to the destination where modules should - be installed - :type destination: str - :param names: list of paths (relative or absolute) to files that - are contained in the archive that was just extracted. - :type names: list + :param unit: unit whose tarball was extracted at the destination + :type unit: pulp.plugins.model.AssociatedUnit + :param destination: absolute path to the destination where modules should be installed + :type destination: str + :param names: list of paths (relative or absolute) to files that are contained in the + archive that was just extracted. + :type names: list - :raise: IOError, ValueError + :raise: IOError, ValueError """ if not destination.endswith('/'): destination += '/' dest_length = len(destination) - dir_names = set([os.path.join(destination, name)[dest_length:].split('/')[0] for name in names]) + dir_names = set( + [os.path.join(destination, name)[dest_length:].split('/')[0] for name in names] + ) if len(dir_names) != 1: raise ValueError('too many directories extracted') before = os.path.normpath(os.path.join(destination, dir_names.pop())) - after = os.path.normpath(os.path.join(destination, unit.unit_key['name'])) + after = os.path.normpath(os.path.join(destination, unit.name)) if before != after: shutil.move(before, after) - def _ensure_destination_dir(self, destination): - """ - Ensure that the directory specified by destination exists - - :param destination: The full path to the directory to create - :type destination: str - """ - mkdir(destination) - def _check_for_unsafe_archive_paths(self, units, destination): """ Check the paths of files in each tarball to make sure none include path @@ -254,12 +243,10 @@ def _check_for_unsafe_archive_paths(self, units, destination): the destination directory. Adds errors to the detail report for each unit that has one or more offending paths. - :param units: list of pulp.plugins.model.AssociatedUnit whose - tarballs should be checked for unsafe paths - :type units: list - :param destination: absolute path to the destination where modules should - be installed - :type destination: str + :param units: list of units whose tarballs should be checked for unsafe paths + :type units: list of pulp_puppet.plugins.db.models.Module objects + :param destination: absolute path to the destination where modules should be installed + :type destination: str """ for unit in units: try: @@ -278,14 +265,13 @@ def _archive_paths_are_safe(destination, archive): Checks a tarball archive for paths that include components such as "../" that would cause files to be placed outside of the destination_path. - :param destination: absolute path to the destination where modules should - be installed - :type destination: str - :param archive: tarball archive that should be checked - :type archive tarfile.TarFile + :param destination: absolute path to the destination where modules should be installed + :type destination: str + :param archive: tarball archive that should be checked + :type archive: tarfile.TarFile - :return: True iff all paths in the archive are safe, else False - :rtype: bool + :return: True iff all paths in the archive are safe, else False + :rtype: bool """ for name in archive.getnames(): result = os.path.normpath(os.path.join(destination, name)) @@ -298,11 +284,10 @@ def _archive_paths_are_safe(destination, archive): @staticmethod def _clear_destination_directory(destination): """ - deletes every directory found in the given destination + Deletes every directory found in the given destination. - :param destination: absolute path to the destination where modules should - be installed - :type destination: str + :param destination: absolute path to the destination where modules should be installed + :type destination: str """ for directory in os.listdir(destination): path = os.path.join(destination, directory) @@ -316,11 +301,11 @@ def _create_temporary_destination_directory(destination, mode=0755): This is so that the move is hopefully taking place on the same filesystem so it will be as fast as possible. - :param destination: absolute path to the destination where modules should - be installed - :type destination: str + :param destination: absolute path to the destination where modules should be installed + :type destination: str :param mode: the directory permissions - :type mode: int + :type mode: int + :return: absolute path to temporary created directory :rtype: str """ @@ -329,22 +314,21 @@ def _create_temporary_destination_directory(destination, mode=0755): os.makedirs(basedir, mode) except OSError, e: if e.errno == errno.EEXIST and os.path.isdir(basedir): - pass # ignored + pass else: - raise e + raise return tempfile.mkdtemp(prefix='pulp', dir=basedir) @staticmethod def _move_to_destination_directory(source, destination): """ - move the subdirectories of a source directory to - a destination directory and then delete the source directory. + Move the subdirectories of a source directory to a destination directory and then delete + the source directory. :param source: absolute path to where modules are installed - :type source: str - + :type source: str :param destination: absolute path to where the modules should be copied to - :type destination: str + :type destination: str """ for directory in os.listdir(source): path = os.path.join(source, directory) @@ -365,11 +349,10 @@ def __init__(self): def success(self, unit_key): """ - Call for each unit that is successfully published. This adds that unit - key to the report. + Call for each unit that is successfully published. This adds that unit key to the report. - :param unit_key: unit key for a successfully published unit - :type unit_key: dict + :param unit_key: unit key for a successfully published unit + :type unit_key: dict """ self.report['success_unit_keys'].append(unit_key) @@ -378,17 +361,17 @@ def error(self, unit_key, error_message): Call for each unit that has an error during publish. This adds that unit key to the report. - :param unit_key: unit key for unit that had an error during publish - :type unit_key: dict - :param error_message: error message indicating what went wrong for this - particular unit + :param unit_key: unit key for unit that had an error during publish + :type unit_key: dict + :param error_message: error message indicating what went wrong for this particular unit + :type error_message: str """ self.report['errors'].append((unit_key, error_message)) @property def has_errors(self): """ - :return: True iff this report has one or more errors, else False - :rtype: bool + :return: True iff this report has one or more errors, else False + :rtype: bool """ return bool(self.report['errors']) diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/publish.py b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/publish.py index a87dd0da..ef470f5c 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/distributors/publish.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/distributors/publish.py @@ -1,6 +1,3 @@ -from datetime import datetime -from gettext import gettext as _ -import copy import gdbm import hashlib import json @@ -9,12 +6,15 @@ import shutil import sys -from pulp.server.db.model.criteria import UnitAssociationCriteria +from datetime import datetime +from gettext import gettext as _ + +from pulp.server.controllers.repository import find_repo_content_units from pulp_puppet.common import constants -from pulp_puppet.common.constants import (STATE_FAILED, STATE_RUNNING, STATE_SUCCESS, STATE_SKIPPED) -from pulp_puppet.common.model import RepositoryMetadata, Module +from pulp_puppet.common.constants import (STATE_FAILED, STATE_RUNNING, STATE_SKIPPED, STATE_SUCCESS) from pulp_puppet.common.publish_progress import PublishProgressReport +from pulp_puppet.plugins.db.models import RepositoryMetadata _logger = logging.getLogger(__name__) @@ -26,7 +26,10 @@ class PuppetModulePublishRun(object): maintain state relevant to the run and should not be reused across runs. :ivar repo: repository being published - :type repo: pulp.plugins.model.Repository + :type repo: pulp.server.db.model.Repository + + :ivar repo_transfer: repository being published + :type repo_transfer: pulp.plugins.model.Repository :ivar publish_conduit: used to communicate with Pulp for this repo's run :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit @@ -38,12 +41,12 @@ class PuppetModulePublishRun(object): :type is_cancelled_call: callable """ - def __init__(self, repo, publish_conduit, config, is_cancelled_call): + def __init__(self, repo, repo_transfer, publish_conduit, config, is_cancelled_call): self.repo = repo + self.repo_transfer = repo_transfer self.publish_conduit = publish_conduit self.config = config self.is_cancelled_call = is_cancelled_call - self.progress_report = PublishProgressReport(self.publish_conduit) def perform_publish(self): @@ -60,29 +63,28 @@ def perform_publish(self): :return: the report object to return to Pulp from the publish call :rtype: pulp.plugins.model.PublishReport """ - _logger.info('Beginning publish for repository <%s>' % self.repo.id) + msg = _('Beginning publish for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.repo_id} + _logger.info(msg, msg_dict) try: modules = self._modules_step() - if modules is not None: self._metadata_step(modules) finally: # One final update before finishing self.progress_report.update_progress() - report = self.progress_report.build_final_report() return report def _modules_step(self): """ - Performs all of the necessary actions in the modules section of the - publish. Calls in here should *only* update the modules-related steps - in the progress report. + Performs all of the necessary actions in the modules section of the publish. + + Calls in here should *only* update the modules-related steps in the progress report. - :return: list of modules in the repository; None if the modules step - failed - :rtype: list of pulp.plugins.model.AssociatedUnit + :return: list of modules in the repository; None if the modules step failed. + :rtype: list of pulp_puppet.plugins.db.models.Module or None """ self.progress_report.modules_state = STATE_RUNNING # Do not update here; the counts need to be set first by the @@ -95,7 +97,9 @@ def _modules_step(self): modules = self._retrieve_repo_modules() self._symlink_modules(modules) except Exception, e: - _logger.exception('Exception during modules step for repository <%s>' % self.repo.id) + msg = _('Exception during modules step for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.repo_id} + _logger.exception(msg, msg_dict) self.progress_report.modules_state = STATE_FAILED self.progress_report.modules_error_message = _('Error assembling modules') @@ -126,7 +130,8 @@ def _metadata_step(self, modules): publish. Calls in here should *only* update the metadata-related steps in the progress report. - :type modules: list of pulp.plugins.model.AssociatedUnit + :param modules: list of modules in the repository; empty list if there are none + :type modules: list of pulp_puppet.plugins.db.models.Module """ self.progress_report.metadata_state = STATE_RUNNING self.progress_report.update_progress() @@ -139,7 +144,9 @@ def _metadata_step(self, modules): self._copy_to_published() self._cleanup_build_dir() except Exception, e: - _logger.exception('Exception during metadata generation step for repository <%s>' % self.repo.id) + msg = _('Exception during metadata generation step for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.repo_id} + _logger.exception(msg, msg_dict) self.progress_report.metadata_state = STATE_FAILED self.progress_report.metadata_error_message = _('Error generating repository metadata') self.progress_report.metadata_exception = e @@ -161,15 +168,15 @@ def _metadata_step(self, modules): self.progress_report.update_progress() - # -- publishing steps ----------------------------------------------------- - def _init_build_dir(self): """ Initializes the directory in which the repository will be assembled prior to making it live. If this directory already exists from a previous partial run, it will be deleted. """ - _logger.info('Initializing build directory for repository <%s>' % self.repo.id) + msg = _('Initializing build directory for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.repo_id} + _logger.info(msg, msg_dict) build_dir = self._build_dir() if os.path.exists(build_dir): @@ -181,7 +188,9 @@ def _cleanup_build_dir(self): """ Deletes the build directory after a successful publish. """ - _logger.info('Cleaning up build directory for repository <%s>' % self.repo.id) + msg = _('Cleaning up build directory for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.repo_id} + _logger.info(msg, msg_dict) build_dir = self._build_dir() shutil.rmtree(build_dir) @@ -191,11 +200,11 @@ def _retrieve_repo_modules(self): Retrieves all modules in the repository. :return: list of modules in the repository; empty list if there are none - :rtype: list of pulp.plugins.model.AssociatedUnit + :rtype: list of pulp_puppet.plugins.db.models.Module objects """ - criteria = UnitAssociationCriteria(type_ids=[constants.TYPE_PUPPET_MODULE]) - all_units = self.publish_conduit.get_units(criteria=criteria) - return all_units + modules_generator = find_repo_content_units(self.repo, yield_content_unit=True) + modules = list(modules_generator) + return modules def _symlink_modules(self, modules): """ @@ -204,9 +213,12 @@ def _symlink_modules(self, modules): this call will match the expected structure of how the repository will be served. - :type modules: list of pulp.plugins.model.AssociatedUnit + :param modules: list of modules in the repository; empty list if there are none + :type modules: list of pulp_puppet.plugins.db.models.Module """ - _logger.info('Creating symlinks for modules in repository <%s>' % self.repo.id) + msg = _('Creating symlinks for modules in repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.repo_id} + _logger.info(msg, msg_dict) build_dir = self._build_dir() @@ -218,7 +230,7 @@ def _symlink_modules(self, modules): for module in modules: served_relative_path = self._build_relative_path(module) symlink_path = os.path.join(build_dir, served_relative_path) - symlink_dir = os.path.split(symlink_path)[0] + symlink_dir = os.path.dirname(symlink_path) try: if not os.path.exists(symlink_dir): @@ -232,14 +244,15 @@ def _symlink_modules(self, modules): def _build_relative_path(self, module): """ - build a relative path from the repository root to the module + Build a relative path from the repository root to the module. - :param module: puppet module - :type module: pulp.plugins.model.AssociatedUnit - :return: relative path to module file - :rtype: str + :param module: puppet module + :type module: pulp_puppet.plugins.db.models.Module + + :return: relative path to module file + :rtype: str """ - subs = (module.unit_key['author'][0], module.unit_key['author']) + subs = (module.author[0], module.author) served_relative_path = constants.HOSTED_MODULE_FILE_RELATIVE_PATH % subs return os.path.join(served_relative_path, os.path.basename(module.storage_path)) @@ -252,24 +265,21 @@ def _repo_path(self): :rtype: str """ base_path = self.config.get(constants.CONFIG_ABSOLUTE_PATH, constants.DEFAULT_ABSOLUTE_PATH) - return os.path.join(base_path, self.repo.id) + return os.path.join(base_path, self.repo.repo_id) def _generate_metadata(self, modules): """ Generates the repository metadata document for all modules in the - :type modules: list of pulp.plugins.model.AssociatedUnit + :param modules: list of modules in the repository; empty list if there are none + :type modules: list of pulp_puppet.plugins.db.models.Module """ - _logger.info('Generating metadata for repository <%s>' % self.repo.id) + msg = _('Generating metadata for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.repo_id} + _logger.info(msg, msg_dict) - # Convert the Pulp data types into the local model metadata = RepositoryMetadata() - - for m in modules: - combined = copy.copy(m.unit_key) - combined.update(m.metadata) - module = Module.from_dict(combined) - metadata.modules.append(module) + metadata.modules = modules # Write the JSON representation of the metadata to the repository json_metadata = metadata.to_json() @@ -282,47 +292,55 @@ def _generate_metadata(self, modules): def _generate_dependency_data(self, modules): """ - generate the dependency metadata that is required to provide the API - that the "puppet module" tool uses. Store the metadata in a gdbm - database at the root of the repo. Generating and storing it at publish - time means the API requests will always return results that are in-sync - with the most recent publish and are not influenced by more recent - changes to the repo or its contents. - - :type modules: list of pulp.plugins.model.AssociatedUnit + Generate the dependency metdata file. + + Generate the dependency metadata that is required to provide the API used by the + "puppet module" tool. Store the metadata in a gdbm database at the root of the repo. + This always overwrites previously published dependency metadata by opening the gdbm + database with the 'n' option. + + Generating and storing it at publish time means the API requests will always return + results that are in-sync with the most recent publish and are not influenced by more + recent changes to the repo or its contents. + + :param modules: list of modules in the repository; empty list if there are none + :type modules: list of pulp_puppet.plugins.db.models.Module """ filename = os.path.join(self._build_dir(), constants.REPO_DEPDATA_FILENAME) - _logger.debug('generating dependency metadata in file %s' % filename) + msg = _('generating dependency metadata in file %(filename)s') + msg_dict = {'filename': filename} + _logger.debug(msg, msg_dict) # opens a new file for writing and overwrites any existing file db = gdbm.open(filename, 'n') try: for module in modules: - version = module.unit_key['version'] - deps = module.metadata.get('dependencies', []) path = os.path.join(self._repo_path, self._build_relative_path(module)) # calculate the checksum - with open(module.storage_path) as file_handle: + with open(module.storage_path, 'rb') as file_handle: file_hash = hashlib.md5() while True: + # This leverages the style of 128 chunking size of MD5 and does + # compute the checksum on the entire file. content = file_handle.read(128) if not content: break file_hash.update(content) md5_sum = file_hash.hexdigest() - value = {'file': path, 'version': version, 'dependencies': deps, - 'file_md5': md5_sum} - - name = module.unit_key['name'] - author = module.unit_key['author'] - key = '%s/%s' % (author, name) - - # db is not a dictionary as assumed by flake8 - if db.has_key(key): # noqa - module_list = json.loads(db[key]) - else: + value = { + 'file': path, + 'version': module.version, + 'dependencies': module.dependencies, + 'file_md5': md5_sum + } + + forge_key = '%s/%s' % (module.author, module.name) + + try: + module_list = json.loads(db[forge_key]) + except KeyError: module_list = [] module_list.append(value) - db[key] = json.dumps(module_list) + db[forge_key] = json.dumps(module_list) finally: db.close() @@ -332,7 +350,8 @@ def _copy_to_published(self): hosted. If a directory is found at the destination, it will be deleted first. """ - _logger.info('Making newly built repository live for repository <%s>' % self.repo.id) + msg = ('Making newly built repository live for repository <%s>') % self.repo.repo_id + _logger.info(msg) build_dir = self._build_dir() @@ -342,9 +361,9 @@ def _copy_to_published(self): # -- HTTP -------- proto_dir = self.config.get(constants.CONFIG_HTTP_DIR) - repo_dest_dir = os.path.join(proto_dir, self.repo.id) + repo_dest_dir = os.path.join(proto_dir, self.repo.repo_id) - unpublish(proto_dir, self.repo) + unpublish(proto_dir, self.repo_transfer) should_serve = self.config.get_boolean(constants.CONFIG_SERVE_HTTP) if should_serve: @@ -357,9 +376,9 @@ def _copy_to_published(self): # -- HTTPS -------- proto_dir = self.config.get(constants.CONFIG_HTTPS_DIR) - repo_dest_dir = os.path.join(proto_dir, self.repo.id) + repo_dest_dir = os.path.join(proto_dir, self.repo.repo_id) - unpublish(proto_dir, self.repo) + unpublish(proto_dir, self.repo_transfer) should_serve = self.config.get_boolean(constants.CONFIG_SERVE_HTTPS) if should_serve: @@ -370,8 +389,6 @@ def _copy_to_published(self): self.progress_report.update_progress() - # -- helpers -------------------------------------------------------------- - def _build_dir(self): """ Returns the location in which the repository should be assembled during @@ -381,7 +398,7 @@ def _build_dir(self): :return: full path to the directory in which to build the repo :rtype: str """ - build_dir = os.path.join(self.repo.working_dir, 'build', self.repo.id) + build_dir = os.path.join(self.repo_transfer.working_dir, 'build', self.repo.repo_id) return build_dir diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/__init__.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/__init__.py index 8b137891..e69de29b 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/__init__.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/__init__.py @@ -1 +0,0 @@ - diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/configuration.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/configuration.py index aee949a8..72bbf90f 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/configuration.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/configuration.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ from pulp.plugins.util import importer_config @@ -24,12 +11,11 @@ def validate(config): Validates the configuration for the puppet module importer. :param config: configuration passed in by Pulp - :type config: pulp.plugins.config.PluginCallConfiguration + :type config: pulp.plugins.config.PluginCallConfiguration :return: the expected return from the plugin's validate_config method - :rtype: tuple + :rtype: tuple """ - validations = ( _validate_feed, _validate_remove_missing, @@ -48,7 +34,7 @@ def validate(config): except importer_config.InvalidConfig, e: # Because the validate() API is silly, we must concatenate all the failure messages into # one. - msg = _(u'Configuration errors:\n') + msg = _(u'Configuration errors:') + '\n' msg += '\n'.join(e.failure_messages) # Remove the last newline msg = msg.rstrip() @@ -59,7 +45,6 @@ def _validate_feed(config): """ Validates the location of the puppet modules. """ - # The feed is optional if constants.CONFIG_FEED not in config.keys(): return True, None @@ -68,7 +53,7 @@ def _validate_feed(config): feed = config.get(constants.CONFIG_FEED) is_valid = downloader_factory.is_valid_feed(feed) if not is_valid: - return False, _('The feed <%(f)s> is invalid') % {'f': feed} + return False, _('The feed <%(feed_name)s> is invalid') % {'feed_name': feed} return True, None @@ -77,15 +62,14 @@ def _validate_queries(config): """ Validates the query parameters to apply to the source feed. """ - # The queries are optional if constants.CONFIG_QUERIES not in config.keys(): return True, None queries = config.get(constants.CONFIG_QUERIES) if not isinstance(queries, (list, tuple)): - msg = _('The value for <%(q)s> must be specified as a list') - msg = msg % {'q': constants.CONFIG_QUERIES} + error_dict = {'query': constants.CONFIG_QUERIES} + msg = _('The value for <%(query)s> must be specified as a list') % error_dict return False, msg return True, None @@ -95,7 +79,6 @@ def _validate_remove_missing(config): """ Validates the remove missing modules value if it is specified. """ - # The flag is optional if constants.CONFIG_REMOVE_MISSING not in config.keys(): return True, None @@ -103,8 +86,8 @@ def _validate_remove_missing(config): # Make sure it's a boolean parsed = config.get_boolean(constants.CONFIG_REMOVE_MISSING) if parsed is None: - msg = _('The value for <%(r)s> must be either "true" or "false"') - msg = msg % {'r': constants.CONFIG_REMOVE_MISSING} + error_dict = {'remove_missing': constants.CONFIG_REMOVE_MISSING} + msg = _('The value for <%(remove_missing)s> must be either "true" or "false"') % error_dict return False, msg return True, None diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/copier.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/copier.py index 43735d42..ea1e22f3 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/copier.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/copier.py @@ -1,6 +1,5 @@ -from pulp.server.db.model.criteria import UnitAssociationCriteria - -from pulp_puppet.common import constants +from pulp.server.controllers.repository import find_repo_content_units +from pulp.server.db.model import Repository def copy_units(import_conduit, units): @@ -12,11 +11,13 @@ def copy_units(import_conduit, units): # Determine which units are being copied if units is None: - criteria = UnitAssociationCriteria(type_ids=[constants.TYPE_PUPPET_MODULE]) - units = import_conduit.get_source_units(criteria=criteria) + repo = Repository.objects.get(repo_id=import_conduit.source_repo_id) + units = find_repo_content_units(repo, yield_content_unit=True) # Associate to the new repository + units_to_return = [] for u in units: + units_to_return.append(u) import_conduit.associate_unit(u) - return units + return units_to_return diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/directory.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/directory.py index 27331c38..a22be54d 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/directory.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/directory.py @@ -15,11 +15,11 @@ from nectar.listener import AggregatingEventListener from nectar.request import DownloadRequest from pulp.plugins.util.nectar_config import importer_config_to_nectar_config -from pulp.server.db.model.criteria import UnitAssociationCriteria +from pulp.server.controllers import repository as repo_controller from pulp_puppet.common import constants -from pulp_puppet.common.model import Module from pulp_puppet.common.sync_progress import SyncProgressReport +from pulp_puppet.plugins.db.models import Module _logger = logging.getLogger(__name__) @@ -42,6 +42,8 @@ class SynchronizeWithDirectory(object): The source of the import is a directory containing a PULP_MANIFEST and multiple puppet built puppet modules. + :ivar repo: A Pulp repository object + :type repo: pulp.plugins.model.Repository :ivar conduit: Provides access to relevant Pulp functionality. :type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit :ivar config: Plugin configuration. @@ -59,10 +61,11 @@ def _extract_metadata(module_path): """ Extract the puppet module metadata from the tarball at the specified path. Search the tarball content for a file named: */metadata.json and extract - it into temporary directory. Then read the file and return the json decoded content. + it into temporary directory. Then read the file and return the json decoded content. :param module_path: The fully qualified path to the module. :type module_path: str + :return: The puppet module metadata. :rtype: dict """ @@ -78,13 +81,16 @@ def _extract_metadata(module_path): finally: shutil.rmtree(tmp_dir) - def __init__(self, conduit, config): + def __init__(self, repo, conduit, config): """ + :param repo: A Pulp repository object + :type repo: pulp.plugins.model.Repository :param conduit: Provides access to relevant Pulp functionality. :type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit :param config: Plugin configuration. :type config: pulp.plugins.config.PluginCallConfiguration """ + self.repo = repo self.conduit = conduit self.config = config self.report = None @@ -93,8 +99,9 @@ def __init__(self, conduit, config): def feed_url(self): """ - Get the feed URL from the configuration and ensure it has a - trailing '/' so urljoin will work correctly. + Get the feed URL from the configuration and ensure it has a trailing '/' so urljoin will + work correctly. + :return: The feed URL. :rtype: str """ @@ -112,13 +119,14 @@ def cancel(self): def _download(self, urls): """ Download files by URL. - Encapsulates nectar details and provides a simplified method - of downloading files. - :param urls: A list of tuples: (url, destination). The *url* and - *destination* are both strings. The *destination* is the fully - qualified path to where the file is to be downloaded. + Encapsulates nectar details and provides a simplified method of downloading files. + + :param urls: A list of tuples: (url, destination). The *url* and *destination* are both + strings. The *destination* is the fully qualified path to where the file is + to be downloaded. :type urls: list + :return: The nectar reports. Tuple of: (succeeded_reports, failed_reports) :rtype: tuple """ @@ -135,15 +143,16 @@ def _download(self, urls): nectar_config.finalize() for report in listener.succeeded_reports: - _logger.info(FETCH_SUCCEEDED % dict(url=report.url, dst=report.destination)) + _logger.info(FETCH_SUCCEEDED, dict(url=report.url, dst=report.destination)) for report in listener.failed_reports: - _logger.error(FETCH_FAILED % dict(url=report.url, msg=report.error_msg)) + _logger.error(FETCH_FAILED, dict(url=report.url, msg=report.error_msg)) return listener.succeeded_reports, listener.failed_reports def _fetch_manifest(self): """ Fetch the PULP_MANIFEST. + After the manifest is fetched, the file is parsed into a list of tuples. :return: The manifest content. List of: (name,checksum,size). @@ -188,10 +197,10 @@ def _fetch_modules(self, manifest): Fetch all of the modules referenced in the manifest. :param manifest: A parsed PULP_MANIFEST. List of: (name,checksum,size). - :type manifest: list + :type manifest: list :return: A list of paths to the fetched module files. - :rtype: list + :rtype: list """ self.started_fetch_modules = time() @@ -232,26 +241,31 @@ def _import_modules(self, module_paths): :param module_paths: A list of paths to puppet module files. :type module_paths: list """ - criteria = UnitAssociationCriteria(type_ids=[constants.TYPE_PUPPET_MODULE], - unit_fields=Module.UNIT_KEY_NAMES) - local_units = self.conduit.get_units(criteria=criteria) - local_unit_keys = [unit.unit_key for unit in local_units] + existing_module_ids_by_key = {} + for module in Module.objects.only(*Module.unit_key_fields).all(): + existing_module_ids_by_key[module.unit_key_str] = module.id + remote_unit_keys = [] for module_path in module_paths: if self.canceled: return puppet_manifest = self._extract_metadata(module_path) - module = Module.from_json(puppet_manifest) - remote_unit_keys.append(module.unit_key()) + module = Module.from_metadata(puppet_manifest) + remote_unit_keys.append(module.unit_key_str) # Even though we've already basically processed this unit, not doing this makes the # progress reporting confusing because it shows Pulp always importing all the modules. - if module.unit_key() in local_unit_keys: + if module.unit_key_str in existing_module_ids_by_key: self.report.modules_total_count -= 1 continue - _logger.debug(IMPORT_MODULE % dict(mod=module_path)) - self._add_module(module_path, module) + _logger.debug(IMPORT_MODULE, dict(mod=module_path)) + + module.set_content(module_path) + module.save() + + repo_controller.associate_single_unit(self.repo.repo_obj, module) + self.report.modules_finished_count += 1 self.report.update_progress() @@ -265,56 +279,36 @@ def _import_modules(self, module_paths): if remove_missing is None: remove_missing = constants.DEFAULT_REMOVE_MISSING if remove_missing: - self._remove_missing(local_units, remote_unit_keys) + self._remove_missing(existing_module_ids_by_key, remote_unit_keys) + repo_controller.rebuild_content_unit_counts(self.repo.repo_obj) - def _remove_missing(self, local_units, remote_unit_keys): + def _remove_missing(self, existing_module_ids_by_key, remote_unit_keys): """ Removes units from the local repository if they are missing from the remote repository. - :param local_units: A list of units associated with the current repository - :type local_units: list of AssociatedUnit - :param remote_unit_keys: a list of all the unit keys in the remote repository - :type remote_unit_keys: list of dict + :param existing_module_ids_by_key: A dict keyed on Module unit key associated with the + current repository. The values are the mongoengine id of the corresponding Module. + :type existing_module_ids_by_key: dict of Module.id values keyed on unit_key_str + :param remote_unit_keys: A list of all the Module keys in the remote repository + :type remote_unit_keys: list of strings """ - for missing in [unit for unit in local_units if unit.unit_key not in remote_unit_keys]: - if self.canceled: - return - self.conduit.remove_unit(missing) + keys_to_remove = list(set(existing_module_ids_by_key.keys()) - set(remote_unit_keys)) + doomed_ids = [existing_module_ids_by_key[key] for key in keys_to_remove] + doomed_module_iterator = Module.objects.in_bulk(doomed_ids).itervalues() + repo_controller.disassociate_units(self.repo, doomed_module_iterator) - def _add_module(self, path, module): - """ - Add the specified module to Pulp using the conduit. This will both create the module - and associate it to a repository. The module tarball is copied to the *storage path* - only if it does not already exist at the *storage path*. - - :param path: The path to the downloaded module tarball. - :type path: str - :param module: A puppet module model object. - :type module: Module - """ - type_id = constants.TYPE_PUPPET_MODULE - unit_key = module.unit_key() - unit_metadata = module.unit_metadata() - relative_path = constants.STORAGE_MODULE_RELATIVE_PATH % module.filename() - unit = self.conduit.init_unit(type_id, unit_key, unit_metadata, relative_path) - if not os.path.exists(unit.storage_path): - shutil.copy(path, unit.storage_path) - self.conduit.save_unit(unit) - - def __call__(self, repository): + def __call__(self): """ Invoke the callable object. - All work is performed in the repository working directory and - cleaned up after the call. - :param repository: A Pulp repository object. - :type repository: pulp.server.plugins.model.Repository + All work is performed in the repository working directory and cleaned up after the call. + :return: The final synchronization report. :rtype: SyncProgressReport """ self.canceled = False self.report = SyncProgressReport(self.conduit) - self.tmp_dir = mkdtemp(dir=repository.working_dir) + self.tmp_dir = mkdtemp(dir=self.repo.working_dir) try: manifest = self._fetch_manifest() if manifest is not None: @@ -332,8 +326,8 @@ def __call__(self, repository): class DownloadListener(AggregatingEventListener): """ - An extension of the nectar AggregatingEventListener used primarily - to detect cancellation and cancel the associated nectar downloader. + An extension of the nectar AggregatingEventListener used primarily to detect cancellation and + cancel the associated nectar downloader. :ivar synchronizer: The object performing the synchronization. :type synchronizer: SynchronizeWithDirectory @@ -356,6 +350,7 @@ def __init__(self, synchronizer, downloader): def download_progress(self, report): """ A download progress event. + Cancel the download if the import_function call has been canceled. :param report: A nectar download report. diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/__init__.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/__init__.py index 0be8f97a..e69de29b 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/__init__.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/__init__.py @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/base.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/base.py index 1335acad..68c9dcbe 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/base.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/base.py @@ -1,17 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - - class BaseDownloader(object): """ Base class all downloaders should extend. The factory will pass the @@ -32,10 +18,10 @@ def retrieve_metadata(self, progress_report): downloads take place. :param progress_report: used to communicate the progress of this operation - :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport + :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport :return: list of JSON documents describing all modules to import - :rtype: list + :rtype: list """ raise NotImplementedError() @@ -49,13 +35,13 @@ def retrieve_module(self, progress_report, module): :param progress_report: used if any updates need to be made as the download runs - :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport + :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport :param module: module to download - :type module: pulp_puppet.common.model.Module + :type module: pulp_puppet.common.model.Module :return: full path to the temporary location where the module file is - :rtype: str + :rtype: str """ raise NotImplementedError() @@ -65,13 +51,13 @@ def retrieve_modules(self, progress_report, module_list): :param progress_report: used if any updates need to be made as the download runs - :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport + :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport :param module_list: list of modules to be downloaded - :type module_list: iterable + :type module_list: iterable :return: list of full paths to the temporary locations where the modules are - :rtype: list + :rtype: list """ raise NotImplementedError() @@ -88,6 +74,6 @@ def cleanup_module(self, module): deleting any temporary copies of the file). :param module: module to clean up - :type module: pulp_puppet.common.model.Module + :type module: pulp_puppet.common.model.Module """ raise NotImplementedError() diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/exceptions.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/exceptions.py index 68413c26..491167b3 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/exceptions.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/exceptions.py @@ -1,17 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - - class InvalidFeed(Exception): def __init__(self, feed, *args): Exception.__init__(self, feed, *args) @@ -23,7 +9,6 @@ def __init__(self, feed_type, *args): Exception.__init__(self, feed_type, *args) self.feed_type = feed_type -# -- file retrieval exceptions ------------------------------------------------ class FileRetrievalException(Exception): """ diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/factory.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/factory.py index 8adcead4..27c72070 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/factory.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/factory.py @@ -1,21 +1,9 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - """ Determines the correct downloader implementation to return based on the feed type. """ +from gettext import gettext as _ import logging import urlparse @@ -101,5 +89,7 @@ def _determine_feed_type(feed): proto, netloc, path, params, query, frag = urlparse.urlparse(feed) return proto except Exception: - logger.exception('Exception parsing feed type for feed <%s>' % feed) + msg = _('Exception parsing feed type for feed <%(feed)s>') + msg_dict = {'feed': feed} + logger.exception(msg, msg_dict) raise InvalidFeed(feed) diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/local.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/local.py index 7d3e86b7..78fd1bbf 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/local.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/local.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os from StringIO import StringIO @@ -21,7 +8,8 @@ from pulp.plugins.util.nectar_config import importer_config_to_nectar_config from pulp_puppet.plugins.importers.downloaders.base import BaseDownloader -from pulp_puppet.plugins.importers.downloaders.exceptions import FileNotFoundException, FileRetrievalException +from pulp_puppet.plugins.importers.downloaders.exceptions import (FileNotFoundException, + FileRetrievalException) from pulp_puppet.common import constants diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/web.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/web.py index ce41df74..4fd0c08b 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/web.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/downloaders/web.py @@ -1,18 +1,7 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import copy +import errno import os + from cStringIO import StringIO from nectar.downloaders.threaded import HTTPThreadedDownloader @@ -36,17 +25,15 @@ class HttpDownloader(BaseDownloader): def retrieve_metadata(self, progress_report): """ - Retrieves all metadata documents needed to fulfill the configuration - set for the repository. The progress report will be updated as the - downloads take place. + Retrieves all metadata documents needed to fulfill the configuration set for the + repository. The progress report will be updated as the downloads take place. :param progress_report: used to communicate the progress of this operation - :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport + :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport :return: list of JSON documents describing all modules to import - :rtype: list + :rtype: list """ - urls = self._create_metadata_download_urls() # Update the progress report to reflect the number of queries it will take @@ -62,7 +49,6 @@ def retrieve_metadata(self, progress_report): # the progress report as necessary try: self.downloader.download(request_list) - finally: self.downloader.config.finalize() self.downloader = None @@ -74,18 +60,13 @@ def retrieve_metadata(self, progress_report): def retrieve_module(self, progress_report, module): """ - Retrieves the given module and returns where on disk it can be - found. It is the caller's job to copy this file to where Pulp - wants it to live as its final resting place. This downloader will - then be allowed to clean up the downloaded file in the - cleanup_module call. - - :param progress_report: used if any updates need to be made as the - download runs - :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport + Retrieves the given module and returns where on disk it can be found. + + :param progress_report: used if any updates need to be made as the download runs + :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport :param module: module to download - :type module: pulp_puppet.common.model.Module + :type module: pulp_puppet.plugins.db.models.Module :return: full path to the temporary location where the module file is :rtype: str @@ -94,19 +75,17 @@ def retrieve_module(self, progress_report, module): def retrieve_modules(self, progress_report, module_list): """ - Batch version of the retrieve_module method + Batch version of the retrieve_module method. - :param progress_report: used if any updates need to be made as the - download runs - :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport + :param progress_report: used if any updates need to be made as the download runs + :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport :param module_list: list of modules to be downloaded - :type module_list: iterable + :type module_list: list of pulp_puppet.plugins.db.models.Module objects :return: list of full paths to the temporary locations where the modules are - :rtype: list + :rtype: list """ - listener = HTTPModuleDownloadEventListener(progress_report) self.downloader = self._create_and_configure_downloader(listener) @@ -115,13 +94,12 @@ def retrieve_modules(self, progress_report, module_list): for module in module_list: url = self._create_module_url(module) module_tmp_dir = _create_download_tmp_dir(self.repo.working_dir) - module_tmp_filename = os.path.join(module_tmp_dir, module.filename()) + module_tmp_filename = os.path.join(module_tmp_dir, module.puppet_standard_filename()) request = DownloadRequest(url, module_tmp_filename) request_list.append(request) try: self.downloader.download(request_list) - finally: self.downloader.config.finalize() self.downloader = None @@ -141,27 +119,32 @@ def cancel(self): def cleanup_module(self, module): """ - Called once the unit has been copied into Pulp's storage location to - let the downloader do any post-processing it needs (for instance, - deleting any temporary copies of the file). + Cleanup up after the module has been moved into place. + + Called once the unit has been copied into Pulp's storage location. This allows the + downloader do any post-processing it needs such as deleting any temporary copies of the + file. :param module: module to clean up - :type module: pulp_puppet.common.model.Module + :type module: pulp_puppet.plugins.db.models.Module """ - module_tmp_dir = _create_download_tmp_dir(self.repo.working_dir) - module_tmp_filename = os.path.join(module_tmp_dir, module.filename()) + module_tmp_filename = os.path.join(module_tmp_dir, module.puppet_standard_filename()) - if os.path.exists(module_tmp_filename): + try: os.remove(module_tmp_filename) + except OSError as e: + if e.errno != errno.ENOENT: + raise + def _create_metadata_download_urls(self): """ - Uses the plugin configuration to determine a list of URLs for all - metadata documents that should be used in the sync. + Uses the plugin configuration to determine a list of URLs for all metadata documents + that should be used in the sync. :return: list of URLs to be downloaded - :rtype: list + :rtype: list """ feed = self.config.get(constants.CONFIG_FEED) # Puppet forge is sensitive about a double slash, so strip the trailing here @@ -198,17 +181,17 @@ def _create_module_url(self, module): Generates the URL for a module at the configured source. :param module: module instance being downloaded - :type module: pulp_puppet.common.model.Module + :type module: pulp_puppet.plugins.db.models.Module :return: full URL to download the module - :rtype: str + :rtype: str """ url = self.config.get(constants.CONFIG_FEED) if not url.endswith('/'): url += '/' url += constants.HOSTED_MODULE_FILE_RELATIVE_PATH % (module.author[0], module.author) - url += module.filename() + url += module.puppet_standard_filename() return url def _create_and_configure_downloader(self, listener): @@ -224,8 +207,7 @@ class HTTPMetadataDownloadEventListener(AggregatingEventListener): def __init__(self, progress_report): """ - :param progress_report: used if any updates need to be made as the - download runs + :param progress_report: used if any updates need to be made as the download runs :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport """ super(HTTPMetadataDownloadEventListener, self).__init__() @@ -234,7 +216,7 @@ def __init__(self, progress_report): def download_started(self, report): """ :param report: download report for a specific download - :type report: nectar.report.DownloadReport + :type report: nectar.report.DownloadReport """ self.progress_report.metadata_current_query = report.url self.progress_report.update_progress() @@ -242,7 +224,7 @@ def download_started(self, report): def download_succeeded(self, report): """ :param report: download report for a specific download - :type report: nectar.report.DownloadReport + :type report: nectar.report.DownloadReport """ super(HTTPMetadataDownloadEventListener, self).download_succeeded(report) self.progress_report.metadata_query_finished_count += 1 @@ -251,15 +233,13 @@ def download_succeeded(self, report): class HTTPModuleDownloadEventListener(AggregatingEventListener): """ - Nectar event listener that updates the progress report when downloading - modules from the web. + Nectar event listener that updates the progress report when downloading modules from the web. """ def __init__(self, progress_report): """ - :param progress_report: used if any updates need to be made as the - download runs - :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport + :param progress_report: used if any updates need to be made as the download runs + :type progress_report: pulp_puppet.importer.sync_progress.ProgressReport """ super(HTTPModuleDownloadEventListener, self).__init__() self.progress_report = progress_report @@ -267,6 +247,9 @@ def __init__(self, progress_report): def _create_download_tmp_dir(repo_working_dir): tmp_dir = os.path.join(repo_working_dir, DOWNLOAD_TMP_DIR) - if not os.path.exists(tmp_dir): + try: os.mkdir(tmp_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise return tmp_dir diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/forge.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/forge.py index efd87e65..b7d6419a 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/forge.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/forge.py @@ -1,18 +1,15 @@ from datetime import datetime from gettext import gettext as _ import logging -import os -import shutil import sys -from pulp.common.util import encode_unicode -from pulp.server.db.model.criteria import UnitAssociationCriteria +from pulp.server.controllers import repository as repo_controller from pulp_puppet.common import constants from pulp_puppet.common.constants import (STATE_FAILED, STATE_RUNNING, STATE_SUCCESS, STATE_CANCELED) -from pulp_puppet.common.model import RepositoryMetadata, Module from pulp_puppet.common.sync_progress import SyncProgressReport +from pulp_puppet.plugins.db.models import Module, RepositoryMetadata from pulp_puppet.plugins.importers import metadata as metadata_module from pulp_puppet.plugins.importers.downloaders import factory as downloader_factory @@ -33,32 +30,32 @@ def __init__(self, repo, sync_conduit, config): self.progress_report = SyncProgressReport(sync_conduit) self.downloader = None - # Since SynchronizeWithPuppetForge creats a Nectar downloader for each unit, we cannot - # rely on telling the current downloader to cancel. Therefore, we need another state tracker - # to check in the download units loop. + # Since SynchronizeWithPuppetForge creates a Nectar downloader for each unit, we cannot + # rely on telling the current downloader to cancel. Therefore, we need another state + # tracker to check in the download units loop. self._canceled = False def __call__(self): """ - Performs the sync operation according to the configured state of the - instance. The report to be sent back to Pulp is returned from this - call. This call will make calls into the conduit's progress update - as appropriate. + Sync according to the configured state of the instance and return a report. - This call executes serially. No threads are created by this call. It - will not return until either a step fails or the entire sync is - completed. + This function will make update progress as appropriate. + + This function executes serially, and does not create any threads. It will not return until + either a step fails or the entire sync is complete. :return: the report object to return to Pulp from the sync call - :rtype: SyncProgressReport + :rtype: SyncProgressReport """ - _logger.info('Beginning sync for repository <%s>' % self.repo.id) + msg = _('Beginning sync for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.id} + _logger.info(msg, msg_dict) # quit now if there is no feed URL defined if not self.config.get(constants.CONFIG_FEED): self.progress_report.metadata_state = STATE_FAILED - self.progress_report.metadata_error_message = _( - 'Cannot perform repository sync on a repository with no feed') + msg = _('Cannot perform repository sync on a repository with no feed') + self.progress_report.metadata_error_message = msg self.progress_report.update_progress() return self.progress_report.build_final_report() @@ -91,13 +88,15 @@ def _parse_metadata(self): either the successfully parsed metadata or None if it could not be retrieved or parsed. The progress report will be updated with the appropriate description of what went wrong in the event of an error, - so the caller should interpet a None return as an error occuring and + so the caller should interpret a None return as an error occurring and not continue the sync. :return: object representation of the metadata :rtype: RepositoryMetadata """ - _logger.info('Beginning metadata retrieval for repository <%s>' % self.repo.id) + msg = _('Beginning metadata retrieval for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.id} + _logger.info(msg, msg_dict) self.progress_report.metadata_state = STATE_RUNNING self.progress_report.update_progress() @@ -110,12 +109,16 @@ def _parse_metadata(self): self.downloader = downloader metadata_json_docs = downloader.retrieve_metadata(self.progress_report) - except Exception, e: + except Exception as e: if self._canceled: - _logger.warn('Exception occurred on canceled metadata download: %s' % e) + msg = _('Exception occurred on canceled metadata download: %(exc)s') + msg_dict = {'exc': e} + _logger.warn(msg, msg_dict) self.progress_report.metadata_state = STATE_CANCELED return None - _logger.exception('Exception while retrieving metadata for repository <%s>' % self.repo.id) + msg = _('Exception while retrieving metadata for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.id} + _logger.exception(msg, msg_dict) self.progress_report.metadata_state = STATE_FAILED self.progress_report.metadata_error_message = _('Error downloading metadata') self.progress_report.metadata_exception = e @@ -137,8 +140,10 @@ def _parse_metadata(self): metadata = RepositoryMetadata() for doc in metadata_json_docs: metadata.update_from_json(doc) - except Exception, e: - _logger.exception('Exception parsing metadata for repository <%s>' % self.repo.id) + except Exception as e: + msg = _('Exception parsing metadata for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.id} + _logger.exception(msg, msg_dict) self.progress_report.metadata_state = STATE_FAILED self.progress_report.metadata_error_message = _('Error parsing repository modules metadata document') self.progress_report.metadata_exception = e @@ -175,22 +180,25 @@ def _import_modules(self, metadata): containing the modules to import :type metadata: RepositoryMetadata """ - _logger.info('Retrieving modules for repository <%s>' % self.repo.id) + msg = _('Retrieving modules for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.id} + _logger.info(msg, msg_dict) self.progress_report.modules_state = STATE_RUNNING # Do not send the update about the state yet. The counts need to be # set later once we know how many are new, so to prevent a situation - # where the report reflectes running but does not have counts, wait + # where the report reflects running but does not have counts, wait # until they are populated before sending the update to Pulp. start_time = datetime.now() - # Perform the actual logic try: self._do_import_modules(metadata) - except Exception, e: - _logger.exception('Exception importing modules for repository <%s>' % self.repo.id) + except Exception as e: + msg = _('Exception importing modules for repository <%(repo_id)s>') + msg_dict = {'repo_id': self.repo.id} + _logger.exception(msg, msg_dict) self.progress_report.modules_state = STATE_FAILED self.progress_report.modules_error_message = _('Error retrieving modules') self.progress_report.modules_exception = e @@ -220,34 +228,21 @@ def _do_import_modules(self, metadata): continue. This method will only raise an exception in an extreme case where it cannot react and continue. """ - - def unit_key_str(unit_key_dict): - """ - Converts the unit key dict form into a single string that can be - used as the key in a dict lookup. - """ - template = '%s-%s-%s' - return template % (encode_unicode(unit_key_dict['name']), - encode_unicode(unit_key_dict['version']), - encode_unicode(unit_key_dict['author'])) - downloader = self._create_downloader() self.downloader = downloader - # Ease lookup of modules - modules_by_key = dict([(unit_key_str(m.unit_key()), m) for m in metadata.modules]) + # Ease module lookup + metadata_modules_by_key = dict([(m.unit_key_str, m) for m in metadata.modules]) # Collect information about the repository's modules before changing it - module_criteria = UnitAssociationCriteria(type_ids=[constants.TYPE_PUPPET_MODULE]) - existing_units = self.sync_conduit.get_units(criteria=module_criteria) - existing_modules = [Module.from_unit(x) for x in existing_units] - existing_module_keys = [unit_key_str(m.unit_key()) for m in existing_modules] + existing_module_ids_by_key = {} + for module in Module.objects.only(*Module.unit_key_fields).all(): + existing_module_ids_by_key[module.unit_key_str] = module.id - new_unit_keys = self._resolve_new_units(existing_module_keys, modules_by_key.keys()) - remove_unit_keys = self._resolve_remove_units(existing_module_keys, modules_by_key.keys()) + new_unit_keys = self._resolve_new_units(existing_module_ids_by_key.keys(), + metadata_modules_by_key.keys()) - # Once we know how many things need to be processed, we can update the - # progress report + # Once we know how many things need to be processed, we can update the progress report self.progress_report.modules_total_count = len(new_unit_keys) self.progress_report.modules_finished_count = 0 self.progress_report.modules_error_count = 0 @@ -257,89 +252,69 @@ def unit_key_str(unit_key_dict): for key in new_unit_keys: if self._canceled: break - module = modules_by_key[key] + module = metadata_modules_by_key[key] try: self._add_new_module(downloader, module) self.progress_report.modules_finished_count += 1 - except Exception, e: + except Exception as e: self.progress_report.add_failed_module(module, e, sys.exc_info()[2]) self.progress_report.update_progress() # Remove missing units if the configuration indicates to do so if self._should_remove_missing(): - existing_units_by_key = {} - for u in existing_units: - unit_key = Module.generate_unit_key(u.unit_key['name'], u.unit_key['version'], u.unit_key['author']) - s = unit_key_str(unit_key) - existing_units_by_key[s] = u - - for key in remove_unit_keys: - doomed = existing_units_by_key[key] - self.sync_conduit.remove_unit(doomed) + remove_unit_keys = self._resolve_remove_units(existing_module_ids_by_key.keys(), + metadata_modules_by_key.keys()) + doomed_ids = [existing_module_ids_by_key[key] for key in remove_unit_keys] + doomed_module_iterator = Module.objects.in_bulk(doomed_ids).itervalues() + repo_controller.disassociate_units(self.repo, doomed_module_iterator) + repo_controller.rebuild_content_unit_counts(self.repo.repo_obj) self.downloader = None def _add_new_module(self, downloader, module): """ Performs the tasks for downloading and saving a new unit in Pulp. - :param downloader: downloader instance to use for retrieving the unit - :param module: module instance to download - :type module: Module - """ - # Initialize the unit in Pulp - type_id = constants.TYPE_PUPPET_MODULE - unit_key = module.unit_key() - unit_metadata = {} # populated later but needed for the init call - relative_path = constants.STORAGE_MODULE_RELATIVE_PATH % module.filename() + This method entirely skips modules that are already in the repository. - unit = self.sync_conduit.init_unit(type_id, unit_key, unit_metadata, - relative_path) + :param downloader: downloader instance to use for retrieving the unit + :type downloader: child of pulp_puppet.plugins.importers.downloaders.base.BaseDownloader + :param module: module to download and add + :type module: pulp_puppet.plugins.db.models.Module + """ try: - if not self._module_exists(unit.storage_path): - # Download the bits - downloaded_filename = downloader.retrieve_module(self.progress_report, module) - - # Copy them to the final location - shutil.copy(downloaded_filename, unit.storage_path) + # Download the bits + downloaded_filename = downloader.retrieve_module(self.progress_report, module) # Extract the extra metadata into the module - metadata_json = metadata_module.extract_metadata(unit.storage_path, self.repo.working_dir) - module = Module.from_json(metadata_json) + metadata = metadata_module.extract_metadata(downloaded_filename, + self.repo.working_dir) + + # Overwrite the author and name + metadata.update(Module.split_filename(metadata['name'])) - # Update the unit with the extracted metadata - unit.metadata.update(module.unit_metadata()) + # Create and save the Module + module = Module.from_metadata(metadata) + module.set_content(downloaded_filename) + module.save() - # Save the unit and associate it to the repository - self.sync_conduit.save_unit(unit) + # Associate the module with the repo + repo_controller.associate_single_unit(self.repo.repo_obj, module) finally: - # Clean up the temporary module downloader.cleanup_module(module) - def _module_exists(self, filename): - """ - Determines if the module at the given filename is already downloaded. - - :param filename: full path to the module in Pulp - :type filename: str - - :return: true if the module file already exists; false otherwise - :rtype: bool - """ - return os.path.exists(filename) - - def _resolve_new_units(self, existing_unit_keys, found_unit_keys): + def _resolve_new_units(self, existing_unit_keys, metadata_unit_keys): """ - Returns a list of unit keys that are new to the repository. + Returns a list of metadata keys that are new to the repository. :return: list of unit keys; empty list if none are new :rtype: list """ - return list(set(found_unit_keys) - set(existing_unit_keys)) + return list(set(metadata_unit_keys) - set(existing_unit_keys)) - def _resolve_remove_units(self, existing_unit_keys, found_unit_keys): + def _resolve_remove_units(self, existing_unit_keys, metadata_unit_keys): """ Returns a list of unit keys that are in the repository but not in the current repository metadata. @@ -347,7 +322,7 @@ def _resolve_remove_units(self, existing_unit_keys, found_unit_keys): :return: list of unit keys; empty list if none have been removed :rtype: list """ - return list(set(existing_unit_keys) - set(found_unit_keys)) + return list(set(existing_unit_keys) - set(metadata_unit_keys)) def _create_downloader(self): """ @@ -358,8 +333,7 @@ def _create_downloader(self): """ feed = self.config.get(constants.CONFIG_FEED) - downloader = downloader_factory.get_downloader(feed, self.repo, self.sync_conduit, self.config) - return downloader + return downloader_factory.get_downloader(feed, self.repo, self.sync_conduit, self.config) def _should_remove_missing(self): """ diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/importer.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/importer.py index d6f41927..cb001412 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/importer.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/importer.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import logging from gettext import gettext as _ @@ -23,6 +10,7 @@ from pulp_puppet.plugins.importers.directory import SynchronizeWithDirectory from pulp_puppet.plugins.importers.forge import SynchronizeWithPuppetForge + # The platform currently doesn't support automatic loading of conf files when the plugin # uses entry points. The current thinking is that the conf files will be named the same as # the plugin and put in a conf.d type of location. For now, this implementation will assume @@ -71,8 +59,8 @@ def sync_repo(self, repo, sync_conduit, config): # method is used. Otherwise, the puppet forge synchronization method is used. # synchronize with a directory - self.sync_method = SynchronizeWithDirectory(sync_conduit, config) - report = self.sync_method(repo) + self.sync_method = SynchronizeWithDirectory(repo, sync_conduit, config) + report = self.sync_method() # When fetching the PULP_MANIFEST is not successful, it's assumed that the # feed points to a puppet forge instance and the synchronization is retried @@ -90,7 +78,8 @@ def import_units(self, source_repo, dest_repo, import_conduit, config, units=Non def upload_unit(self, repo, type_id, unit_key, metadata, file_path, conduit, config): try: - report = upload.handle_uploaded_unit(repo, type_id, unit_key, metadata, file_path, conduit) + report = upload.handle_uploaded_unit(repo, type_id, unit_key, metadata, file_path, + conduit) except Exception, e: _logger.exception(e) report = {'success_flag': False, 'summary': e.message, 'details': {}} @@ -107,10 +96,9 @@ def cancel_sync_repo(self): def is_sync_cancelled(self): """ - Hook back into this plugin to check if a cancel request has been issued - for a sync. + Hook into the plugin to check if a cancel request has been issued for a sync. - :return: true if the sync should stop running; false otherwise + :return: True if the sync should stop running; False otherwise :rtype: bool """ return self.sync_cancelled diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/metadata.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/metadata.py index d5b80cc0..47cb9094 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/metadata.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/metadata.py @@ -47,11 +47,11 @@ def extract_metadata(filename, temp_dir): Pulls the module's metadata file out of the module's tarball and returns it. :param filename: full path to the module file - :type filename: str + :type filename: str - :param temp_dir: location the module's files should be extracted to; - must exist prior to this call - :type temp_dir: str + :param temp_dir: location the module's files should be extracted to; must exist prior to this + call + :type temp_dir: str :raise InvalidTarball: if the module file cannot be opened :raise MissingModuleFile: if the module's metadata file cannot be found @@ -66,6 +66,7 @@ def calculate_checksum(filename): :param filename: the filename including path of the file to calculate a checksum for :type filename: str + :return: The checksum for the file :rtype: str """ @@ -81,17 +82,16 @@ def calculate_checksum(filename): def _extract_json(filename, temp_dir): """ - The entire module will be extracted to a temporary location and an attempt - will be made to find the module file. If it still cannot be found, an - exception is raised. The temporary location is deleted at the end of this - call regardless. + The entire module will be extracted to a temporary location and an attempt will be made to + find the module file. If it still cannot be found, an exception is raised. The temporary + location is deleted at the end of this call regardless. :param filename: full path to the module file - :type filename: str + :type filename: str - :param temp_dir: location the module's files should be extracted to; - must exist prior to this call - :type temp_dir: str + :param temp_dir: location the module's files should be extracted to; must exist prior to this + call + :type temp_dir: str :raise InvalidTarball: if the module file cannot be opened :raise MissingModuleFile: if the module's metadata file cannot be found diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/importers/upload.py b/pulp_puppet_plugins/pulp_puppet/plugins/importers/upload.py index 482aada4..e953da0f 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/importers/upload.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/importers/upload.py @@ -1,21 +1,10 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -import copy +import os import shutil +from pulp.server.controllers import repository as repo_controller + from pulp_puppet.common import constants -from pulp_puppet.common.model import Module +from pulp_puppet.plugins.db.models import Module from pulp_puppet.plugins.importers import metadata as metadata_parser @@ -28,42 +17,37 @@ def handle_uploaded_unit(repo, type_id, unit_key, metadata, file_path, conduit): and its association to the repository. :param repo: repository into which the unit is being uploaded - :type repo: pulp.plugins.model.Repository + :type repo: pulp.plugins.model.Repository :param type_id: type of unit being uploaded - :type type_id: str + :type type_id: str :param unit_key: unique identifier for the unit - :type unit_key: dict + :type unit_key: dict :param metadata: extra data about the unit - :type metadata: dict + :type metadata: dict :param file_path: temporary location of the uploaded file - :type file_path: str + :type file_path: str :param conduit: for calls back into Pulp - :type conduit: pulp.plugins.conduit.upload.UploadConduit + :type conduit: pulp.plugins.conduit.upload.UploadConduit """ - if type_id != constants.TYPE_PUPPET_MODULE: raise NotImplementedError() # Extract the metadata from the module extracted_data = metadata_parser.extract_metadata(file_path, repo.working_dir) - checksum = metadata_parser.calculate_checksum(file_path) - - # Create a module from the metadata - module = Module.from_json(extracted_data) - module.checksum = checksum - # Create the Pulp unit - type_id = constants.TYPE_PUPPET_MODULE - unit_key = module.unit_key() - unit_metadata = module.unit_metadata() - relative_path = constants.STORAGE_MODULE_RELATIVE_PATH % module.filename() + # rename the file so it has the original module name + original_filename = extracted_data['name'] + '-' + extracted_data['version'] + '.tar.gz' + new_file_path = os.path.join(os.path.dirname(file_path), original_filename) + shutil.move(file_path, new_file_path) - unit = conduit.init_unit(type_id, unit_key, unit_metadata, relative_path) + # Overwrite the author and name + extracted_data.update(Module.split_filename(extracted_data['name'])) - # Copy from the upload temporary location into where Pulp wants it to live - shutil.copy(file_path, unit.storage_path) + uploaded_module = Module.from_metadata(extracted_data) + uploaded_module.set_content(new_file_path) + uploaded_module.save() - # Save the unit into the destination repository - conduit.save_unit(unit) + repo_controller.associate_single_unit(repo.repo_obj, uploaded_module) + repo_controller.rebuild_content_unit_counts(repo.repo_obj) return {'success_flag': True, 'summary': '', 'details': {}} diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0001_puppet_module_unit_checksum.py b/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0001_puppet_module_unit_checksum.py index 11471232..7975a644 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0001_puppet_module_unit_checksum.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0001_puppet_module_unit_checksum.py @@ -1,16 +1,4 @@ -# -*- coding: utf-8 -*- -# Migration script for existing rpm units to include repodata -# -# Copyright © 2010-2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +from gettext import gettext as _ import logging from pulp.server.managers.content.query import ContentQueryManager @@ -23,7 +11,7 @@ def migrate(*args, **kwargs): """ - for each puppet module, calculate a checksum for the source file on the filesystem + For each puppet module, calculate a checksum for the source file on the filesystem. """ query_manager = ContentQueryManager() collection = query_manager.get_content_unit_collection(type_id=constants.TYPE_PUPPET_MODULE) diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0002_puppet_publishing_directory_change.py b/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0002_puppet_publishing_directory_change.py index 0f253f39..6dc3e8dc 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0002_puppet_publishing_directory_change.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0002_puppet_publishing_directory_change.py @@ -1,16 +1,4 @@ -# -*- coding: utf-8 -*- -# Migration script to move published repositories to the new location. -# -# Copyright © 2014 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +from gettext import gettext as _ import logging import os import shutil @@ -32,7 +20,7 @@ def migrate(*args, **kwargs): if os.path.exists(old_puppet_publish_dir) and os.listdir(old_puppet_publish_dir): # Move contents of '/var/www/pulp_puppet' into '/var/lib/pulp/published/puppet' move_directory_contents(old_puppet_publish_dir, new_puppet_publish_dir) - _log.info("Migrated published puppet repositories to the new location") + _log.info(_("Migrated published puppet repositories to the new location")) def move_directory_contents(src_dir, dest_dir): diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0003_puppet_drop_module_indexes.py b/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0003_puppet_drop_module_indexes.py new file mode 100644 index 00000000..53634127 --- /dev/null +++ b/pulp_puppet_plugins/pulp_puppet/plugins/migrations/0003_puppet_drop_module_indexes.py @@ -0,0 +1,30 @@ +import logging + +from pulp.server.db.connection import get_collection +from pymongo.errors import OperationFailure + + +_log = logging.getLogger('pulp') + + +def migrate(*args, **kwargs): + """ + Drop old indexes for units_puppet_module so that mongoengine will re-create the new ones. + """ + units_puppet_module = get_collection('units_puppet_module') + + try: + units_puppet_module.drop_index('name_1_version_1_author_1') + except OperationFailure: + # The index is already dropped + pass + try: + units_puppet_module.drop_index('author_1') + except OperationFailure: + # The index is already dropped + pass + try: + units_puppet_module.drop_index('tag_list_1') + except OperationFailure: + # The index is already dropped + pass diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/migrations/__init__.py b/pulp_puppet_plugins/pulp_puppet/plugins/migrations/__init__.py index daf79d1b..e69de29b 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/migrations/__init__.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/migrations/__init__.py @@ -1,10 +0,0 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/profilers/wholerepo.py b/pulp_puppet_plugins/pulp_puppet/plugins/profilers/wholerepo.py index 299d2b4a..d1673dee 100644 --- a/pulp_puppet_plugins/pulp_puppet/plugins/profilers/wholerepo.py +++ b/pulp_puppet_plugins/pulp_puppet/plugins/profilers/wholerepo.py @@ -1,21 +1,10 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - from gettext import gettext as _ import logging from pulp.plugins.profiler import Profiler from pulp.server.config import config as pulp_conf +from pulp.server.controllers.repository import find_repo_content_units +from pulp.server.db.model import Repository from pulp_puppet.common import constants @@ -28,6 +17,7 @@ def entry_point(): class WholeRepoProfiler(Profiler): + @classmethod def metadata(cls): """ @@ -76,25 +66,25 @@ def install_units(self, consumer, units, options, config, conduit): :type conduit: pulp.plugins.conduits.profiler.ProfilerConduit :return: The translated units - :rtype: list of: { type_id:, unit_key: } + :rtype: list of: {'type_id': , unit_key: {'author': , 'name': } """ repo_id = options.get(constants.REPO_ID_OPTION) self._inject_forge_settings(options) if options.get(constants.WHOLE_REPO_OPTION) and repo_id: - _LOGGER.debug('installing whole repo %s on %s' % (repo_id, consumer.id)) - unit_keys = [unit.unit_key for unit in conduit.get_units(repo_id)] + msg = _('installing whole repo %(repo_id)s on %(consumer_id)s') + msg_dict = {'repo_id': repo_id, 'consumer_id': consumer.id} + _LOGGER.debug(msg, msg_dict) - for unit_key in unit_keys: - # lets the install tool automatically choose the newest version - # available in the repo - unit_key.pop('version', None) + repo = Repository.objects.get(repo_id=repo_id) + units = find_repo_content_units(repo, yield_content_unit=True) - # this just makes sure we don't have duplicate copies of the same - # unit leftover from having multiple versions unit_key_dict = {} - for unit_key in unit_keys: - fullname = '%s/%s' % (unit_key['author'], unit_key['name']) - unit_key_dict[fullname] = {'unit_key': unit_key, 'type_id': constants.TYPE_PUPPET_MODULE} + for unit in units: + fullname = '%s/%s' % (unit.author, unit.name) + unit_key_dict[fullname] = { + 'unit_key': {'author': unit.author, 'name': unit.name}, + 'type_id': constants.TYPE_PUPPET_MODULE + } return unit_key_dict.values() @@ -131,7 +121,8 @@ def _inject_forge_settings(self, options): Inject the puppet forge settings into the options. Add the pulp server host and port information to the options. Used by the agent handler. + :param options: An options dictionary. :type options: dict """ - options[constants.FORGE_HOST] = pulp_conf.get('server', 'server_name') \ No newline at end of file + options[constants.FORGE_HOST] = pulp_conf.get('server', 'server_name') diff --git a/pulp_puppet_plugins/pulp_puppet/plugins/types/puppet.json b/pulp_puppet_plugins/pulp_puppet/plugins/types/puppet.json deleted file mode 100644 index d91a88a5..00000000 --- a/pulp_puppet_plugins/pulp_puppet/plugins/types/puppet.json +++ /dev/null @@ -1,9 +0,0 @@ -{"types": [ - { - "id" : "puppet_module", - "display_name" : "Puppet Module", - "description" : "Puppet Module", - "unit_key" : ["name", "version", "author"], - "search_indexes" : ["author", "tag_list"] - } -]} diff --git a/pulp_puppet_plugins/setup.py b/pulp_puppet_plugins/setup.py index d796a6c2..c5185fd9 100644 --- a/pulp_puppet_plugins/setup.py +++ b/pulp_puppet_plugins/setup.py @@ -22,5 +22,8 @@ 'pulp.server.db.migrations': [ 'pulp_puppet = pulp_puppet.plugins.migrations', ], + 'pulp.unit_models': [ + 'puppet_module=pulp_puppet.plugins.db.models:Module' + ], } ) diff --git a/pulp_puppet_plugins/test/integration/test_import_directory.py b/pulp_puppet_plugins/test/integration/test_import_directory.py index fe642b25..51c32f27 100644 --- a/pulp_puppet_plugins/test/integration/test_import_directory.py +++ b/pulp_puppet_plugins/test/integration/test_import_directory.py @@ -1,14 +1,3 @@ -# Copyright (c) 2014 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os import shutil diff --git a/pulp_puppet_plugins/test/integration/test_live_http_downloader.py b/pulp_puppet_plugins/test/integration/test_live_http_downloader.py index 9bf141e9..2d9afa43 100644 --- a/pulp_puppet_plugins/test/integration/test_live_http_downloader.py +++ b/pulp_puppet_plugins/test/integration/test_live_http_downloader.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os import shutil import tempfile @@ -24,6 +11,7 @@ from pulp_puppet.plugins.importers import metadata from pulp_puppet.plugins.importers.downloaders.web import HttpDownloader, DOWNLOAD_TMP_DIR + TEST_SOURCE = 'http://forge.puppetlabs.com/' class LiveHttpDownloaderTests(unittest.TestCase): diff --git a/pulp_puppet_plugins/test/unit/forge/test_unit.py b/pulp_puppet_plugins/test/unit/forge/test_unit.py index 11fc95a3..ba3969d2 100644 --- a/pulp_puppet_plugins/test/unit/forge/test_unit.py +++ b/pulp_puppet_plugins/test/unit/forge/test_unit.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import functools import json import unittest diff --git a/pulp_puppet_plugins/test/unit/plugins/db/__init__.py b/pulp_puppet_plugins/test/unit/plugins/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pulp_puppet_common/test/unit/test_common_model.py b/pulp_puppet_plugins/test/unit/plugins/db/test_models.py similarity index 53% rename from pulp_puppet_common/test/unit/test_common_model.py rename to pulp_puppet_plugins/test/unit/plugins/db/test_models.py index 55929939..49179ffa 100644 --- a/pulp_puppet_common/test/unit/test_common_model.py +++ b/pulp_puppet_plugins/test/unit/plugins/db/test_models.py @@ -1,21 +1,8 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest from pulp.common.compat import json -from pulp_puppet.common.model import RepositoryMetadata, Module +from pulp_puppet.plugins.db.models import RepositoryMetadata, Module # -- constants ---------------------------------------------------------------- @@ -135,88 +122,3 @@ def test_to_json(self): self.assertEqual(sorted_modules[1]['author'], 'lab42') self.assertEqual(sorted_modules[1]['version'], '0.0.2') self.assertEqual(sorted_modules[1]['tag_list'], ['postfix', 'applications']) - - -class ModuleTests(unittest.TestCase): - - def test_update_from_json(self): - # Setup - module = Module('jdob-valid', '1.0.0', 'jdob') - - # Test - module.update_from_json(VALID_MODULE_METADATA_JSON) - - # Verify - self.assert_valid_module(module) - - def test_from_dict(self): - # Setup - data = json.loads(VALID_MODULE_METADATA_JSON) - - # Test - module = Module.from_dict(data) - - # Verify - self.assert_valid_module(module) - - def test_from_json(self): - # Setup - data = json.loads(VALID_MODULE_METADATA_JSON) - - # Test - module = Module.from_json(data) - - # Verify - self.assertEqual(module.name, "valid") - self.assertEqual(module.author, "jdob") - - module.name = "jdob-valid" # rename the module to use the assert - self.assert_valid_module(module) - - def test_from_json_old_name(self): - """ - Test that the Module.from_json method handles the old naming style - """ - # Setup - metadata = { - 'name': 'oldauthor/oldmodule', - 'version': '0.1.0', - } - - # Test - module = Module.from_json(metadata) - - # Verify - self.assertEqual(module.author, 'oldauthor') - self.assertEqual(module.name, 'oldmodule') - self.assertEqual(module.version, '0.1.0') - - def assert_valid_module(self, module): - self.assertEqual(module.name, 'jdob-valid') - self.assertEqual(module.version, '1.0.0') - self.assertEqual(module.author, 'jdob') - self.assertEqual(module.source, 'http://example.org/jdob-valid/source') - self.assertEqual(module.license, 'Apache License, Version 2.0') - self.assertEqual(module.summary, 'Valid Module Summary') - self.assertEqual(module.description, 'Valid Module Description') - self.assertEqual(module.project_page, 'http://example.org/jdob-valid') - self.assertEqual(module.checksum, 'anvil') - self.assertEqual(module.checksum_type, 'acme_checksum') - - self.assertEqual(2, len(module.dependencies)) - sorted_deps = sorted(module.dependencies, key=lambda x : x['name']) - self.assertEqual(sorted_deps[0]['name'], 'jdob/dep-alpha') - self.assertEqual(sorted_deps[0]['version_requirement'], '>= 1.0.0') - self.assertEqual(sorted_deps[1]['name'], 'ldob/dep-beta') - self.assertEqual(sorted_deps[1]['version_requirement'], '>= 2.0.0') - - self.assertEqual(module.types, []) - - expected_checksums = { - 'Modulefile': '704cecf2957448dcf7fa20cffa2cf7c1', - 'README': '11edd8578497566d8054684a8c89c6cb', - 'manifests/init.pp': '1d1fb26825825b4d64d37d377016869e', - 'spec/spec_helper.rb': 'a55d1e6483344f8ec6963fcb2c220372', - 'tests/init.pp': '7043c7ef0c4b0ac52b4ec6bb76008ebd' - } - self.assertEqual(module.checksums, expected_checksums) diff --git a/pulp_puppet_plugins/test/unit/plugins/distributors/test_filedistributor.py b/pulp_puppet_plugins/test/unit/plugins/distributors/test_filedistributor.py index 15827cc6..311af04c 100644 --- a/pulp_puppet_plugins/test/unit/plugins/distributors/test_filedistributor.py +++ b/pulp_puppet_plugins/test/unit/plugins/distributors/test_filedistributor.py @@ -1,33 +1,21 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. """ -Tests for pulp_puppet.plugins.distributors.filesdistributor +Tests for pulp_puppet.plugins.distributors.filedistributor """ import os import shutil import tempfile import unittest -from mock import MagicMock, patch -from pulp.devel.mock_distributor import get_basic_config -from pulp.plugins.model import Unit +import mock from pulp.plugins.config import PluginCallConfiguration from pulp_puppet.common import constants -from pulp_puppet.plugins.distributors import configuration from pulp_puppet.plugins.distributors import filedistributor +MODULE_PATH = 'pulp_puppet.plugins.distributors.filedistributor' + + class TestEntryPoint(unittest.TestCase): """ Test the entry_point method. This is really just to get good coverage numbers, but hey. @@ -47,7 +35,7 @@ def setUp(self): self.temp_dir = tempfile.mkdtemp() self.files_path = os.path.join(self.temp_dir, 'files') os.makedirs(self.files_path) - self.unit = MagicMock() + self.unit = mock.MagicMock() self.unit.storage_path = os.path.join(self.temp_dir, 'source', "foo.tgz") self.config = PluginCallConfiguration({constants.CONFIG_FILE_HTTPS_DIR: self.files_path}, {}) @@ -57,7 +45,7 @@ def tearDown(self): shutil.rmtree(self.temp_dir) def _get_default_repo(self): - repo = MagicMock() + repo = mock.MagicMock() repo.id = 'awesome_repo' return repo @@ -96,25 +84,24 @@ def test_validate_config_files_dir_override_from_config(self): finally: shutil.rmtree(tempdir) - def test_get_hosting_locations(self): + @mock.patch(MODULE_PATH + '.os.path.join') + def test_get_hosting_locations(self, mock_join): locations = self.distributor.get_hosting_locations(self.repo, self.config) self.assertEquals(1, len(locations)) - self.assertEquals(os.path.join(self.files_path, self.repo.id), locations[0]) + self.assertEquals(locations, [mock_join.return_value]) def test_get_paths_for_unit(self): paths = self.distributor.get_paths_for_unit(self.unit) self.assertEquals(1, len(paths)) self.assertEquals('foo.tgz', paths[0]) - def test_publish_metadata_for_unit(self): - unit = Unit(constants.TYPE_PUPPET_MODULE, - {'name': 'foo'}, - {'checksum': 'alpha', 'checksum_type': 'beta'}, - os.path.join(self.temp_dir, 'foo.tgz')) + @mock.patch(MODULE_PATH + '.os.path.basename') + def test_publish_metadata_for_unit(self, mock_path): + mock_unit = mock.MagicMock() metadata_distributor = filedistributor.PuppetFileDistributor() - metadata_distributor.metadata_csv_writer = MagicMock() - metadata_distributor.publish_metadata_for_unit(unit) - metadata_distributor.metadata_csv_writer.writerow.assert_called_with(['foo.tgz', - 'alpha', - 'beta']) + metadata_distributor.metadata_csv_writer = mock.MagicMock() + metadata_distributor.publish_metadata_for_unit(mock_unit) + expected_row = [mock_path.return_value, mock_unit.checksum, mock_unit.checksum_type] + metadata_distributor.metadata_csv_writer.writerow.assert_called_with(expected_row) + mock_path.assert_called_once_with(mock_unit.storage_path) diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/base_downloader.py b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/base_downloader.py index c0628594..3bbb455e 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/base_downloader.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/base_downloader.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - """ Utilities for testing downloader implementations. """ @@ -24,8 +11,6 @@ from pulp.plugins.config import PluginCallConfiguration from pulp.plugins.model import Repository -from pulp_puppet.common import model - class BaseDownloaderTests(unittest.TestCase): @@ -40,7 +25,7 @@ def setUp(self): self.author = 'jdob' self.name = 'valid' self.version = '1.1.0' - self.module = model.Module(self.name, self.version, self.author) + self.module = mock.Mock() def tearDown(self): if os.path.exists(self.working_dir): diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_base.py b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_base.py index 2fb699b7..0fa001cc 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_base.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_base.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest from pulp_puppet.plugins.importers.downloaders import base diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_factory.py b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_factory.py index fe59d7ad..a8bdd58b 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_factory.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_factory.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest from pulp_puppet.plugins.importers.downloaders import factory diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_local.py b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_local.py index 7960a441..ab5ee1c0 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_local.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_local.py @@ -1,29 +1,18 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - - import os import mock import base_downloader -from pulp_puppet.common import constants, model +from pulp_puppet.common import constants +from pulp_puppet.plugins.db.models import RepositoryMetadata from pulp_puppet.plugins.importers.downloaders.exceptions import FileRetrievalException from pulp_puppet.plugins.importers.downloaders.local import LocalDownloader + DATA_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../../../../data' VALID_REPO_DIR = os.path.join(DATA_DIR, 'repos', 'valid') INVALID_REPO_DIR = os.path.join(DATA_DIR, 'repos', 'invalid') +MODULE_PATH = 'pulp_puppet.plugins.importers.downloaders.local' class LocalDownloaderTests(base_downloader.BaseDownloaderTests): @@ -40,7 +29,7 @@ def test_retrieve_metadata(self, mock_finalize): # Verify self.assertEqual(1, len(docs)) - metadata = model.RepositoryMetadata() + metadata = RepositoryMetadata() metadata.update_from_json(docs[0]) self.assertEqual(2, len(metadata.modules)) @@ -63,7 +52,10 @@ def test_retrieve_metadata_no_metadata_found(self): except FileRetrievalException: pass - def test_retrieve_module(self): + @mock.patch(MODULE_PATH + '.os.path.exists') + def test_retrieve_module(self, mock_exists): + mock_exists.return_value = True + # Test mod_path = self.downloader.retrieve_module(self.mock_progress_report, self.module) @@ -71,7 +63,10 @@ def test_retrieve_module(self): expected = os.path.join(VALID_REPO_DIR, self.module.filename()) self.assertEqual(expected, mod_path) - def test_retrieve_module_no_file(self): + @mock.patch(MODULE_PATH + '.os.path.exists') + def test_retrieve_module_no_file(self, mock_exists): + mock_exists.return_value = False + # Setup self.module.author = 'foo' diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_web.py b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_web.py index a09eb2aa..f2bace23 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_web.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/downloaders/test_web.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os import mock @@ -18,10 +5,11 @@ from nectar.report import DownloadReport import base_downloader -from pulp_puppet.common import constants, model +from pulp_puppet.common import constants from pulp_puppet.plugins.importers.downloaders import exceptions, web from pulp_puppet.plugins.importers.downloaders.web import HttpDownloader + TEST_SOURCE = 'http://forge.puppetlabs.com/' @@ -69,21 +57,23 @@ def test_retrieve_metadata_with_error(self, mock_downloader_download, mock_liste except exceptions.FileRetrievalException: pass - @mock.patch('nectar.config.DownloaderConfig.finalize') - @mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download') - def test_retrieve_module(self, mock_downloader_download, mock_finalize): + @mock.patch.object(HttpDownloader, 'retrieve_modules') + def test_retrieve_module(self, mock_retrieve_modules): + mock_retrieve_modules.return_value = ['foo', 'bar'] try: stored_filename = self.downloader.retrieve_module(self.mock_progress_report, self.module) except: self.fail() - mock_downloader_download.assert_called_once() - mock_finalize.assert_called_once() + mock_retrieve_modules.assert_called_once_with(self.mock_progress_report, [self.module]) + self.assertEqual(stored_filename, 'foo') @mock.patch('pulp_puppet.plugins.importers.downloaders.web.HTTPModuleDownloadEventListener') @mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download') def test_retrieve_module_missing_module(self, mock_downloader_download, mock_listener_constructor): # Setup + self.module.author = 'asdf' + self.module.puppet_standard_filename.return_value = 'puppet-filename.tar.gz' mock_listener = mock.MagicMock() report = DownloadReport(None, None) report.error_msg = 'oops' @@ -97,10 +87,11 @@ def test_retrieve_module_missing_module(self, mock_downloader_download, mock_lis except exceptions.FileRetrievalException: expected_filename = web._create_download_tmp_dir(self.working_dir) expected_filename = os.path.join(expected_filename, self.module.filename()) - self.assertFalse(os.path.exists(os.path.join(expected_filename))) @mock.patch('nectar.downloaders.threaded.HTTPThreadedDownloader.download') def test_cleanup_module(self, mock_downloader_download): + self.module.author = 'asdf' + self.module.puppet_standard_filename.return_value = 'puppet-filename.tar.gz' stored_filename = self.downloader.retrieve_module(self.mock_progress_report, self.module) self.downloader.cleanup_module(self.module) self.assertTrue(not os.path.exists(stored_filename)) @@ -127,6 +118,11 @@ def test_create_metadata_download_urls_no_queries(self): self.assertEqual(urls[0], TEST_SOURCE + 'modules.json') def test_create_module_url(self): + # Setup + self.module.author = 'asdf' + self.module.puppet_standard_filename.return_value = 'puppet-filename.tar.gz' + self.module.filename.return_value = 'puppet-filename.tar.gz' + # Test # Strip the trailing / off to make sure that branch is followed diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/test_configuration.py b/pulp_puppet_plugins/test/unit/plugins/importer/test_configuration.py index 8b7ac5fd..48eb236f 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/test_configuration.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/test_configuration.py @@ -1,15 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. """ This module contains tests for the pulp_puppet.plugins.importers.configuration module. """ diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/test_copier.py b/pulp_puppet_plugins/test/unit/plugins/importer/test_copier.py index f0b12720..97b57581 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/test_copier.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/test_copier.py @@ -1,34 +1,12 @@ import unittest import mock -from pulp.server.db.model.criteria import UnitAssociationCriteria -from pulp_puppet.common import constants from pulp_puppet.plugins.importers import copier class CopierTests(unittest.TestCase): - def test_copy_units_all_units(self): - # Setup - conduit = mock.MagicMock() - all_source_units = ['a', 'b', 'c'] # content is irrelevant - conduit.get_source_units.return_value = all_source_units - - # Test - returned = copier.copy_units(conduit, None) - - # Verify - self.assertEqual(returned, all_source_units) - self.assertEqual(1, conduit.get_source_units.call_count) - call_args = conduit.get_source_units.call_args - self.assertTrue('criteria' in call_args[1]) - self.assertTrue(isinstance(call_args[1]['criteria'], UnitAssociationCriteria)) - self.assertEqual(call_args[1]['criteria'].type_ids, [constants.TYPE_PUPPET_MODULE]) - - self.assertEqual(len(all_source_units), conduit.associate_unit.call_count) - self._assert_associated_units(conduit, all_source_units) - def test_copy_units_only_specified(self): # Setup conduit = mock.MagicMock() diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/test_directory.py b/pulp_puppet_plugins/test/unit/plugins/importer/test_directory.py index 91b25147..ddcec278 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/test_directory.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/test_directory.py @@ -1,14 +1,3 @@ -# Copyright (c) 2014 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os from uuid import uuid4 @@ -26,12 +15,13 @@ class TestSynchronizeWithDirectory(TestCase): def test_constructor(self): + mock_repo = Mock() conduit = Mock() config = {} # testing - method = SynchronizeWithDirectory(conduit, config) + method = SynchronizeWithDirectory(mock_repo, conduit, config) # validation @@ -40,24 +30,26 @@ def test_constructor(self): def test_feed_url(self): feed_url = 'http://abc.com/repository' + mock_repo = Mock() conduit = Mock() config = {constants.CONFIG_FEED: feed_url} # testing - method = SynchronizeWithDirectory(conduit, config) + method = SynchronizeWithDirectory(mock_repo, conduit, config) # validation self.assertEqual(method.feed_url(), feed_url + '/') def test_cancel(self): + mock_repo = Mock() conduit = Mock() config = {} # testing - method = SynchronizeWithDirectory(conduit, config) + method = SynchronizeWithDirectory(mock_repo, conduit, config) method.cancel() # validation @@ -74,6 +66,7 @@ def test_call(self, mock_mkdtemp, mock_rmtree, mock_fetch_manifest, mock_fetch_m mock_import_modules, mock_remove_missing): mock_fetch_manifest.return_value = 'manifest_destiny' mock_fetch_modules.return_value = 'some modules' + mock_repo = Mock() conduit = Mock() config = {constants.CONFIG_FEED: 'http://host/root/PULP_MANAFEST'} repository = Mock() @@ -81,8 +74,8 @@ def test_call(self, mock_mkdtemp, mock_rmtree, mock_fetch_manifest, mock_fetch_m mock_mkdtemp.return_value = '/abc' # testing - method = SynchronizeWithDirectory(conduit, config) - report = method(repository) + method = SynchronizeWithDirectory(mock_repo, conduit, config) + report = method() # validation self.assertEqual(1, mock_fetch_manifest.call_count) @@ -92,7 +85,7 @@ def test_call(self, mock_mkdtemp, mock_rmtree, mock_fetch_manifest, mock_fetch_m self.assertFalse(method.canceled) self.assertTrue(isinstance(method.report, SyncProgressReport)) self.assertTrue(isinstance(report, SyncProgressReport)) - mock_mkdtemp.assert_called_with(dir=repository.working_dir) + mock_mkdtemp.assert_called_with(dir=mock_repo.working_dir) mock_rmtree.assert_called_with(os.path.join(repository.working_dir, mock_mkdtemp())) @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._import_modules') @@ -102,6 +95,7 @@ def test_call(self, mock_mkdtemp, mock_rmtree, mock_fetch_manifest, mock_fetch_m @patch('pulp_puppet.plugins.importers.directory.mkdtemp') def test_call_no_manifest(self, mock_mkdtemp, mock_rmtree, mock_fetch_manifest, *mocks): mock_fetch_manifest.return_value = None + mock_repo = Mock() conduit = Mock() config = {constants.CONFIG_FEED: 'http://host/root/PULP_MANAFEST'} repository = Mock() @@ -109,8 +103,8 @@ def test_call_no_manifest(self, mock_mkdtemp, mock_rmtree, mock_fetch_manifest, mock_mkdtemp.return_value = '/abc' # testing - method = SynchronizeWithDirectory(conduit, config) - report = method(repository) + method = SynchronizeWithDirectory(mock_repo, conduit, config) + report = method() # validation self.assertEqual(1, mock_fetch_manifest.call_count) @@ -119,7 +113,7 @@ def test_call_no_manifest(self, mock_mkdtemp, mock_rmtree, mock_fetch_manifest, self.assertFalse(method.canceled) self.assertTrue(isinstance(method.report, SyncProgressReport)) self.assertTrue(isinstance(report, SyncProgressReport)) - mock_mkdtemp.assert_called_with(dir=repository.working_dir) + mock_mkdtemp.assert_called_with(dir=mock_repo.working_dir) mock_rmtree.assert_called_with(os.path.join(repository.working_dir, mock_mkdtemp())) @patch('pulp_puppet.plugins.importers.directory.URL_TO_DOWNLOADER') @@ -132,6 +126,7 @@ def test_download(self, mock_listener, mock_nectar_config, mock_downloader_mappi mock_http_class = Mock(return_value=mock_http_downloader) mock_downloader_mapping.__getitem__.return_value = mock_http_class + mock_repo = Mock() conduit = Mock() config = Mock() config.get = Mock(side_effect={constants.CONFIG_FEED: 'http://host/root/PULP_MANAFEST'}) @@ -150,7 +145,7 @@ def test_download(self, mock_listener, mock_nectar_config, mock_downloader_mappi # test - method = SynchronizeWithDirectory(conduit, config) + method = SynchronizeWithDirectory(mock_repo, conduit, config) succeeded_reports, failed_reports = method._download(urls) # validation @@ -183,6 +178,7 @@ def test_download(self, mock_listener, mock_nectar_config, mock_downloader_mappi def test_fetch_manifest(self, mock_download, mock_get_value): feed_url = 'http://host/root/' + mock_repo = Mock() conduit = Mock() config = {constants.CONFIG_FEED: feed_url} succeeded_report = Mock() @@ -192,7 +188,7 @@ def test_fetch_manifest(self, mock_download, mock_get_value): # test - method = SynchronizeWithDirectory(conduit, config) + method = SynchronizeWithDirectory(mock_repo, conduit, config) method.report = Mock() manifest = method._fetch_manifest() @@ -213,6 +209,7 @@ def test_fetch_manifest(self, mock_download, mock_get_value): def test_fetch_manifest_failed(self, mock_download): feed_url = 'http://host/root/' + mock_repo = Mock() conduit = Mock() config = {constants.CONFIG_FEED: feed_url} failed_report = Mock() @@ -222,7 +219,7 @@ def test_fetch_manifest_failed(self, mock_download): # test - method = SynchronizeWithDirectory(conduit, config) + method = SynchronizeWithDirectory(mock_repo, conduit, config) method.report = Mock() manifest = method._fetch_manifest() @@ -242,6 +239,7 @@ def test_fetch_modules(self, mock_download): tmp_dir = '/tmp/puppet-testing' feed_url = 'http://host/root/' + mock_repo = Mock() conduit = Mock() config = {constants.CONFIG_FEED: feed_url} @@ -255,7 +253,7 @@ def test_fetch_modules(self, mock_download): # test - method = SynchronizeWithDirectory(conduit, config) + method = SynchronizeWithDirectory(mock_repo, conduit, config) method.report = Mock() method.tmp_dir = '/tmp/puppet-testing' module_paths = method._fetch_modules(manifest) @@ -282,6 +280,7 @@ def test_fetch_modules_failures(self, mock_download): tmp_dir = '/tmp/puppet-testing' feed_url = 'http://host/root/' + mock_repo = Mock() conduit = Mock() config = {constants.CONFIG_FEED: feed_url} @@ -296,7 +295,7 @@ def test_fetch_modules_failures(self, mock_download): # test - method = SynchronizeWithDirectory(conduit, config) + method = SynchronizeWithDirectory(mock_repo, conduit, config) method.report = Mock() method.tmp_dir = '/tmp/puppet-testing' module_paths = method._fetch_modules(manifest) @@ -362,214 +361,6 @@ def test_extract_metadata(self, *mocks): mock_json.load.assert_called_with(mock_fp) mock_shutil.rmtree.assert_called_with(mock_mkdtemp()) - @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._remove_missing') - @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._add_module') - @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._extract_metadata') - def test_import_modules(self, mock_extract, mock_add, mock_remove_missing): - # These manifests represent the parsed metadata.json file. These contain a 'name' - # field, where we retrieve both the unit key's 'name' and 'author' field. - manifests = [ - {'name': 'john-pulp1', 'author': 'Johnathon', 'version': '1.0'}, - {'name': 'john-pulp2', 'author': 'Johnathon', 'version': '2.0'}, - {'name': 'john/pulp3', 'author': 'Johnathon', 'version': '3.0'}, - ] - mock_extract.side_effect = manifests - unit_keys = [ - {'name': 'pulp1', 'author': 'john', 'version': '1.0'}, - {'name': 'pulp2', 'author': 'john', 'version': '2.0'}, - {'name': 'pulp3', 'author': 'john', 'version': '3.0'}, - ] - module_paths = [ - '/tmp/module_1', - '/tmp/module_2', - '/tmp/module_3', - ] - mock_pulp2 = Mock() - mock_pulp2.unit_key = unit_keys[1] - conduit = Mock() - conduit.get_units.return_value = [mock_pulp2] - config = Mock() - config.get_boolean.return_value = False - - # test - method = SynchronizeWithDirectory(conduit, config) - method.started_fetch_modules = 10 - method.report = Mock() - method.report.modules_total_count = 3 - method.report.modules_finished_count = 0 - method._import_modules(module_paths) - - # validation - self.assertEqual(3, mock_extract.call_count) - self.assertEqual(mock_extract.mock_calls[0][1][0], module_paths[0]) - self.assertEqual(mock_extract.mock_calls[1][1][0], module_paths[1]) - self.assertEqual(mock_extract.mock_calls[2][1][0], module_paths[2]) - - self.assertEqual(2, mock_add.call_count) - self.assertEqual(mock_add.mock_calls[0][1][0], module_paths[0]) - self.assertEqual(mock_add.mock_calls[1][1][0], module_paths[2]) - - config.get_boolean.assert_called_once_with(constants.CONFIG_REMOVE_MISSING) - self.assertEqual(0, mock_remove_missing.call_count) - - # Check that the progress reporting was called as expected - self.assertEquals(3, method.report.update_progress.call_count) - self.assertEquals(2, method.report.modules_finished_count) - self.assertEquals(2, method.report.modules_total_count) - - @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._remove_missing') - @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._add_module') - @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._extract_metadata') - def test_import_modules(self, mock_extract, mock_add, mock_remove_missing): - # These manifests represent the parsed metadata.json file. These contain a 'name' - # field, where we retrieve both the unit key's 'name' and 'author' field. - manifest = [{'name': 'john-pulp1', 'author': 'Johnathon', 'version': '1.0'}] - mock_extract.side_effect = manifest - module_paths = ['/tmp/module_1'] - mock_pulp1, mock_pulp2 = (Mock(), Mock()) - mock_pulp1.unit_key = {'name': 'pulp1', 'author': 'john', 'version': '1.0'} - mock_pulp2.unit_key = {'name': 'pulp2', 'author': 'john', 'version': '2.0'} - conduit = Mock() - conduit.get_units.return_value = [mock_pulp1, mock_pulp2] - config = Mock() - config.get_boolean.return_value = True - - # test - method = SynchronizeWithDirectory(conduit, config) - method.started_fetch_modules = 10 - method.report = Mock() - method.report.modules_total_count = 2 - method.report.modules_finished_count = 0 - method._import_modules(module_paths) - - # validation - config.get_boolean.assert_called_once_with(constants.CONFIG_REMOVE_MISSING) - mock_remove_missing.assert_called_once_with([mock_pulp1, mock_pulp2], [mock_pulp1.unit_key]) - - @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._extract_metadata') - def test_import_modules_cancelled(self, mock_extract): - config = {} - mock_conduit = Mock() - mock_conduit.get_units.return_value = [] - - # test - method = SynchronizeWithDirectory(mock_conduit, config) - method.canceled = True - method._import_modules(['/path1', '/path2']) - - # validation - self.assertFalse(mock_extract.called) - - @patch('pulp_puppet.common.model.Module.from_json') - @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._add_module') - @patch('pulp_puppet.plugins.importers.directory.SynchronizeWithDirectory._extract_metadata') - def test_import_modules_failed(self, *mocks): - """ - Test that when there was some failure in a previous step, _import_modules does not - overwrite the failed state - """ - mock_conduit = Mock() - mock_conduit.get_units.return_value = [] - - method = SynchronizeWithDirectory(mock_conduit, Mock()) - method.started_fetch_modules = 0 - metadata = {'name': 'j-p', 'author': 'J', 'version': '1.1'} - method._extract_metadata = Mock(return_value=metadata) - method.report = Mock() - method.report.modules_total_count = 1 - method.report.modules_finished_count = 0 - method.report.modules_state = constants.STATE_FAILED - - # test - method._import_modules(['/path1']) - - # validation - self.assertEquals(constants.STATE_FAILED, method.report.modules_state) - self.assertEquals(2, method.report.update_progress.call_count) - self.assertEquals(1, method.report.modules_total_count) - self.assertEquals(1, method.report.modules_finished_count) - - def test_remove_missing(self): - """ - Test that when there are units to remove, the conduit is called correctly. - """ - mock_unit = Mock() - mock_conduit = Mock() - method = SynchronizeWithDirectory(mock_conduit, {}) - - method._remove_missing([mock_unit], []) - mock_conduit.remove_unit.assert_called_once_with(mock_unit) - - def test_remove_missing_canceled(self): - """ - Test that when the sync is canceled, no units are removed. - """ - mock_unit = Mock() - mock_conduit = Mock() - method = SynchronizeWithDirectory(mock_conduit, {}) - method.canceled = True - - method._remove_missing([mock_unit], []) - self.assertEqual(0, mock_conduit.remove_unit.call_count) - - @patch('pulp_puppet.plugins.importers.directory.shutil') - def test_add_module(self, mock_shutil): - module_path = '/tmp/mod.tar.gz' - feed_url = 'http://host/root/PULP_MANAFEST' - unit_key = {'name': 'puppet-module'} - unit_metadata = {'A': 1, 'B': 2} - unit = Mock() - unit.storage_path = '/tmp/%s' % uuid4() - - mock_conduit = Mock() - mock_conduit.init_unit = Mock(return_value=unit) - - config = {constants.CONFIG_FEED: feed_url} - - mock_module = Mock() - mock_module.unit_key = Mock(return_value=unit_key) - mock_module.unit_metadata = Mock(return_value=unit_metadata) - mock_module.filename = Mock(return_value='puppet-module') - - # test - - method = SynchronizeWithDirectory(mock_conduit, config) - method._add_module(module_path, mock_module) - - # validation - - mock_conduit.init_unit.assert_called_with( - constants.TYPE_PUPPET_MODULE, unit_key, unit_metadata, mock_module.filename()) - mock_shutil.copy.assert_called_with(module_path, unit.storage_path) - - @patch('pulp_puppet.plugins.importers.directory.shutil') - def test_add_module_not_copied(self, mock_shutil): - module_path = '/tmp/mod.tar.gz' - feed_url = 'http://host/root/PULP_MANAFEST' - unit_key = {'name': 'puppet-module'} - unit_metadata = {'A': 1, 'B': 2} - unit = Mock() - unit.storage_path = os.path.join(os.getcwd(), __file__) - - mock_conduit = Mock() - mock_conduit.init_unit = Mock(return_value=unit) - - config = {constants.CONFIG_FEED: feed_url} - - mock_module = Mock() - mock_module.unit_key = Mock(return_value=unit_key) - mock_module.unit_metadata = Mock(return_value=unit_metadata) - mock_module.filename = Mock(return_value='puppet-module') - - # test - - method = SynchronizeWithDirectory(mock_conduit, config) - method._add_module(module_path, mock_module) - - # validation - - self.assertFalse(mock_shutil.copy.called) - class TestListener(TestCase): diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/test_forge.py b/pulp_puppet_plugins/test/unit/plugins/importer/test_forge.py index e815858d..75965680 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/test_forge.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/test_forge.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os import shutil import tempfile @@ -21,7 +8,8 @@ from pulp.plugins.config import PluginCallConfiguration from pulp.plugins.model import Repository, SyncReport, Unit -from pulp_puppet.common import constants, model, sync_progress +from pulp_puppet.common import constants, sync_progress +from pulp_puppet.plugins.db.models import Module, RepositoryMetadata from pulp_puppet.plugins.importers.forge import SynchronizeWithPuppetForge @@ -92,34 +80,6 @@ def test___init__(self): self.assertEqual(swpf.downloader, None) self.assertEqual(swpf._canceled, False) - @mock.patch('pulp_puppet.plugins.importers.forge.SynchronizeWithPuppetForge._add_new_module') - def test__do_import_modules_handles_cancel(self, _add_new_module): - """ - Make sure _do_import_modules() handles the cancel signal correctly. We'll do this by setting - up a side effect with the first module to call cancel so the second never happens. - """ - swpf = SynchronizeWithPuppetForge(self.repo, self.conduit, self.config) - - def _side_effect(*args, **kwargs): - swpf.cancel() - - _add_new_module.side_effect = _side_effect - metadata = model.RepositoryMetadata() - module_1 = model.Module('module_1', '1.0.0', 'simon') - module_2 = model.Module('module_2', '2.0.3', 'garfunkel') - metadata.modules = [module_1, module_2] - - swpf._do_import_modules(metadata) - - # If _add_new_module was called exactly once, then our cancel was successful because the - # first call to _add_new_module set the cancel flag, and the loop exited the next time. - # Because dictionaries are involved in the order in which the modules get downloaded, we - # don't have a documented guarantee about which module will be the one. Therefore, we'll - # just assert that only one was downloaded and that it was one of the two. - self.assertEqual(_add_new_module.call_count, 1) - downloaded_module = _add_new_module.mock_calls[0][1][1] - self.assertTrue(downloaded_module in [module_1, module_2]) - def test_cancel_downloader_none(self): """ Ensure correct operation of the cancel() method when the downloader is None. @@ -143,50 +103,6 @@ def test_cancel_downloader_set(self): self.assertEqual(swpf._canceled, True) swpf.downloader.cancel.assert_called_once_with() - def test_synchronize(self): - # Test - report = self.method().build_final_report() - - # Verify - - # Units copied to simulated Pulp storage - expected_module_filenames = ['adob-good-2.0.0.tar.gz', 'jdob-valid-1.1.0.tar.gz'] - for f in expected_module_filenames: - expected_path = os.path.join(MOCK_PULP_STORAGE_LOCATION, f) - self.assertTrue(os.path.exists(expected_path)) - - # Final Report - self.assertTrue(report.success_flag) - self.assertTrue(report.summary['total_execution_time'] is not None) - self.assertTrue(report.summary['total_execution_time'] > -1) - - self.assertEqual(report.details['total_count'], 2) - self.assertEqual(report.details['finished_count'], 2) - self.assertEqual(report.details['error_count'], 0) - - # Progress Reporting - pr = self.method.progress_report - self.assertEqual(pr.metadata_state, constants.STATE_SUCCESS) - self.assertEqual(pr.metadata_query_total_count, 1) - self.assertEqual(pr.metadata_query_finished_count, 1) - self.assertTrue(pr.metadata_execution_time is not None) - self.assertEqual(pr.metadata_error_message, None) - self.assertEqual(pr.metadata_exception, None) - self.assertEqual(pr.metadata_traceback, None) - - self.assertEqual(pr.modules_state, constants.STATE_SUCCESS) - self.assertEqual(pr.modules_total_count, 2) - self.assertEqual(pr.modules_error_count, 0) - self.assertEqual(pr.modules_finished_count, 2) - self.assertTrue(pr.modules_execution_time is not None) - self.assertEqual(pr.modules_error_message, None) - self.assertEqual(pr.modules_exception, None) - self.assertEqual(pr.modules_traceback, None) - self.assertEqual(pr.modules_individual_errors, []) - - # Number of times update was called on the progress report - self.assertEqual(self.conduit.set_progress.call_count, 9) - def test_synchronize_metadata_error(self): # Setup self.config.repo_plugin_config[constants.CONFIG_FEED] = INVALID_FEED @@ -220,23 +136,6 @@ def test_synchronize_no_feed(self): self.assertTrue(len(pr.metadata_error_message) > 0) self.assertEqual(pr.modules_state, constants.STATE_NOT_STARTED) - @mock.patch('pulp_puppet.plugins.importers.forge.SynchronizeWithPuppetForge._resolve_remove_units') - def test_synchronize_with_remove_units(self, mock_resolve): - # Setup - remove_me = 'valid-1.1.0-jdob' - mock_resolve.return_value = [remove_me] - - self.conduit = UnitsMockConduit() - self.method.sync_conduit = self.conduit - - self.config.repo_plugin_config[constants.CONFIG_REMOVE_MISSING] = 'true' - - # Test - report = self.method() - - # Verify - self.assertEqual(1, self.conduit.remove_unit.call_count) - @mock.patch('pulp_puppet.plugins.importers.forge.SynchronizeWithPuppetForge._parse_metadata') def test_synchronize_no_metadata(self, mock_parse): # Setup @@ -342,36 +241,3 @@ def test_import_modules_exception(self, mock_import): self.assertTrue(pr.modules_error_message is not None) self.assertTrue(pr.modules_exception is not None) self.assertTrue(pr.modules_traceback is not None) - - @mock.patch('pulp_puppet.plugins.importers.forge.SynchronizeWithPuppetForge._add_new_module') - def test_do_import_add_exception(self, mock_add): - # Setup - mock_add.side_effect = Exception() - - # Test - report = self.method().build_final_report() - - # Verify - - # Failed modules still represent a successful sync and import modules - # step as far as states are concerned. But at the individual module - # level, the errors should be stored per failed module and the counts - # accurately reflect successes v. failures. - - self.assertTrue(report.success_flag) - - pr = self.method.progress_report - self.assertEqual(pr.metadata_state, constants.STATE_SUCCESS) - - self.assertEqual(pr.modules_state, constants.STATE_SUCCESS) - self.assertEqual(pr.modules_total_count, 2) - self.assertEqual(pr.modules_finished_count, 0) - self.assertEqual(pr.modules_error_count, 2) - self.assertEqual(len(pr.modules_individual_errors), 2) - # Make sure the individual_errors are the correct format - for error in pr.modules_individual_errors: - self.assertEqual(set(error.keys()), set(['exception', 'traceback', 'module', 'author'])) - self.assertTrue(pr.modules_execution_time is not None) - self.assertTrue(pr.modules_error_message is None) - self.assertTrue(pr.modules_exception is None) - self.assertTrue(pr.modules_traceback is None) diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/test_importer.py b/pulp_puppet_plugins/test/unit/plugins/importer/test_importer.py index 81e3dd45..dbda6d6c 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/test_importer.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/test_importer.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest from mock import Mock, patch @@ -50,7 +37,7 @@ def test_directory_synchronization(self, forge_call, mock_call): # validation - mock_call.assert_called_with(repository) + mock_call.assert_called_with() self.assertEquals(report, conduit.build_success_report.return_value) self.assertFalse(forge_call.called) diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/test_metadata.py b/pulp_puppet_plugins/test/unit/plugins/importer/test_metadata.py index 1258bc83..22ebb488 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/test_metadata.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/test_metadata.py @@ -3,122 +3,14 @@ import tempfile import unittest -from mock import patch - +import mock from pulp.server.exceptions import InvalidValue -from pulp_puppet.common.model import Module from pulp_puppet.plugins.importers import metadata DATA_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../../../data' - - -class SuccessfulMetadataTests(unittest.TestCase): - - def setUp(self): - self.author = 'jdob' - self.name = 'valid' - self.version = '1.0.0' - - self.module = Module(self.name, self.version, self.author) - - self.module_dir = os.path.join(DATA_DIR, 'good-modules', 'jdob-valid', 'pkg') - self.tmp_dir = tempfile.mkdtemp(prefix='puppet-metadata-tests') - - def tearDown(self): - if os.path.exists(self.tmp_dir): - shutil.rmtree(self.tmp_dir) - - def test_extract_metadata(self): - # Setup - filename = os.path.join(self.module_dir, self.module.filename()) - - # Test - metadata_json = metadata.extract_metadata(filename, self.tmp_dir) - self.module = Module.from_json(metadata_json) - - # Verify - self.assertEqual(self.module.name, 'valid') - self.assertEqual(self.module.version, '1.0.0') - self.assertEqual(self.module.author, 'jdob') - - self._assert_test_module_metadata() - - @patch("tempfile.mkdtemp") - def test_extract_metadata_non_standard_packaging(self, mkdtemp): - # Setup - self.module = Module('misnamed', '1.0.0', 'ldob') - self.module_dir = os.path.join(DATA_DIR, 'bad-modules') - filename = os.path.join(self.module_dir, self.module.filename()) - extraction_dir = os.path.join(self.tmp_dir, "test") - mkdtemp.return_value = extraction_dir - - # Test - metadata_json = metadata.extract_metadata(filename, self.tmp_dir) - self.module.update_from_dict(metadata_json) - - # Verify - contains the same module as jdob-valid-1.0.0, so this is safe - self.assertEqual(self.module.name, 'misnamed') - self.assertEqual(self.module.version, '1.0.0') - self.assertEqual(self.module.author, 'ldob') - - self._assert_test_module_metadata() - - self.assertTrue(not os.path.exists(extraction_dir)) - - @patch("tempfile.mkdtemp") - def test_extract_metadata_no_module(self, mkdtemp): - # Setup - filename = os.path.join(self.module_dir, self.module.filename()) - extraction_dir = os.path.join(self.tmp_dir, "1234") - mkdtemp.return_value = extraction_dir - - metadata_json = metadata.extract_metadata(filename, self.tmp_dir) - self.module = Module.from_json(metadata_json) - - # Verify - self.assertEqual(self.module.name, 'valid') - self.assertEqual(self.module.version, '1.0.0') - self.assertEqual(self.module.author, 'jdob') - - self._assert_test_module_metadata() - - self.assertTrue(not os.path.exists(extraction_dir)) - - def _assert_test_module_metadata(self): - - # Assumes the content in jdob-valid-1.0.0 - - self.assertEqual(self.module.source, 'http://example.org/jdob-valid/source') - self.assertEqual(self.module.license, 'Apache License, Version 2.0') - self.assertEqual(self.module.summary, 'Valid Module Summary') - self.assertEqual(self.module.description, 'Valid Module Description') - self.assertEqual(self.module.project_page, 'http://example.org/jdob-valid') - - self.assertEqual(2, len(self.module.dependencies)) - sorted_deps = sorted(self.module.dependencies, key=lambda x :x['name']) - self.assertEqual(sorted_deps[0]['name'], 'jdob/dep-alpha') - self.assertEqual(sorted_deps[0]['version_requirement'], '>= 1.0.0') - self.assertEqual(sorted_deps[1]['name'], 'ldob/dep-beta') - self.assertEqual(sorted_deps[1]['version_requirement'], '>= 2.0.0') - - self.assertEqual(self.module.types, []) - - expected_checksums = { - 'Modulefile': '704cecf2957448dcf7fa20cffa2cf7c1', - 'README': '11edd8578497566d8054684a8c89c6cb', - 'manifests/init.pp': '1d1fb26825825b4d64d37d377016869e', - 'spec/spec_helper.rb': 'a55d1e6483344f8ec6963fcb2c220372', - 'tests/init.pp': '7043c7ef0c4b0ac52b4ec6bb76008ebd' - } - self.assertEqual(self.module.checksums, expected_checksums) - - def test_checksum_calculation(self): - sample_module = os.path.join(self.module_dir, "jdob-valid-1.1.0.tar.gz") - sample_checksum = metadata.calculate_checksum(sample_module) - self.assertEquals(sample_checksum, - "108e8d1d9bb42c869344fc2d327c80e7f079d2ba0119da446a6a1c6659e0f0aa") +MODULE_STRING = 'pulp_puppet.plugins.importers.metadata' class NegativeMetadataTests(unittest.TestCase): @@ -128,7 +20,7 @@ def setUp(self): self.name = None # set in test itself self.version = '1.0.0' - self.module = Module(self.name, self.version, self.author) + self.module = mock.Mock() self.module_dir = os.path.join(DATA_DIR, 'bad-modules') self.tmp_dir = tempfile.mkdtemp(prefix='puppet-metadata-tests') @@ -163,7 +55,8 @@ def test_extract_non_standard_bad_tarball(self): except metadata.ExtractionException, e: self.assertEqual(e.module_filename, filename) - def test_extract_metadata_no_metadata(self): + @mock.patch(MODULE_STRING + '.tarfile') + def test_extract_metadata_no_metadata(self, mock_tarfile): # Setup self.module.name = 'no-metadata' filename = os.path.join(self.module_dir, self.module.filename()) diff --git a/pulp_puppet_plugins/test/unit/plugins/importer/test_upload.py b/pulp_puppet_plugins/test/unit/plugins/importer/test_upload.py index 921012f3..ac9f18f0 100644 --- a/pulp_puppet_plugins/test/unit/plugins/importer/test_upload.py +++ b/pulp_puppet_plugins/test/unit/plugins/importer/test_upload.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os import shutil import tempfile @@ -23,6 +10,7 @@ from pulp_puppet.plugins.importers import upload DATA_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../../../data' +MODULE_STRING = 'pulp_puppet.plugins.importers.upload' class UploadTests(unittest.TestCase): @@ -51,7 +39,9 @@ def tearDown(self): if os.path.exists(self.dest_dir): shutil.rmtree(self.dest_dir) - def test_handle_uploaded_unit(self): + @mock.patch(MODULE_STRING + '.Module') + @mock.patch(MODULE_STRING + '.repo_controller') + def test_handle_uploaded_unit(self, mock_repo_controller, mock_module): # Setup initialized_unit = mock.MagicMock() initialized_unit.storage_path = self.dest_dir @@ -62,10 +52,7 @@ def test_handle_uploaded_unit(self): self.unit_metadata, self.source_file, self.conduit) # Verify - self.assertTrue(os.path.exists(self.dest_file)) - - self.assertEqual(1, self.conduit.init_unit.call_count) - self.assertEqual(1, self.conduit.save_unit.call_count) + mock_module.from_metadata.return_value.save.assert_called_once_with() self.assertTrue(isinstance(report, dict)) self.assertTrue('success_flag' in report) @@ -73,7 +60,9 @@ def test_handle_uploaded_unit(self): self.assertTrue('summary' in report) self.assertTrue('details' in report) - def test_handle_uploaded_unit_with_no_data(self): + @mock.patch(MODULE_STRING + '.Module') + @mock.patch(MODULE_STRING + '.repo_controller') + def test_handle_uploaded_unit_with_no_data(self, mock_repo_controller, mock_module): # Setup initialized_unit = mock.MagicMock() initialized_unit.storage_path = self.dest_dir @@ -83,11 +72,7 @@ def test_handle_uploaded_unit_with_no_data(self): report = upload.handle_uploaded_unit(self.repo, constants.TYPE_PUPPET_MODULE, {}, {}, self.source_file, self.conduit) - # Verify - self.assertTrue(os.path.exists(self.dest_file)) - - self.assertEqual(1, self.conduit.init_unit.call_count) - self.assertEqual(1, self.conduit.save_unit.call_count) + mock_module.from_metadata.return_value.save.assert_called_once_with() self.assertTrue(report['success_flag']) diff --git a/pulp_puppet_plugins/test/unit/plugins/migrations/test_0001_puppet_module_unit_checksum.py b/pulp_puppet_plugins/test/unit/plugins/migrations/test_0001_puppet_module_unit_checksum.py index ef6ed8b5..4e577154 100644 --- a/pulp_puppet_plugins/test/unit/plugins/migrations/test_0001_puppet_module_unit_checksum.py +++ b/pulp_puppet_plugins/test/unit/plugins/migrations/test_0001_puppet_module_unit_checksum.py @@ -1,15 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. """ Tests for pulp_rpm.plugins.migrations.0001_puppet_module_unit_checksum """ @@ -45,5 +33,3 @@ def test_migration(self, mock_query_manager, mock_calc_checksum): save.call_args[0][0] self.assertEquals(target_unit['checksum'], 'foo_checksum') self.assertEquals(target_unit['checksum_type'], constants.DEFAULT_HASHLIB) - - diff --git a/pulp_puppet_plugins/test/unit/plugins/migrations/test_0002_puppet_publishing_directory_change.py b/pulp_puppet_plugins/test/unit/plugins/migrations/test_0002_puppet_publishing_directory_change.py index dce4641f..2ac8de6d 100644 --- a/pulp_puppet_plugins/test/unit/plugins/migrations/test_0002_puppet_publishing_directory_change.py +++ b/pulp_puppet_plugins/test/unit/plugins/migrations/test_0002_puppet_publishing_directory_change.py @@ -1,17 +1,5 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2014 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. """ -Tests for pulp_rpm.plugins.migrations.0002_puppet_publishing_directory_change +Tests for pulp_puppet.plugins.migrations.0002_puppet_publishing_directory_change """ import os import tempfile @@ -36,7 +24,7 @@ def test_migration(self, mock_listdir, mock_path_exists, mock_move_directory): 'publishing_directory_change') migration.migrate() mock_listdir.assert_called_once_with('/var/www/pulp_puppet') - mock_path_exists.assert_called_once_with('/var/www/pulp_puppet') + mock_path_exists.assert_has_call('/var/www/pulp_puppet') def test_move_directory_contents(self): test_old_publish_dir = tempfile.mkdtemp(prefix='test_0002_migration_old') diff --git a/pulp_puppet_plugins/test/unit/plugins/migrations/test_0003_puppet_drop_module_indexes.py b/pulp_puppet_plugins/test/unit/plugins/migrations/test_0003_puppet_drop_module_indexes.py new file mode 100644 index 00000000..d7c7d38c --- /dev/null +++ b/pulp_puppet_plugins/test/unit/plugins/migrations/test_0003_puppet_drop_module_indexes.py @@ -0,0 +1,24 @@ +""" +Tests for pulp_puppet.plugins.migrations.0003_puppet_drop_module_indexes +""" +import unittest + +from mock import patch, call + +from pulp.server.db.migrate.models import _import_all_the_way + + +migration = _import_all_the_way('pulp_puppet.plugins.migrations.0003_puppet_drop_module_indexes') + + +class Test0003PuppetIndexesDropped(unittest.TestCase): + """ + Test the migration of dropping the puppet module indexes + """ + + @patch.object(migration, 'get_collection') + def test_migration(self, mock_get_collection): + migration.migrate() + mock_get_collection.assert_called_once_with('units_puppet_module') + calls = [call('name_1_version_1_author_1'), call('author_1'), call('tag_list_1')] + mock_get_collection.return_value.drop_index.assert_has_calls(calls) diff --git a/pulp_puppet_plugins/test/unit/test_distributor.py b/pulp_puppet_plugins/test/unit/test_distributor.py index 9bf7aaa4..cbd96ac4 100644 --- a/pulp_puppet_plugins/test/unit/test_distributor.py +++ b/pulp_puppet_plugins/test/unit/test_distributor.py @@ -1,14 +1,3 @@ -# Copyright (C) 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest from pulp_puppet.plugins.distributors import distributor diff --git a/pulp_puppet_plugins/test/unit/test_distributor_configuration.py b/pulp_puppet_plugins/test/unit/test_distributor_configuration.py index c9cc2e0e..145fb58e 100644 --- a/pulp_puppet_plugins/test/unit/test_distributor_configuration.py +++ b/pulp_puppet_plugins/test/unit/test_distributor_configuration.py @@ -1,14 +1,3 @@ -# Copyright (c) 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import mock import unittest diff --git a/pulp_puppet_plugins/test/unit/test_distributor_publish.py b/pulp_puppet_plugins/test/unit/test_distributor_publish.py deleted file mode 100644 index f0c857f9..00000000 --- a/pulp_puppet_plugins/test/unit/test_distributor_publish.py +++ /dev/null @@ -1,327 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2012 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -import gdbm -import hashlib -import json -import os -import shutil -import tempfile -import unittest - -import mock -from pulp.plugins.config import PluginCallConfiguration -from pulp.plugins.model import Repository, PublishReport, Unit - -from pulp_puppet.common import constants -from pulp_puppet.plugins.distributors import publish - -# -- constants ---------------------------------------------------------------- - -DATA_DIR = os.path.abspath(os.path.dirname(__file__)) + '/../data' -FAKE_PULP_STORAGE_DIR = os.path.join(DATA_DIR, 'repos', 'valid') - -# -- test cases --------------------------------------------------------------- - -class MockConduit(mock.MagicMock): - - def build_success_report(self, summary, details): - return PublishReport(True, summary, details) - - def build_failure_report(self, summary, details): - return PublishReport(False, summary, details) - - -class PublishRunTests(unittest.TestCase): - - def setUp(self): - # Set up fake published location to write repositories to - self.test_httpd_base = tempfile.mkdtemp(prefix='pulp-puppet-dist-publish') - self.test_http_dir = os.path.join(self.test_httpd_base, 'http') - self.test_https_dir = os.path.join(self.test_httpd_base, 'https') - - os.mkdir(self.test_http_dir) - os.mkdir(self.test_https_dir) - - # Fake repository working directory - self.working_dir = tempfile.mkdtemp(prefix='pulp-puppet-repo-dir') - - # Have the conduit return units to copy (real units from the data dir) - self.units = [] - for module in [m for m in os.listdir(FAKE_PULP_STORAGE_DIR) if m.endswith('.tar.gz')]: - author, name, version = module.split('-') - version = version[0:5] - key = {'name' : name, 'version' : version, 'author' : author} - storage_dir = os.path.join(FAKE_PULP_STORAGE_DIR, module) - - metadata = {'checksums' : [['a', 'a'], ['b', 'b']]} - u = Unit(constants.TYPE_PUPPET_MODULE, key, metadata, storage_dir) - self.units.append(u) - self.conduit = MockConduit() - self.conduit.get_units.return_value = self.units - - # Configuration for the run - self.repo = Repository('test-repo', working_dir=self.working_dir) - self.config = PluginCallConfiguration( - { - constants.CONFIG_HTTP_DIR : self.test_http_dir, - constants.CONFIG_HTTPS_DIR : self.test_https_dir, - }, - { - constants.CONFIG_SERVE_HTTP : True, - constants.CONFIG_SERVE_HTTPS : True, - } - ) - self.is_cancelled_call = mock.MagicMock().is_cancelled_call - - self.run = publish.PuppetModulePublishRun(self.repo, self.conduit, - self. config, self.is_cancelled_call) - - def tearDown(self): - if os.path.exists(self.test_httpd_base): - shutil.rmtree(self.test_httpd_base) - - if os.path.exists(self.working_dir): - shutil.rmtree(self.working_dir) - - @mock.patch('gdbm.open') - def test_generate_dep_data(self, mock_open): - class FakeDB(dict): - """Fake version of gdbm database""" - close_called = False - def close(self): - self.close_called = True - mock_open.return_value = FakeDB() - file_to_test = os.path.join(DATA_DIR, 'simple', 'xinetd-1.2.0.tar.gz') - with open(file_to_test, 'r') as file_handle: - content = file_handle.read() - md5_sum = hashlib.md5(content).hexdigest() - - units = [ - Unit(constants.TYPE_PUPPET_MODULE, - {'name': 'foo', 'version': '1.0.3', 'author': 'me'}, - {'dependencies': []}, file_to_test), - Unit(constants.TYPE_PUPPET_MODULE, - {'name': 'foo', 'version': '1.1.0', 'author': 'me'}, - {'dependencies': []}, file_to_test), - Unit(constants.TYPE_PUPPET_MODULE, - {'name': 'bar', 'version': '1.0.0', 'author': 'me'}, - {'dependencies': [{'name': 'me/foo', 'version_requirement': '>= 1.0.0'}]}, - file_to_test), - ] - self.run._generate_dependency_data(units) - - db = mock_open.return_value - self.assertTrue(db.close_called) - self.assertTrue('me/foo' in db) - self.assertTrue('me/bar' in db) - foo_data = json.loads(db['me/foo']) - self.assertEqual(len(foo_data), 2) - bar_data = json.loads(db['me/bar']) - self.assertEqual(len(bar_data), 1) - self.assertEqual(bar_data[0]['dependencies'][0]['name'], 'me/foo') - self.assertEqual(bar_data[0]['dependencies'][0]['version_requirement'], '>= 1.0.0') - self.assertEqual(bar_data[0]['file_md5'], md5_sum) - - def test_perform_publish(self): - # Test - report = self.run.perform_publish() - - # Verify - - # Units copied to the proper published dirs - for unit in self.units: - filename = os.path.basename(unit.storage_path) - author = unit.unit_key['author'] - relative_path = constants.HOSTED_MODULE_FILE_RELATIVE_PATH % (author[0], author) - - http_published_filename = os.path.join(self.test_http_dir, self.repo.id, relative_path, filename) - self.assertTrue(os.path.exists(http_published_filename), msg='%s does not exist' % http_published_filename) - self.assertTrue(os.path.islink(http_published_filename), msg='%s is not a symlink' % http_published_filename) - - https_published_filename = os.path.join(self.test_https_dir, self.repo.id, relative_path, filename) - self.assertTrue(os.path.exists(https_published_filename), msg='%s does not exist' % https_published_filename) - self.assertTrue(os.path.islink(https_published_filename), msg='%s is not a symlink' % https_published_filename) - - # Build directory was cleaned up - self.assertTrue(not os.path.exists(self.run._build_dir())) - - # Dependency metadata was generated - expected_dep_file = os.path.join(self.test_http_dir, self.repo.id, constants.REPO_DEPDATA_FILENAME) - self.assertTrue(os.path.exists(expected_dep_file)) - db = gdbm.open(expected_dep_file) - for unit in self.units: - name = '%s/%s' % (unit.unit_key['author'], unit.unit_key['name']) - self.assertTrue(db.has_key(name)) - data = json.loads(db[name]) - self.assertTrue(isinstance(data, list)) - if len(data) == 1: - self.assertEqual(data[0]['version'], unit.unit_key['version']) - self.assertTrue(isinstance(data[0].get('dependencies'), list)) - self.assertTrue('file' in data[0]) - db.close() - - self.assertTrue(os.path.exists(self.test_http_dir)) - - # Final report - self.assertTrue(report.success_flag) - self.assertTrue(report.summary['total_execution_time'] is not None) - self.assertTrue(report.summary['total_execution_time'] > -1) - self.assertEqual(len(report.details), 0) - - # Progress report was updated - pr = self.run.progress_report - self.assertEqual(pr.modules_state, constants.STATE_SUCCESS) - self.assertEqual(pr.modules_total_count, 2) - self.assertEqual(pr.modules_finished_count, 2) - self.assertEqual(pr.modules_error_count, 0) - self.assertTrue(pr.modules_execution_time is not None) - self.assertEqual(pr.modules_error_message, None) - self.assertEqual(pr.modules_exception, None) - self.assertEqual(pr.modules_traceback, None) - self.assertEqual(pr.modules_individual_errors, None) - - self.assertEqual(pr.metadata_state, constants.STATE_SUCCESS) - self.assertTrue(pr.metadata_execution_time is not None) - self.assertEqual(pr.metadata_error_message, None) - self.assertEqual(pr.metadata_exception, None) - self.assertEqual(pr.metadata_traceback, None) - - self.assertEqual(pr.publish_http, constants.STATE_SUCCESS) - self.assertEqual(pr.publish_https, constants.STATE_SUCCESS) - - def test_unpublish_http(self): - """ - After a successful publish, run another without HTTP to make sure the - previously published repository is removed. - """ - - # Setup - self.run.perform_publish() - - # Test - self.config.override_config = {constants.CONFIG_SERVE_HTTP : False,} - self.run.perform_publish() - - # Verify - published_repo_dir = os.path.join(self.test_http_dir, self.repo.id) - self.assertTrue(not os.path.exists(published_repo_dir)) - - def test_publish_skip_http_https(self): - # Setup - self.config.override_config = { - constants.CONFIG_SERVE_HTTP : False, - constants.CONFIG_SERVE_HTTPS : False, - } - - # Test - self.run.perform_publish() - - # Verify - pr = self.run.progress_report - self.assertEqual(pr.publish_http, constants.STATE_SKIPPED) - self.assertEqual(pr.publish_https, constants.STATE_SKIPPED) - - def test_error_in_modules_step(self): - # Setup - self.repo.working_dir = '/foo' # init build dir will fail - - # Test - report = self.run.perform_publish() - - # Verify - self.assertTrue(not report.success_flag) - self.assertEqual(report.summary['total_execution_time'], -1) - self.assertEqual(len(report.details), 0) - - pr = self.run.progress_report - self.assertEqual(pr.modules_state, constants.STATE_FAILED) - self.assertEqual(pr.modules_total_count, None) - self.assertEqual(pr.modules_finished_count, None) - self.assertEqual(pr.modules_error_count, None) - self.assertTrue(pr.modules_execution_time is not None) - self.assertTrue(pr.modules_error_message is not None) - self.assertTrue(pr.modules_exception is not None) - self.assertTrue(pr.modules_traceback is not None) - self.assertEqual(pr.modules_individual_errors, None) - - self.assertEqual(pr.metadata_state, constants.STATE_NOT_STARTED) - - def test_error_in_metadata_step(self): - # Setup - self.config.override_config = {constants.CONFIG_HTTP_DIR : '/foo'} - - # Test - report = self.run.perform_publish() - - # Verify - self.assertTrue(not report.success_flag) - self.assertTrue(report.summary['total_execution_time'] > -1) - self.assertEqual(len(report.details), 0) - - pr = self.run.progress_report - self.assertEqual(pr.modules_state, constants.STATE_SUCCESS) - - self.assertEqual(pr.metadata_state, constants.STATE_FAILED) - self.assertTrue(pr.metadata_error_message is not None) - self.assertTrue(pr.metadata_exception is not None) - self.assertTrue(pr.metadata_traceback is not None) - self.assertTrue(pr.metadata_execution_time is not None) - - def test_existing_build_dir(self): - # Setup - build_dir = self.run._build_dir() - os.makedirs(build_dir) - - sample_file = os.path.join(build_dir, 'sample') - f = open(sample_file, 'w') - f.write('foo') - f.close() - - # Test - self.run._init_build_dir() - - # Verify - self.assertTrue(os.path.exists(build_dir)) - self.assertTrue(not os.path.exists(sample_file)) - - @mock.patch('os.symlink') - def test_failed_symlink(self, mock_symlink): - # Setup - mock_symlink.side_effect = Exception() # simulate write permission error - - # Test - report = self.run.perform_publish() - - # Verify - self.assertTrue(report.success_flag) # still an overall success - - pr = self.run.progress_report - self.assertEqual(pr.modules_state, constants.STATE_SUCCESS) # still a success - self.assertEqual(pr.modules_error_count, 2) - self.assertEqual(pr.modules_finished_count, 0) - self.assertEqual(len(pr.modules_individual_errors), 2) - - self.assertEqual(pr.metadata_state, constants.STATE_SUCCESS) - - def test_unpublish_repo(self): - # Setup - os.makedirs(os.path.join(self.test_http_dir, self.repo.id)) - os.makedirs(os.path.join(self.test_https_dir, self.repo.id)) - - # Test - publish.unpublish_repo(self.repo, self.config) - - # Verify - self.assertTrue(not os.path.exists(os.path.join(self.test_http_dir, self.repo.id))) - self.assertTrue(not os.path.exists(os.path.join(self.test_https_dir, self.repo.id))) diff --git a/pulp_puppet_plugins/test/unit/test_install_distributor.py b/pulp_puppet_plugins/test/unit/test_install_distributor.py index 7f1a59e9..301c46c2 100644 --- a/pulp_puppet_plugins/test/unit/test_install_distributor.py +++ b/pulp_puppet_plugins/test/unit/test_install_distributor.py @@ -73,54 +73,6 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.working_directory) - - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_ensure_destination_dir', - return_value=None) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_move_to_destination_directory', - return_value=None) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_rename_directory', - return_value=None) - @mock.patch('tarfile.open', autospec=True) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_clear_destination_directory', - return_value=None) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_create_temporary_destination_directory', - return_value=None) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_check_for_unsafe_archive_paths', - return_value=None) - def test_workflow(self, mock_check_paths, mock_mkdir, mock_clear, mock_open, mock_rename, - mock_move, mock_ensure_destination): - config = PluginCallConfiguration({}, {constants.CONFIG_INSTALL_PATH: self.puppet_dir}) - mock_open.return_value.getnames.return_value = ['a/b', 'a/c'] - - report = self.distributor.publish_repo(self.repo, self.conduit, config) - - self.assertTrue(isinstance(report, PublishReport)) - self.assertTrue(report.success_flag) - self.assertEqual(len(report.details['errors']), 0) - self.assertEqual(len(report.details['success_unit_keys']), 2) - self.assertTrue(self.uk1 in report.details['success_unit_keys']) - self.assertTrue(self.uk2 in report.details['success_unit_keys']) - - self.assertEqual(mock_open.call_count, 2) - mock_open.assert_any_call(self.units[0].storage_path) - mock_open.assert_any_call(self.units[1].storage_path) - - self.assertEqual(mock_rename.call_count, 2) - - mock_mkdir.assert_called_once_with(self.puppet_dir) - mock_clear.assert_called_once_with(self.puppet_dir) - mock_ensure_destination.assert_called_once_with(self.puppet_dir) - mock_check_paths.assert_called_once_with(self.units, self.puppet_dir) - - self.assertEqual(mock_move.call_count, 1) - - def test_no_destination(self): """this one should fail very early since the destination is missing""" config = PluginCallConfiguration({}, {}) @@ -132,181 +84,6 @@ def test_no_destination(self): self.assertEqual(len(report.details['errors']), 0) self.assertEqual(len(report.details['success_unit_keys']), 0) - def test_duplicate_unit_names(self): - config = PluginCallConfiguration({}, {constants.CONFIG_INSTALL_PATH: self.puppet_dir}) - uk3 = {'author': 'puppetlabs', 'name': 'stdlib', 'version': '1.3.1'} - unit3 = AssociatedUnit(constants.TYPE_PUPPET_MODULE, uk3, {}, '/a/b/z', '', '') - self.units.append(unit3) - - report = self.distributor.publish_repo(self.repo, self.conduit, config) - - self.assertFalse(report.success_flag) - self.assertTrue(isinstance(report.summary, basestring)) - self.assertEqual(len(report.details['errors']), 2) - self.assertTrue(report.summary.find('duplicate') >= 0) - - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_check_for_unsafe_archive_paths', - return_value=None) - def test_unsafe_paths(self, mock_check): - config = PluginCallConfiguration({}, {constants.CONFIG_INSTALL_PATH: self.puppet_dir}) - mock_check.side_effect = self._add_error - - report = self.distributor.publish_repo(self.repo, self.conduit, config) - - self.assertFalse(report.success_flag) - self.assertTrue(isinstance(report.summary, basestring)) - self.assertTrue(len(report.summary) > 0) - self.assertEqual(len(report.details['errors']), 1) - self.assertEqual(len(report.details['success_unit_keys']), 0) - - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_check_for_unsafe_archive_paths', - return_value=None) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_clear_destination_directory', - side_effect=OSError) - def test_cannot_remove_destination(self, mock_clear, mock_check): - config = PluginCallConfiguration({}, {constants.CONFIG_INSTALL_PATH: self.puppet_dir}) - - report = self.distributor.publish_repo(self.repo, self.conduit, config) - - self.assertFalse(report.success_flag) - self.assertTrue(isinstance(report.summary, basestring)) - self.assertEqual(len(report.details['errors']), 2) - self.assertEqual(len(report.details['success_unit_keys']), 0) - - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_check_for_unsafe_archive_paths', - return_value=None) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_clear_destination_directory', - return_value=None) - def test_cannot_open_tarballs(self, mock_clear, mock_check): - """ - This is easy to simulate, because we can let the real tarfile module try - to open the fake paths. - """ - config = PluginCallConfiguration({}, {constants.CONFIG_INSTALL_PATH: self.puppet_dir}) - - report = self.distributor.publish_repo(self.repo, self.conduit, config) - - self.assertFalse(report.success_flag) - self.assertTrue(isinstance(report.summary, basestring)) - self.assertEqual(len(report.details['errors']), 2) - self.assertTrue(report.details['errors'][0][0] in [self.uk1, self.uk2]) - self.assertTrue(report.details['errors'][1][0] in [self.uk1, self.uk2]) - self.assertEqual(len(report.details['success_unit_keys']), 0) - - @mock.patch('tarfile.open', autospec=True) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_check_for_unsafe_archive_paths', - return_value=None) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_clear_destination_directory', - return_value=None) - def test_cannot_extract_tarballs(self, mock_clear, mock_check, mock_open): - config = PluginCallConfiguration({}, {constants.CONFIG_INSTALL_PATH: self.puppet_dir}) - mock_open.return_value.extractall.side_effect = OSError - - report = self.distributor.publish_repo(self.repo, self.conduit, config) - - self.assertFalse(report.success_flag) - self.assertTrue(isinstance(report.summary, basestring)) - self.assertEqual(len(report.details['errors']), 2) - self.assertTrue(report.details['errors'][0][0] in [self.uk1, self.uk2]) - self.assertTrue(report.details['errors'][1][0] in [self.uk1, self.uk2]) - self.assertEqual(len(report.details['success_unit_keys']), 0) - - @mock.patch('shutil.move', side_effect=IOError) - @mock.patch('tarfile.open', autospec=True) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_check_for_unsafe_archive_paths', - return_value=None) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_clear_destination_directory', - return_value=None) - def test_cannot_move(self, mock_clear, mock_check, mock_open, mock_move): - config = PluginCallConfiguration({}, {constants.CONFIG_INSTALL_PATH: self.puppet_dir}) - mock_open.return_value.getnames.return_value = ['a/b', 'a/c'] - - report = self.distributor.publish_repo(self.repo, self.conduit, config) - - self.assertFalse(report.success_flag) - self.assertTrue(isinstance(report.summary, basestring)) - self.assertEqual(len(report.details['errors']), 2) - self.assertTrue(report.details['errors'][0][0] in [self.uk1, self.uk2]) - self.assertTrue(report.details['errors'][1][0] in [self.uk1, self.uk2]) - self.assertEqual(len(report.details['success_unit_keys']), 0) - - @mock.patch('tarfile.open', autospec=True) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_check_for_unsafe_archive_paths', - return_value=None) - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_clear_destination_directory', - return_value=None) - def test_multiple_extraction_dirs(self, mock_clear, mock_check, mock_open): - config = PluginCallConfiguration({}, {constants.CONFIG_INSTALL_PATH: self.puppet_dir}) - mock_open.return_value.getnames.return_value = ['a/b', 'c/b'] - - report = self.distributor.publish_repo(self.repo, self.conduit, config) - - self.assertFalse(report.success_flag) - self.assertTrue(isinstance(report.summary, basestring)) - self.assertEqual(len(report.details['errors']), 2) - self.assertTrue(report.details['errors'][0][0] in [self.uk1, self.uk2]) - self.assertTrue(report.details['errors'][1][0] in [self.uk1, self.uk2]) - self.assertEqual(len(report.details['success_unit_keys']), 0) - - @mock.patch.object(installdistributor.PuppetModuleInstallDistributor, - '_clear_destination_directory', - return_value=None) - def test_no_units(self, mock_clear): - config = PluginCallConfiguration({}, {constants.CONFIG_INSTALL_PATH: self.puppet_dir}) - self.conduit.get_units.return_value = [] - - report = self.distributor.publish_repo(self.repo, self.conduit, config) - - self.assertTrue(report.success_flag) - self.assertEqual(len(report.details['errors']), 0) - self.assertEqual(len(report.details['success_unit_keys']), 0) - - # we still need to clear the destination - mock_clear.assert_called_once_with(self.puppet_dir) - - def _add_error(self, *args, **kwargs): - """ - add an error to the detail report. This gives us a chance to add an error - during a particular step in the workflow. - """ - if not self.distributor.detail_report.report['errors']: - self.distributor.detail_report.error(self.uk1, 'failed') - - -class TestFindDuplicateNames(unittest.TestCase): - def setUp(self): - self.uk1 = {'author': 'puppetlabs', 'name': 'stdlib', 'version': '1.2.0'} - self.uk2 = {'author': 'puppetlabs', 'name': 'java', 'version': '1.3.1'} - self.uk3 = {'author': 'puppetlabs', 'name': 'stdlib', 'version': '1.3.1'} - self.unit3 = AssociatedUnit(constants.TYPE_PUPPET_MODULE, self.uk3, {}, '/a/b/z', '', '') - self.units = [ - AssociatedUnit(constants.TYPE_PUPPET_MODULE, self.uk1, {}, '/a/b/x', '', ''), - AssociatedUnit(constants.TYPE_PUPPET_MODULE, self.uk2, {}, '/a/b/y', '', ''), - ] - self.method = installdistributor.PuppetModuleInstallDistributor._find_duplicate_names - - def test_no_dups(self): - ret = self.method(self.units) - - self.assertEqual(ret, []) - - def test_with_dups(self): - self.units.append(self.unit3) - ret = self.method(self.units) - self.assertTrue(self.units[0] in ret) - self.assertTrue(self.units[2] in ret) - class TestMoveToDestinationDirectory(unittest.TestCase): def setUp(self): @@ -345,21 +122,21 @@ def test_move_dirs(self): class TestRenameDirectory(unittest.TestCase): def setUp(self): - uk = {'author': 'puppetlabs', 'name': 'stdlib', 'version': '1.2.0'} - self.unit = AssociatedUnit(constants.TYPE_PUPPET_MODULE, uk, {}, '/a/b/x', '', '') + self.unit = mock.Mock() + self.unit.name = 'foobar' self.method = installdistributor.PuppetModuleInstallDistributor._rename_directory @mock.patch('shutil.move', autospec=True) def test_trailing_slash(self, mock_move): self.method(self.unit, '/tmp/', ['a/b', 'a/c']) - mock_move.assert_called_once_with('/tmp/a', '/tmp/stdlib') + mock_move.assert_called_once_with('/tmp/a', '/tmp/foobar') @mock.patch('shutil.move', autospec=True) def test_no_trailing_slash(self, mock_move): self.method(self.unit, '/tmp', ['a/b', 'a/c']) - mock_move.assert_called_once_with('/tmp/a', '/tmp/stdlib') + mock_move.assert_called_once_with('/tmp/a', '/tmp/foobar') @mock.patch('shutil.move', autospec=True) def test_too_many_dirs(self, mock_move): @@ -373,18 +150,18 @@ def test_no_dirs(self, mock_move): def test_absolute_paths(self, mock_move): self.method(self.unit, '/tmp', ['/tmp/a/b', '/tmp/a/c']) - mock_move.assert_called_once_with('/tmp/a', '/tmp/stdlib') + mock_move.assert_called_once_with('/tmp/a', '/tmp/foobar') @mock.patch('shutil.move', autospec=True) def test_empty_dir(self, mock_move): """weird scenario, but you never know...""" self.method(self.unit, '/tmp', ['a']) - mock_move.assert_called_once_with('/tmp/a', '/tmp/stdlib') + mock_move.assert_called_once_with('/tmp/a', '/tmp/foobar') @mock.patch('shutil.move', autospec=True) def test_same_dir(self, mock_move): - self.method(self.unit, '/tmp', ['stdlib']) + self.method(self.unit, '/tmp', ['foobar']) self.assertFalse(mock_move.called) @@ -563,18 +340,6 @@ def test_report_is_dict(self): self.assertTrue(isinstance(self.report.report, dict)) -class TestEnsureDestinationDir(unittest.TestCase): - def setUp(self): - self.distributor = installdistributor.PuppetModuleInstallDistributor() - - @mock.patch('pulp_puppet.plugins.distributors.installdistributor.mkdir') - def test_calls_mkdir(self, fake_mkdir): - # Ensure we are calling the platform util method properly - path = 'path-123' - self.distributor._ensure_destination_dir(path) - fake_mkdir.assert_called_once_with(path) - - class TestException(Exception): pass diff --git a/pulp_puppet_plugins/test/unit/test_profiler_whole_repo.py b/pulp_puppet_plugins/test/unit/test_profiler_whole_repo.py index bb68dbec..9c6a74e2 100644 --- a/pulp_puppet_plugins/test/unit/test_profiler_whole_repo.py +++ b/pulp_puppet_plugins/test/unit/test_profiler_whole_repo.py @@ -1,16 +1,3 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import unittest import mock @@ -64,23 +51,6 @@ def test_option_not_present(self): self.assertEqual(self.conduit.get_units.call_count, 0) self.assertTrue(constants.FORGE_HOST in options) - def test_with_option(self): - special_unit = {'type_id': constants.TYPE_PUPPET_MODULE, 'unit_key': None} - options = {constants.REPO_ID_OPTION: 'repo1', constants.WHOLE_REPO_OPTION: True} - - result = self.profiler.install_units( - self.consumer, [special_unit], options, {}, self.conduit) - - self.conduit.get_units.assert_called_once_with('repo1') - # make sure it combined the two stdlib entries - self.assertEqual(len(result), 2) - for unit in result: - self.assertTrue('version' not in unit['unit_key']) - self.assertEqual(unit['unit_key']['author'], 'puppetlabs') - self.assertTrue('name' in unit['unit_key']) - self.assertEqual(unit['type_id'], constants.TYPE_PUPPET_MODULE) - self.assertTrue(constants.FORGE_HOST in options) - def test_with_repo_but_not_option(self): options = {constants.REPO_ID_OPTION: 'repo1'} result = self.profiler.install_units(self.consumer, self.units, options, {}, self.conduit) diff --git a/pulp_puppet_tools/pulp_puppet/tools/puppet_module_builder.py b/pulp_puppet_tools/pulp_puppet/tools/puppet_module_builder.py index 66d4824b..4fbe8f3c 100644 --- a/pulp_puppet_tools/pulp_puppet/tools/puppet_module_builder.py +++ b/pulp_puppet_tools/pulp_puppet/tools/puppet_module_builder.py @@ -1,14 +1,3 @@ -# Copyright (c) 2014 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os import sys diff --git a/pulp_puppet_tools/test/unit/test_puppet_module_builder.py b/pulp_puppet_tools/test/unit/test_puppet_module_builder.py index 5aff9609..c71970a4 100644 --- a/pulp_puppet_tools/test/unit/test_puppet_module_builder.py +++ b/pulp_puppet_tools/test/unit/test_puppet_module_builder.py @@ -1,14 +1,3 @@ -# Copyright (c) 2014 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os from unittest import TestCase diff --git a/rel-eng/lib/pulptagger.py b/rel-eng/lib/pulptagger.py index 3d01dc0c..76ba66e3 100644 --- a/rel-eng/lib/pulptagger.py +++ b/rel-eng/lib/pulptagger.py @@ -1,14 +1,3 @@ -# Copyright (c) 2011 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - import os import re @@ -17,6 +6,7 @@ from tito.tagger import VersionTagger from tito.common import error_out + # changelog BUGZILLA_REGEX = re.compile('([0-9]+\s+\-\s+)(.+)') FEATURE_REGEX = re.compile('([\-]\s+)(.+)')