From e0bce9e8a815ccacef179d9cc7ce96754c005d69 Mon Sep 17 00:00:00 2001 From: Filip Dobrovolny Date: Fri, 16 Sep 2016 20:55:53 +0200 Subject: [PATCH] Remove pulp_node * Remove all pulp_node code * Remove pulp_node from docs closes #2193 https://pulp.plan.io/issues/2193 --- comps.xml | 25 - docs/dev-guide/integration/index.rst | 1 - docs/dev-guide/integration/nodes.rst | 179 ---- .../integration/rest-api/server_plugins.rst | 22 +- docs/user-guide/admin-client/index.rst | 1 - docs/user-guide/admin-client/nodes.rst | 240 ----- docs/user-guide/consumer-client/index.rst | 1 - docs/user-guide/consumer-client/nodes.rst | 91 -- docs/user-guide/glossary.rst | 11 +- docs/user-guide/index.rst | 1 - .../installation/extra_configuration.rst | 6 +- docs/user-guide/nodes.rst | 497 --------- docs/user-guide/release-notes/2.1.x.rst | 2 +- docs/user-guide/release-notes/2.2.x.rst | 3 +- docs/user-guide/release-notes/2.3.x.rst | 2 +- docs/user-guide/release-notes/3.0.x.rst | 14 + docs/user-guide/release-notes/index.rst | 1 + docs/user-guide/scaling.rst | 4 +- docs/user-guide/troubleshooting.rst | 4 - nodes/__init__.py | 10 - nodes/child/etc/pulp/agent/conf.d/nodes.conf | 11 - .../plugins.conf.d/nodes/importer/http.conf | 2 - nodes/child/pulp_node/__init__.py | 3 - nodes/child/pulp_node/handlers/__init__.py | 10 - nodes/child/pulp_node/handlers/handler.py | 163 --- nodes/child/pulp_node/handlers/model.py | 532 ---------- nodes/child/pulp_node/handlers/reports.py | 148 --- nodes/child/pulp_node/handlers/strategies.py | 269 ----- nodes/child/pulp_node/handlers/validation.py | 95 -- nodes/child/pulp_node/importers/__init__.py | 10 - nodes/child/pulp_node/importers/download.py | 108 -- .../pulp_node/importers/http/__init__.py | 0 .../pulp_node/importers/http/importer.py | 164 --- nodes/child/pulp_node/importers/inventory.py | 101 -- nodes/child/pulp_node/importers/reports.py | 61 -- nodes/child/pulp_node/importers/strategies.py | 436 -------- .../pulp_node/importers/types/nodes.json | 16 - nodes/child/setup.py | 15 - nodes/common/__init__.py | 10 - nodes/common/bin/pulp-gen-nodes-certificate | 81 -- nodes/common/etc/pulp/nodes.conf | 48 - nodes/common/pulp_node/__init__.py | 3 - nodes/common/pulp_node/conduit.py | 120 --- nodes/common/pulp_node/config.py | 49 - nodes/common/pulp_node/constants.py | 81 -- nodes/common/pulp_node/error.py | 222 ---- nodes/common/pulp_node/extension.py | 73 -- nodes/common/pulp_node/manifest.py | 449 --------- nodes/common/pulp_node/pathlib.py | 45 - nodes/common/pulp_node/poller.py | 103 -- nodes/common/pulp_node/reports.py | 167 --- nodes/common/pulp_node/resources.py | 59 -- nodes/common/setup.py | 10 - nodes/extensions/admin/pulp_node/__init__.py | 3 - .../admin/pulp_node/extensions/__init__.py | 3 - .../pulp_node/extensions/admin/__init__.py | 10 - .../pulp_node/extensions/admin/commands.py | 510 ---------- .../pulp_node/extensions/admin/options.py | 41 - .../pulp_node/extensions/admin/rendering.py | 262 ----- .../extensions/admin/sync_schedules.py | 96 -- nodes/extensions/admin/setup.py | 15 - .../extensions/consumer/pulp_node/__init__.py | 3 - .../consumer/pulp_node/extensions/__init__.py | 3 - .../pulp_node/extensions/consumer/__init__.py | 10 - .../pulp_node/extensions/consumer/commands.py | 231 ----- nodes/extensions/consumer/setup.py | 15 - nodes/parent/etc/httpd/conf.d/pulp_nodes.conf | 46 - .../nodes/distributor/http.conf | 15 - nodes/parent/pulp_node/__init__.py | 3 - .../parent/pulp_node/distributors/__init__.py | 10 - .../pulp_node/distributors/http/__init__.py | 0 .../distributors/http/distributor.py | 259 ----- .../pulp_node/distributors/http/publisher.py | 66 -- .../pulp_node/distributors/publisher.py | 178 ---- nodes/parent/pulp_node/profilers/__init__.py | 10 - nodes/parent/pulp_node/profilers/nodes.py | 107 -- nodes/parent/setup.py | 18 - nodes/test/nodes_tests/__init__.py | 18 - nodes/test/nodes_tests/base.py | 104 -- nodes/test/nodes_tests/data/client.conf | 2 - nodes/test/nodes_tests/data/distribution.tar | Bin 10240 -> 0 bytes nodes/test/nodes_tests/data/pulp.conf | 32 - nodes/test/nodes_tests/mocks/mock_plugins.py | 249 ----- nodes/test/nodes_tests/nodes/__init__.py | 0 .../nodes_tests/nodes/handlers/__init__.py | 0 .../nodes/handlers/test_strategies.py | 62 -- .../nodes_tests/nodes/importers/__init__.py | 0 .../nodes/importers/test_reports.py | 53 - .../nodes/importers/test_strategies.py | 25 - .../test/nodes_tests/test_admin_extensions.py | 622 ------------ nodes/test/nodes_tests/test_conduit.py | 119 --- nodes/test/nodes_tests/test_config.py | 82 -- .../nodes_tests/test_consumer_extensions.py | 187 ---- nodes/test/nodes_tests/test_errors.py | 227 ----- .../test_extensions_sync_schedules.py | 126 --- .../nodes_tests/test_handler_strategies.py | 216 ---- .../nodes_tests/test_importer_strategies.py | 263 ----- nodes/test/nodes_tests/test_manifest.py | 122 --- nodes/test/nodes_tests/test_model.py | 64 -- nodes/test/nodes_tests/test_plugins.py | 949 ------------------ nodes/test/nodes_tests/test_publishers.py | 138 --- nodes/test/nodes_tests/test_resources.py | 83 -- pulp-dev.py | 23 - rel-eng/packages/pulp-nodes | 1 - run-tests.py | 2 - 105 files changed, 27 insertions(+), 10435 deletions(-) delete mode 100644 docs/dev-guide/integration/nodes.rst delete mode 100644 docs/user-guide/admin-client/nodes.rst delete mode 100644 docs/user-guide/consumer-client/nodes.rst delete mode 100644 docs/user-guide/nodes.rst create mode 100644 docs/user-guide/release-notes/3.0.x.rst delete mode 100644 nodes/__init__.py delete mode 100644 nodes/child/etc/pulp/agent/conf.d/nodes.conf delete mode 100644 nodes/child/etc/pulp/server/plugins.conf.d/nodes/importer/http.conf delete mode 100644 nodes/child/pulp_node/__init__.py delete mode 100644 nodes/child/pulp_node/handlers/__init__.py delete mode 100644 nodes/child/pulp_node/handlers/handler.py delete mode 100644 nodes/child/pulp_node/handlers/model.py delete mode 100644 nodes/child/pulp_node/handlers/reports.py delete mode 100644 nodes/child/pulp_node/handlers/strategies.py delete mode 100644 nodes/child/pulp_node/handlers/validation.py delete mode 100644 nodes/child/pulp_node/importers/__init__.py delete mode 100644 nodes/child/pulp_node/importers/download.py delete mode 100644 nodes/child/pulp_node/importers/http/__init__.py delete mode 100644 nodes/child/pulp_node/importers/http/importer.py delete mode 100644 nodes/child/pulp_node/importers/inventory.py delete mode 100644 nodes/child/pulp_node/importers/reports.py delete mode 100644 nodes/child/pulp_node/importers/strategies.py delete mode 100644 nodes/child/pulp_node/importers/types/nodes.json delete mode 100755 nodes/child/setup.py delete mode 100644 nodes/common/__init__.py delete mode 100755 nodes/common/bin/pulp-gen-nodes-certificate delete mode 100644 nodes/common/etc/pulp/nodes.conf delete mode 100644 nodes/common/pulp_node/__init__.py delete mode 100644 nodes/common/pulp_node/conduit.py delete mode 100644 nodes/common/pulp_node/config.py delete mode 100644 nodes/common/pulp_node/constants.py delete mode 100644 nodes/common/pulp_node/error.py delete mode 100644 nodes/common/pulp_node/extension.py delete mode 100644 nodes/common/pulp_node/manifest.py delete mode 100644 nodes/common/pulp_node/pathlib.py delete mode 100644 nodes/common/pulp_node/poller.py delete mode 100644 nodes/common/pulp_node/reports.py delete mode 100644 nodes/common/pulp_node/resources.py delete mode 100755 nodes/common/setup.py delete mode 100644 nodes/extensions/admin/pulp_node/__init__.py delete mode 100644 nodes/extensions/admin/pulp_node/extensions/__init__.py delete mode 100644 nodes/extensions/admin/pulp_node/extensions/admin/__init__.py delete mode 100644 nodes/extensions/admin/pulp_node/extensions/admin/commands.py delete mode 100644 nodes/extensions/admin/pulp_node/extensions/admin/options.py delete mode 100644 nodes/extensions/admin/pulp_node/extensions/admin/rendering.py delete mode 100644 nodes/extensions/admin/pulp_node/extensions/admin/sync_schedules.py delete mode 100644 nodes/extensions/admin/setup.py delete mode 100644 nodes/extensions/consumer/pulp_node/__init__.py delete mode 100644 nodes/extensions/consumer/pulp_node/extensions/__init__.py delete mode 100644 nodes/extensions/consumer/pulp_node/extensions/consumer/__init__.py delete mode 100644 nodes/extensions/consumer/pulp_node/extensions/consumer/commands.py delete mode 100644 nodes/extensions/consumer/setup.py delete mode 100644 nodes/parent/etc/httpd/conf.d/pulp_nodes.conf delete mode 100644 nodes/parent/etc/pulp/server/plugins.conf.d/nodes/distributor/http.conf delete mode 100644 nodes/parent/pulp_node/__init__.py delete mode 100644 nodes/parent/pulp_node/distributors/__init__.py delete mode 100644 nodes/parent/pulp_node/distributors/http/__init__.py delete mode 100644 nodes/parent/pulp_node/distributors/http/distributor.py delete mode 100644 nodes/parent/pulp_node/distributors/http/publisher.py delete mode 100644 nodes/parent/pulp_node/distributors/publisher.py delete mode 100644 nodes/parent/pulp_node/profilers/__init__.py delete mode 100644 nodes/parent/pulp_node/profilers/nodes.py delete mode 100755 nodes/parent/setup.py delete mode 100644 nodes/test/nodes_tests/__init__.py delete mode 100644 nodes/test/nodes_tests/base.py delete mode 100644 nodes/test/nodes_tests/data/client.conf delete mode 100644 nodes/test/nodes_tests/data/distribution.tar delete mode 100644 nodes/test/nodes_tests/data/pulp.conf delete mode 100644 nodes/test/nodes_tests/mocks/mock_plugins.py delete mode 100644 nodes/test/nodes_tests/nodes/__init__.py delete mode 100644 nodes/test/nodes_tests/nodes/handlers/__init__.py delete mode 100644 nodes/test/nodes_tests/nodes/handlers/test_strategies.py delete mode 100644 nodes/test/nodes_tests/nodes/importers/__init__.py delete mode 100644 nodes/test/nodes_tests/nodes/importers/test_reports.py delete mode 100644 nodes/test/nodes_tests/nodes/importers/test_strategies.py delete mode 100644 nodes/test/nodes_tests/test_admin_extensions.py delete mode 100644 nodes/test/nodes_tests/test_conduit.py delete mode 100644 nodes/test/nodes_tests/test_config.py delete mode 100644 nodes/test/nodes_tests/test_consumer_extensions.py delete mode 100644 nodes/test/nodes_tests/test_errors.py delete mode 100644 nodes/test/nodes_tests/test_extensions_sync_schedules.py delete mode 100644 nodes/test/nodes_tests/test_handler_strategies.py delete mode 100644 nodes/test/nodes_tests/test_importer_strategies.py delete mode 100644 nodes/test/nodes_tests/test_manifest.py delete mode 100644 nodes/test/nodes_tests/test_model.py delete mode 100644 nodes/test/nodes_tests/test_plugins.py delete mode 100644 nodes/test/nodes_tests/test_publishers.py delete mode 100644 nodes/test/nodes_tests/test_resources.py delete mode 100644 rel-eng/packages/pulp-nodes diff --git a/comps.xml b/comps.xml index 5f479ab2a0..19900124b9 100644 --- a/comps.xml +++ b/comps.xml @@ -81,29 +81,4 @@ python-gofer-qpid - - pulp-nodes-parent - Pulp Nodes Parent - - Pulp nodes parent packages. - - true - - pulp-nodes-parent - pulp-nodes-admin-extensions - - - - pulp-nodes-child - Pulp Nodes Child - - Pulp nodes child packages. - - true - - pulp-nodes-child - pulp-nodes-consumer-extensions - pulp-agent - - diff --git a/docs/dev-guide/integration/index.rst b/docs/dev-guide/integration/index.rst index 8700b20639..f8c34c3e6c 100644 --- a/docs/dev-guide/integration/index.rst +++ b/docs/dev-guide/integration/index.rst @@ -6,4 +6,3 @@ Integrating with Pulp rest-api/index events/index - nodes diff --git a/docs/dev-guide/integration/nodes.rst b/docs/dev-guide/integration/nodes.rst deleted file mode 100644 index 0f327b2c3e..0000000000 --- a/docs/dev-guide/integration/nodes.rst +++ /dev/null @@ -1,179 +0,0 @@ -Pulp Nodes -========== - -Pulp *Nodes* management is performed using the platform REST API. This document identifies the -specific APIs and defines the data values needed for each call. For more information on *Nodes* -concepts, see the Pulp User Guide. - - -Activation ----------- - -Activation is stored as a note on the consumer. - -Activate -^^^^^^^^ - -To activate a consumer as a child node, add a special note to the consumer using the -:ref:`consumer update API`. - -Notes: - - _child-node - The value ``true`` indicates the consumer is a child node. - - _node-update-strategy - The value specifies the *node-level* synchronization strategy. - -Sample POST body: - -:: - - { - "delta": { - "notes": { - "_child-node": true, - "_node-update-strategy": "additive" - - } - } - } - -Deactivate -^^^^^^^^^^ - -To deactivate a child node, remove the special notes from the consumer using the -:ref:`consumer update API`. - -Sample POST body: - -:: - - { - "delta": { - "notes": { - "_child-node": null, - "_node-update-strategy": null - - } - } - } - - -Repositories ------------- - -Enabling -^^^^^^^^ - -Repositories are enabled for use with child nodes by associating the *Nodes* distributor with -the repository using the :ref:`distributor association API`. -The ``distributor_type_id`` is ``nodes_http_distributor``. - -Sample POST body: - -:: - - { - "distributor_id": "nodes_http_distributor", - "distributor_type_id": "nodes_http_distributor", - "distributor_config": {}, - "auto_publish": true - } - -Disabling -^^^^^^^^^ - -Repositories are disabled for use with child nodes by disassociating the *Nodes* distributor and -the repository using the :ref:`distributor disassociation API`. -The ``distributor_id`` in the URL is ``nodes_http_distributor``. - -Publishing -^^^^^^^^^^ - -Manually publishing the *Nodes* data necessary for child node synchronization can be done using -the :ref:`repository publish API`. - -Sample POST body: - -:: - - {"override_config": {}, "id": "nodes_http_distributor"} - -Binding -------- - -Bind -^^^^ - -Binding a child node to a repository can be done using the :ref:`bind API`. In the POST body, -the ``notify_agent`` must be set to ``false`` because node bindings do not require agent -participation. The ``binding_config`` can be used to specify the *repository-level* -synchronization strategy. The default is ``additive`` if not specified. - -Sample POST body: - -:: - - { - "notify_agent": false, - "binding_config": {"strategy": "additive"}, - "repo_id": "elmer", - "distributor_id": "nodes_http_distributor" - } - -Unbind -^^^^^^ - -Unbinding a child node from a repository can be done using the :ref:`unbind API`. -The ``distributor_id`` in the URL is ``nodes_http_distributor``. - - -Synchronization ---------------- - -The synchronization of a child node is seen by the parent server as a content update on a consumer. -In this case, the consumer is a child node. - -Run -^^^ - -An immediate synchronization of a child node can be initiated using the -:ref:`content update API`. In the POST body, an array of (1) unit with ``type_id`` of -``node`` and ``unit_key`` of ``null`` is specified. - -Sample POST body: - -:: - - { - "units": [{"type_id": "node", "unit_key": null}], - "options": {} - } - - -To skip the repository synchronization phase of the update, specify the ``skip_content_update`` option -with a value of ``true``. - -Sample POST body: - -:: - - { - "units": [{"type_id": "node", "unit_key": null}], - "options": {"skip_content_update": true} - } - - -To synchronize individual repositories, use the ``type_id`` of ``repository`` and specify the -repository ID using the ``repo_id`` keyword in the ``unit_key``. - -Sample POST body: - -:: - - { - "units": [{"type_id": "repository", "unit_key": {"repo_id": "abc"}}], - "options": {} - } - diff --git a/docs/dev-guide/integration/rest-api/server_plugins.rst b/docs/dev-guide/integration/rest-api/server_plugins.rst index 01590f49ca..a6949dfb39 100644 --- a/docs/dev-guide/integration/rest-api/server_plugins.rst +++ b/docs/dev-guide/integration/rest-api/server_plugins.rst @@ -199,15 +199,6 @@ Queries the server for the loaded importer plugins. "package_environment" ] }, - { - "_href": "/pulp/api/v2/plugins/importers/nodes_http_importer/", - "display_name": "Pulp Nodes HTTP Importer", - "id": "nodes_http_importer", - "types": [ - "node", - "repository" - ] - } ] @@ -238,7 +229,7 @@ Retrieves information about a specific importer plugin. ] } - + .. _getting_distributors: Retrieve All Distributor Plugins @@ -282,14 +273,6 @@ Queries the server for the loaded distributor plugins. "puppet_module" ] }, - { - "_href": "/pulp/api/v2/plugins/distributors/nodes_http_distributor/", - "display_name": "Pulp Nodes HTTP Distributor", - "id": "nodes_http_distributor", - "types": [ - "node" - ] - }, { "_href": "/pulp/api/v2/plugins/distributors/docker_distributor_web/", "display_name": "Docker Web Distributor", @@ -297,7 +280,7 @@ Queries the server for the loaded distributor plugins. "types": [ "docker_image" ] - } + } ] @@ -334,4 +317,3 @@ Retrieves information about a specific distributor plugin. "yum_repo_metadata_file" ] } - diff --git a/docs/user-guide/admin-client/index.rst b/docs/user-guide/admin-client/index.rst index 942bd5a7f0..fbd508135c 100644 --- a/docs/user-guide/admin-client/index.rst +++ b/docs/user-guide/admin-client/index.rst @@ -11,7 +11,6 @@ Contents: consumer consumer-group events - nodes repositories orphan server diff --git a/docs/user-guide/admin-client/nodes.rst b/docs/user-guide/admin-client/nodes.rst deleted file mode 100644 index 8b203d8d58..0000000000 --- a/docs/user-guide/admin-client/nodes.rst +++ /dev/null @@ -1,240 +0,0 @@ -Nodes -===== - -This guide covers admin client commands for managing *Pulp Nodes* in the Pulp Platform. -For an overview, tips, and, troubleshooting, please visit the :ref:`Pulp Nodes Concepts Guide`. - -Layout ------- - -The root level ``node`` section contains the following features. - -:: - - $ pulp-admin node --help - Usage: pulp-admin [SUB_SECTION, ..] COMMAND - Description: pulp nodes related commands - - Available Sections: - repo - repository related commands - sync - child node synchronization commands - - Available Commands: - activate - activate a consumer as a child node - bind - bind a child node to a repository - deactivate - deactivate a child node - list - list child nodes - unbind - removes the binding between a child node and a repository - -Listing -------- - -The ``node list`` command may be used to list child :term:`nodes`. - -:: - - pulp-admin node list --help - Command: list - Description: list child nodes - - Available Arguments: - - --fields - comma-separated list of consumer fields; Example: - "id,display_name". If specified, only the given fields will be - displayed. - --bindings - if specified, the bindings information is displayed - --details - if specified, all of the consumer information is displayed - -Activation ----------- - -A Pulp server that is registered as a consumer to another Pulp server can be -designated as a :term:`child node`. Once :term:`activated` on the parent server, -the consumer is recognized as a child node of the parent and can be managed using ``node`` commands. - -To activate a consumer as a child node, use the ``node activate`` command. More information -on *node-level* synchronization strategies can be found :ref:`here`. - -:: - - $ pulp-admin node activate --help - Command: activate - Description: activate a consumer as a child node - - Available Arguments: - - --consumer-id - (required) unique identifier; only alphanumeric, -, and _ - allowed - --strategy - synchronization strategy (mirror|additive) default is additive - -A child node may be deactivated using ``node deactivate`` command. Once deactivated, the -node may no longer be managed using ``node`` commands. - -:: - - $ pulp-admin node deactivate --help - Command: deactivate - Description: deactivate a child node - - Available Arguments: - - --node-id - (required) unique identifier; only alphanumeric, -, and _ allowed - - -.. note:: Consumer (child node) un-registration will automatically deactivate the node. When a node - is activated again, it will have the same repositories bound to it as it had before - deactivation. - -Repositories ------------- - -The commands provided in the ``node repo`` section are used to perform *Nodes* specific management -of existing repositories. - -:: - - $ pulp-admin node repo --help - Usage: pulp-admin [SUB_SECTION, ..] COMMAND - Description: repository related commands - - Available Commands: - disable - disables binding to a repository by a child node - enable - enables binding to a repository by a child node - list - list node enabled repositories - publish - publishing commands - - -Listing -^^^^^^^ - -A listing of :term:`enabled repositories` may be obtained by using -the ``node repo list`` command. - -:: - - $ pulp-admin node repo list --help - Command: list - Description: list node enabled repositories - - Available Arguments: - - --details - if specified, detailed configuration information is displayed for - each repository - --fields - comma-separated list of repository fields; Example: - "id,description,display_name,content_unit_counts". If - specified, only the given fields will be displayed. - --all, -a - if specified, information on all Pulp repositories, regardless of - type, will be displayed - -Enabling -^^^^^^^^ - -A repository may be enabled using the ``node repo enable`` command. More information -on *repository-level* synchronization strategies can be found :ref:`here`. - -:: - - $ pulp-admin node repo enable --help - Command: enable - Description: enables binding to a repository by a child node - - Available Arguments: - - --repo-id - (required) unique identifier; only alphanumeric, -, and _ - allowed - --auto-publish - if "true", the nodes information will be automatically - published each time the repository is synchronized; defaults - to "true" - -.. warning:: Using auto-publish causes the *Nodes* information to be published each time the - repository is synchronized. This may increase the time it takes to perform the - synchronization depending on the size of the repository. - -Publishing -^^^^^^^^^^ - -Manually publishing the *Nodes* data necessary for child node synchronization, can be triggered -using the ``node repo publish`` command. - -:: - - $ pulp-admin node repo publish --help - Command: publish - Description: publishing commands - - Available Arguments: - - --repo-id - (required) unique identifier; only alphanumeric, -, and _ allowed - -.. note:: Repositories MUST be published for child node synchronization to be successful. - -Binding -^^^^^^^ - -The ``node bind`` command is used to associate a repository with a child node. This association -determines which repositories may be synchronized to child nodes. The strategy specified here -overrides the default strategy specified when the repository was enabled. More information on -*repository-level* synchronization strategies can be found :ref:`here`. - -:: - - $ pulp-admin node bind --help - Command: bind - Description: bind a child node to a repository - - Available Arguments: - - --repo-id - (required) unique identifier; only alphanumeric, -, and _ allowed - --node-id - (required) unique identifier; only alphanumeric, -, and _ allowed - --strategy - synchronization strategy (mirror|additive) default is additive - -The ``node unbind`` command may be used to remove the association between a child node and -a repository. Once the association is removed, the specified repository can no longer be -be synchronized to the child node. - -:: - - $ pulp-admin node unbind --help - Command: unbind - Description: removes the binding between a child node and a repository - - Available Arguments: - - --repo-id - (required) unique identifier; only alphanumeric, -, and _ allowed - --node-id - (required) unique identifier; only alphanumeric, -, and _ allowed - - - -.. note:: Only activated nodes and enabled repositories may be specified. - - -Synchronizing -------------- - -The synchronization of child nodes may be triggered using the ``node sync`` commands. More -information on node synchronization can be found :ref:`here`. - -:: - - $ pulp-admin node sync --help - Usage: pulp-admin [SUB_SECTION, ..] COMMAND - Description: child node synchronization commands - - Available Commands: - run - triggers an immediate synchronization of a child node - -An immediate synchronization can be triggered using the ``node sync run`` command. - -:: - - $ pulp-admin node sync run --help - Command: run - Description: triggers an immediate synchronization of a child node - - Available Arguments: - - --node-id - (required) unique identifier; only alphanumeric, -, and _ allowed - --max-downloads - maximum number of downloads permitted to run concurrently - --max-speed - maximum bandwidth used per download in bytes/sec - -.. warning:: Make sure repositories have been published. diff --git a/docs/user-guide/consumer-client/index.rst b/docs/user-guide/consumer-client/index.rst index ec0a8033e2..8ad3cc0f52 100644 --- a/docs/user-guide/consumer-client/index.rst +++ b/docs/user-guide/consumer-client/index.rst @@ -14,5 +14,4 @@ Contents: bind history status - nodes repositories diff --git a/docs/user-guide/consumer-client/nodes.rst b/docs/user-guide/consumer-client/nodes.rst deleted file mode 100644 index 60ba257f46..0000000000 --- a/docs/user-guide/consumer-client/nodes.rst +++ /dev/null @@ -1,91 +0,0 @@ -Nodes -===== - -This guide covers consumer client commands for managing *Pulp Nodes* in the Pulp Platform. -For an overview, tips, and, troubleshooting, please visit the :ref:`Pulp Nodes Concepts Guide`. - -Layout ------- - -The root level ``node`` section contains the following features. - -:: - - $ pulp-consumer node --help - Usage: pulp-consumer [SUB_SECTION, ..] COMMAND - Description: pulp nodes related commands - - Available Commands: - activate - activate a consumer as a child node - bind - bind this node to a repository - deactivate - deactivate a child node - unbind - remove the binding between this node and a repository - - -Activation ----------- - -A Pulp server that is registered as a consumer to another Pulp server can be -designated as a :term:`child node`. Once :term:`activated` on the parent server, -the consumer is recognized as a child node of the parent and can be managed using ``node`` commands. - -To activate a consumer as a child node, use the ``node activate`` command. More information -on *node-level* synchronization strategies can be found :ref:`here`. - -:: - - $ pulp-consumer node activate --help - Command: activate - Description: activate a consumer as a child node - - Available Arguments: - - --strategy - synchronization strategy (mirror|additive) default is additive - - -A child node may be deactivated using the ``node deactivate`` command. Once deactivated, the -node may no longer be managed using ``node`` commands. - -:: - - $ pulp-consumer node deactivate --help - Command: deactivate - Description: deactivate a child node - -.. note:: Consumer un-registration will automatically deactivate the node. - - -Binding -^^^^^^^ - -The ``node bind`` command is used to associate a child node with a repository on the parent. This -association determines which repositories may be synchronized to child nodes. The strategy specified -here overrides the default strategy specified when the repository was enabled. More information on -*repository-level* synchronization strategies can be found :ref:`here`. - -:: - - $ pulp-consumer node bind --help - Command: bind - Description: bind this node to a repository - - Available Arguments: - - --repo-id - (required) unique identifier; only alphanumeric, -, and _ allowed - --strategy - synchronization strategy (mirror|additive) default is additive - -The ``node unbind`` command may be used to remove the association between a child node and -a repository. Once the association is removed, the specified repository can no longer be -be synchronized to the child node. - -:: - - $ pulp-consumer node unbind --help - Command: unbind - Description: remove the binding between this node and a repository - - Available Arguments: - - --repo-id - (required) unique identifier; only alphanumeric, -, and _ allowed - -.. note:: Only activated nodes and enabled repositories may be specified. diff --git a/docs/user-guide/glossary.rst b/docs/user-guide/glossary.rst index 3d13a36bab..951c151ccf 100644 --- a/docs/user-guide/glossary.rst +++ b/docs/user-guide/glossary.rst @@ -10,7 +10,7 @@ Glossary & control API which is used by the Pulp server to initiate content changes on the consumer. It also sends scheduled reports concerning consumer status and installed content profiles to the Pulp server. - + binding An association between a :term:`consumer` and a :term:`repository` :term:`distributor` for the purpose of installing :term:`content units ` @@ -29,12 +29,12 @@ Glossary A managed system that is the consumer of content. Consumption refers to the installation of software contained within a :term:`repository` and published by an associated :term:`distributor`. - + content unit An individual piece of content managed by the Pulp server. A unit does not necessarily correspond to a file. It is possible that a content unit is defined as the aggregation of other content units as a grouping mechanism. - + distributor Server-side plugin that takes content from a repository and publishes it for consumption. The process by which a distributor publishes content varies @@ -81,11 +81,6 @@ Glossary an optional start time, and a time interval. More information can be found :ref:`in the conventions section of this guide `. - node - A Pulp node is a Pulp server that has either a parent or child relationship - to another Pulp server. Parent nodes provide content to child nodes. Child - nodes consume content from a parent node as registered :term:`consumers `. - registration The association of a :term:`consumer` to a Pulp server. Once registered, a consumer is added to Pulp's inventory and may be :term:`bound ` to diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst index 4072204e52..da22ff999b 100644 --- a/docs/user-guide/index.rst +++ b/docs/user-guide/index.rst @@ -17,7 +17,6 @@ Contents: authentication admin-client/index consumer-client/index - nodes content-sources deferred-download general-reference diff --git a/docs/user-guide/installation/extra_configuration.rst b/docs/user-guide/installation/extra_configuration.rst index daa21a650c..7be8731af5 100644 --- a/docs/user-guide/installation/extra_configuration.rst +++ b/docs/user-guide/installation/extra_configuration.rst @@ -60,8 +60,8 @@ If you are using a CA certificate that is not already trusted by your operating pack, you may either configure Pulp to trust that CA, or you may configure your operating system to trust that CA. -Pulp has a setting called ``ca_path`` in these files: ``/etc/pulp/admin/admin.conf``, -``/etc/pulp/consumer/consumer.conf``, and ``/etc/pulp/nodes.conf``. This setting indicates which CA +Pulp has a setting called ``ca_path`` in these files: ``/etc/pulp/admin/admin.conf`` and +``/etc/pulp/consumer/consumer.conf``. This setting indicates which CA pack each of these components should use when validating Pulp server certificates. By default, Pulp will use the operating system's CA pack. If you wish, you may adjust this setting to point to a different CA pack. The CA pack may be a single file that contains multiple concatenated @@ -109,7 +109,7 @@ Turning off Validation are communicating with. Pulp has a setting called ``verify_ssl`` in these files: ``/etc/pulp/admin/admin.conf``, -``/etc/pulp/consumer/consumer.conf``, ``/etc/pulp/nodes.conf``, and ``/etc/pulp/repo_auth.conf``. If +``/etc/pulp/consumer/consumer.conf`` and ``/etc/pulp/repo_auth.conf``. If you configure these settings to false, the respective Pulp components will no longer validate the Pulp server's certificate signature. diff --git a/docs/user-guide/nodes.rst b/docs/user-guide/nodes.rst deleted file mode 100644 index 7672271fbe..0000000000 --- a/docs/user-guide/nodes.rst +++ /dev/null @@ -1,497 +0,0 @@ -.. _pulp_nodes: - -Nodes -===== - -Overview --------- - -The *Pulp Nodes* concept describes the relationship between two Pulp servers for the purpose of -sharing content. In this relationship, one is designated the *parent* and the other is designated -the *child*. The *child* node consumes content that is provided by the *parent* node. -It is important to understand that a child :term:`node` is a complete and fully functional Pulp -server capable of operating autonomously. - -The following terms are used when discussing *Nodes*: - -.. glossary:: - - node - A Pulp server that has the *Nodes* support installed and has a content sharing - relationship to another Pulp server. - - parent node - A Pulp node that provides content to another Pulp server that has been registered - as a :term:`consumer` and activated as a node. - - child node - A Pulp node that consumes content from another Pulp server. The child node must be - registered as a consumer to the parent and been activated as a child node. - - node activation - The designation of a registered consumer as a child node. - - enabled repository - A Pulp repository that has been *enabled* for :term:`binding` by child nodes. - - -Node Topologies -^^^^^^^^^^^^^^^ - -Pulp nodes may be associated to form tree structures. Intermediate nodes may be designated -as both a parent and a child node. - -.. image:: /_static/node-topology.png - - -Node Anatomy -^^^^^^^^^^^^ - -The anatomy of both parent and child nodes is simple. Parent nodes are Pulp servers -that have the *Nodes* support installed. A Child node is a Pulp server with both the *Nodes* -and *Consumer* support installed. - -.. image:: /_static/node-anatomy.png - - -Authentication --------------- - -The *child* node is authenticated to the *parent* node's REST API using Oauth. -The connection is SSL but no client certificate is required. The *parent* node publishes -content which is downloaded (as needed) by the *child* node using HTTPS requiring client -certificate that has been signed by the Pulp CA. - -During installation of the *Nodes* packages, an x.509 certificate is generated and signed -using the Pulp CA (specified in server.conf) and stored in ``/etc/pki/pulp/nodes/node.crt``. -The certificate is generated using ``/usr/bin/pulp-gen-nodes-certificate``, which is provided -by the *Nodes* packages. - -The node private key and certificate are sent to the *child* node at the beginning of each -synchronization request to be used as the HTTPS credentials. - -If the Pulp CA is changed after installation, administrators **must** regenerate the *Nodes* -certificate using *pulp-gen-nodes-certificate*. - - -Installation ------------- - -Since Pulp nodes *are* Pulp servers, the installation instructions for *Nodes* support -assumes that the :ref:`server installation ` has been completed. Next, -follow the instructions below on each server depending on its intended role within the -node topology. - -Parent -^^^^^^ - -To install *Nodes* parent support, follow the instructions below. - -1. Install the node parent package group. - -:: - - $ sudo yum install pulp-nodes-parent pulp-nodes-admin-extensions - -2. The communication between the child and parent nodes is secured using OAuth. The parent node - must have OAuth enabled and configured. Please see :ref:`OAuth ` for instructions - on enabling and configuring OAuth. - -3. Run ``pulp-manage-db``. - -4. Restart :ref:`server components`. - - -Child -^^^^^ - -To install *Nodes* child support, follow the instructions below. - -1. Install the node child package group. - -:: - - $ sudo yum install pulp-nodes-child pulp-nodes-consumer-extensions pulp-agent - -2. The communication between the child and parent nodes is secured using OAuth. The child node - must have OAuth enabled and configured. Please see :ref:`OAuth ` for instructions - on enabling and configuring OAuth. - -:: - - [oauth] - enabled: true - oauth_key: Xohkaethaama5eki - oauth_secret: eePa7Bi3gohdir1pai2icohvaidai0io - -.. warning:: - - Do not use the key or secret given in the above example. It is important that you use unique and - secret values for these configuration items. - -3. Edit ``/etc/pulp/nodes.conf`` on the child node and set the parent OAuth *key* and *secret* to - match values found in ``/etc/pulp/server.conf`` on the parent node. The *user_id* must be updated - as needed to match a user with administration privileges on the parent node. - -.. warning:: - - The keys in ``[parent_oauth]`` are ``key`` and ``secret``, whereas the values in the - ``[oauth]`` section are ``oauth_key`` and ``oauth_secret``. If you copy and paste from one - to another without altering the names of the keys then the child node will not be able to - communicate with the parent node. - -:: - - [oauth] - user_id: - - [parent_oauth] - key: - secret: - user_id: - -Example: - -:: - - [oauth] - user_id: admin - - [parent_oauth] - key: Xohkaethaama5eki - secret: eePa7Bi3gohdir1pai2icohvaidai0io - user_id: admin - -4. Run ``pulp-manage-db``. - -5. Restart :ref:`server components`. - -6. Restart ``goferd``. - - -Admin Client Extensions -^^^^^^^^^^^^^^^^^^^^^^^ - -The admin extensions provide *Nodes* specific commands used to perform node administration -which includes the following: - - * Child node activation. - * Child node deactivation. - * List child nodes. - * Enable repositories for node binding. - * Disable repositories for node binding. - * List enabled repositories. - * Bind a child node to a repository. - * Unbind a child node from a repository. - * Initiate repository publishing of *Nodes* content. - * Initiate child node synchronization. - -Install the *Nodes* admin client extensions. - -:: - - $ sudo yum install pulp-nodes-admin-extensions - - -Enabling Repositories ---------------------- - -In Pulp *Nodes*, there is a concept of enabling and disabling repositories for use with child -nodes. Repositories must be *enabled* before being referenced in node bindings. - -Repositories may be enabled using the admin client. See ``node repo`` commands for details. - -:: - - $ pulp-admin node repo enable --repo-id - -:: - - $ pulp-admin node repo disable --repo-id - -Listing the enabled repositories can be done using the admin client. See: the ``node repo list`` -for details. - -:: - - $ pulp-admin node repo list - - -.. _node_repository_publishing: - -Repository Publishing ---------------------- - -After a repository has been enabled, it MUST be published before synchronizing content -to child nodes. Publishing a *Nodes* enabled repository generates the data necessary for -repository content synchronization with child nodes. If auto-publishing is enabled, a normal -repository synchronization will result in publishing this data as well. - -The size of the published data varies based on the number of content units contained in the -repository and the amount of metadata included in each unit. Each published unit consists of a -copy of the metadata and a symlink to the actual file associated with the unit. The metadata is -stored as gzip-compressed JSON. - -The *Nodes* information can be manually published using the admin client. -See: the ``node repo publish`` for details. - -:: - - $ pulp-admin node repo publish --repo-id - - -Registration & Activation -------------------------- - -Once the *Nodes* child support has been installed on a Pulp server, it can be registered to a -parent server. This is accomplished using the Pulp consumer client. As mentioned, a child -node is both a Pulp server and a consumer that is registered to the parent node. - -On the child Pulp server: - -1. Edit the ``/etc/pulp/consumer/consumer.conf`` file and set the ``host`` property the to the - hostname or IP address of the Pulp server to be use as the child node's parent. - -:: - - [server] - host = - -2. Register to the parent server as a consumer. This command will prompt for a password. - -:: - - $ sudo pulp-consumer -u register --consumer-id - -3. Active the Pulp server as a child node. See: the ``node activate`` command for details. - -:: - - $ sudo pulp-consumer node activate - - -.. _node_binding: - -Binding To Repositories ------------------------ - -The selection of content to be replicated to child nodes is defined by repository bindings. -Using the *Nodes* ``bind`` and ``unbind`` commands, users create an association between the -child node and *Nodes* enabled repositories. - -Examples: - -:: - - $ pulp-admin node bind --node-id --repo-id - -:: - - $ pulp-consumer node bind --repo-id - - -.. _node_synchronization: - -Child Synchronization ---------------------- - -A child node's repositories and their content can be synchronized with the parent. Technically, -this action is seen by the parent as a content update on one of it's consumers. But, for most -users, the term synchronization is easier to grasp. During this process, the following objects -and properties are replicated to the child node: - -* Repositories - - * description - * notes - -* Distributors - - * configuration (includes certificates and other credentials) - -* Content Units - - * metadata - * associated files (bits) - - -.. _node_strategies: - -Strategies -^^^^^^^^^^ - -During child node synchronization, named strategies determine how the synchronization -is performed and what the desired effect will be. Strategies are incorporated at two levels -during node synchronization. - -The first is the *node* level strategy which determines how the collection of repository objects are -synchronized. Depending on the selected strategy, repositories are created, updated or deleted -to match the set of repositories to which the node is associated through bindings. - -The second is the *repository* level strategy which determines how each repository's content is -synchronized. Depending on the selected strategy, content units are created, updated or deleted -to match the content contained in the repository on the parent. - -Current, there are two supported strategies. - - additive - Results in objects present in the parent but not in the child being created or updated - as necessary. This strategy should be used when objects created locally in the child - should be preserved. - - mirror - Results in objects present in the parent but not in the child being created or updated - as necessary. Any objects present in the child that do not exist in the parent are removed. - This strategy should be used when the desired effect of synchronization is for the child - repositories to be an exact mirror of those on the parent. - -The *node* level strategy is specified during node activation. Once activated, the strategy may -be changed by performing a node deactivation followed by node activation specifying the desired -strategy. - -The *repository* level strategy is specified during node binding to a repository. Once bound, the -strategy may be changed by performing an unbind followed by a bind to the repository specifying the -desired strategy. - -.. note:: The ``additive`` strategy is the default. - -Running -^^^^^^^ - -The synchronization of a child node can be initiated using the admin client. This results in a -request being sent to the agent on the child node which performs the update. A *partial* -synchronization can be initiated by doing a regular repository synchronization on the child node. -This will synchronize only the content of the repository. - -The synchronization can be requested using the admin client. See: the ``node sync`` command. - -:: - - $ pulp-admin node sync run --node-id - -Scheduling -^^^^^^^^^^ - -Synchronization of a particular child can be scheduled with an optional recurrence. The -format for describing the schedule follows the Pulp standard for -:ref:`date and time intervals `. All commands related to the -creation, removal, and listing of node sync schedules can be found under the -``node sync schedules`` command. - - -.. _node_quick_start: - -Quick Start ------------ - -This assumes there are two Pulp servers up and running. The following steps could generally be -followed to get a basic *Nodes* parent and child setup going. To simplify the writeup, it's -assumed that the parent server's hostname is ``parent.redhat.com`` and it has a repository -named ``pulp-goodness`` that we want to share with our child. - -On The Parent -^^^^^^^^^^^^^ - -On the Pulp server to be used as the parent node: - -1. Install the pulp-nodes-parent package group. - -:: - - $ sudo yum install pulp-nodes-parent pulp-nodes-admin-extensions - $ sudo service httpd restart - -2. Enable the ``pulp-goodness`` repository. - -:: - - $ pulp-admin node repo enable --repo-id pulp-goodness - -3. Publish the ``pulp-goodness`` repository. - -:: - - $ pulp-admin node repo publish --repo-id pulp-goodness - - -On The Child -^^^^^^^^^^^^ - -On the Pulp server to be used as the child node: - -1. Install the pulp-nodes-child package group. - -:: - - $ sudo yum install pulp-nodes-child pulp-nodes-consumer-extensions pulp-agent - -2. Edit ``/etc/pulp/nodes.conf`` and set the parent OAuth *key* and *secret* to match values found in - ``/etc/pulp/server.conf`` on the parent node. - -:: - - [parent_oauth] - key: - secret: - -3. Edit ``/etc/pulp/consumer/consumer.conf`` and change: - -:: - - [server] - host = parent.redhat.com - -4. Restart Apache. For upstart:: - - $ sudo service httpd restart - - For systemd:: - - $sudo systemctl restart httpd - -5. Restart the Pulp agent. For upstart:: - - $ sudo service goferd restart - - For systemd:: - - $ sudo systemctl restart goferd - - -6. Register as a consumer. This command will prompt for a password. - -:: - - $ pulp-consumer register --consumer-id child-1 - -7. Activate the node. - -:: - - $ pulp-consumer node activate - - -8. Bind to the ``pulp-goodness`` repository. - -:: - - $ pulp-consumer node bind --repo-id pulp-goodness - - -Anywhere Using Admin Client -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -1. Synchronize the child. - -:: - - $ pulp-admin node sync run --node-id child-1 - - -Tips & Troubleshooting ----------------------- - -1. Make sure httpd was restarted after installing *Nodes* packages on both the parent and child. -2. Make sure goferd was restarted after installing *Nodes* packages on the child. -3. Make sure that *Nodes* enabled repositories have been published. -4. Make sure that ALL plugins installed on the parent are installed on the child. diff --git a/docs/user-guide/release-notes/2.1.x.rst b/docs/user-guide/release-notes/2.1.x.rst index 8e3fd0e0f4..39d734c3ba 100644 --- a/docs/user-guide/release-notes/2.1.x.rst +++ b/docs/user-guide/release-notes/2.1.x.rst @@ -51,7 +51,7 @@ New Features ------------ #. Pulp now has support for hierarchical collections of Pulp Servers that are able to synchronize with each - other. These are called Pulp Nodes, and you can read more about them :doc:`in the nodes section <../nodes>`. + other. These are called Pulp Nodes, and you can read more about them in the nodes section. #. Unit counts within each repository are now tracked by type. #. We now support Fedora 18 and Apache 2.4. #. It is now possible to upgrade from Pulp 1.1 to 2.1. diff --git a/docs/user-guide/release-notes/2.2.x.rst b/docs/user-guide/release-notes/2.2.x.rst index d0a6554771..f168161aa6 100644 --- a/docs/user-guide/release-notes/2.2.x.rst +++ b/docs/user-guide/release-notes/2.2.x.rst @@ -11,7 +11,7 @@ New Features ------------ * Child node synchronization can now be scheduled with an optional recurrence. More - information can be found in the :doc:`../nodes` section of the user guide. + information can be found in the nodes section of the user guide. * Some consumer operations can now be canceled. * Orphan removal performs much better. @@ -59,4 +59,3 @@ See the RPM-specific user guide for highlights of the most important bug fixes there. All bug fixes for this release can be seen at this link: `All Bug Fixes `_ - diff --git a/docs/user-guide/release-notes/2.3.x.rst b/docs/user-guide/release-notes/2.3.x.rst index 87af26aab9..1a6a659d31 100644 --- a/docs/user-guide/release-notes/2.3.x.rst +++ b/docs/user-guide/release-notes/2.3.x.rst @@ -75,7 +75,7 @@ New Node Features - The authentication method used by *Nodes* has been changed to OAuth. Users upgrading Pulp servers that are functioning as child *Nodes* will need to update a new *Nodes* configuration - file as specified in :doc:`../nodes` section of this user guide. + file as specified in nodes section of this user guide. Bugs ---- diff --git a/docs/user-guide/release-notes/3.0.x.rst b/docs/user-guide/release-notes/3.0.x.rst new file mode 100644 index 0000000000..90169e9c52 --- /dev/null +++ b/docs/user-guide/release-notes/3.0.x.rst @@ -0,0 +1,14 @@ +======================= +Pulp 3 Release Notes +======================= + +Pulp 3 +====== + +Changelog +--------- + +* Nodes are no longer supported and were completely removed with + Pulp 3.0. Instead of using nodes, publish content from one Pulp server + and use that feed url to sync the content into another Pulp server. + This is the recommended approach for a nodes use case. diff --git a/docs/user-guide/release-notes/index.rst b/docs/user-guide/release-notes/index.rst index 867183c940..480abb4c63 100644 --- a/docs/user-guide/release-notes/index.rst +++ b/docs/user-guide/release-notes/index.rst @@ -6,6 +6,7 @@ Contents: .. toctree:: :maxdepth: 2 + 3.0.x 2.11.x 2.10.x 2.9.x diff --git a/docs/user-guide/scaling.rst b/docs/user-guide/scaling.rst index e3d1a5d8da..5e53c39d6a 100644 --- a/docs/user-guide/scaling.rst +++ b/docs/user-guide/scaling.rst @@ -151,9 +151,7 @@ Clustering Pulp --------------- A clustered Pulp installation is comprised of two or more `Pulp clustered -servers`. The term `Pulp clustered server` is used to distinguish it as a -separate concept from :ref:`pulp_nodes`. `Pulp clustered servers` share the -following components: +servers`. `Pulp clustered servers` share the following components: +--------------------+---------------------------------------------------------+ | Pulp Configuration | Pulp reads its configuration from conf files inside | diff --git a/docs/user-guide/troubleshooting.rst b/docs/user-guide/troubleshooting.rst index da763da66d..c5f528591b 100644 --- a/docs/user-guide/troubleshooting.rst +++ b/docs/user-guide/troubleshooting.rst @@ -237,10 +237,6 @@ issue you can: 2. Follow the `Qpid scalability guide`_ for configuring Qpid to handle a large number of persistent queues. - 3. Consider spreading your consumers over multiple Pulp installations, each with its own Qpid - broker to reduce the number of Pulp Consumers per broker. The Pulp nodes feature should make - this architecture manageable. - .. _Qpid scalability guide: https://bugzilla.redhat.com/attachment.cgi?id=930496 User permissions not behaving as expected diff --git a/nodes/__init__.py b/nodes/__init__.py deleted file mode 100644 index 32466f8d26..0000000000 --- a/nodes/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/nodes/child/etc/pulp/agent/conf.d/nodes.conf b/nodes/child/etc/pulp/agent/conf.d/nodes.conf deleted file mode 100644 index fe15a1324b..0000000000 --- a/nodes/child/etc/pulp/agent/conf.d/nodes.conf +++ /dev/null @@ -1,11 +0,0 @@ -[main] -enabled=1 - -[types] -content=node,repository - -[node] -class=pulp_node.handlers.handler.NodeHandler - -[repository] -class=pulp_node.handlers.handlers.RepositoryHandler diff --git a/nodes/child/etc/pulp/server/plugins.conf.d/nodes/importer/http.conf b/nodes/child/etc/pulp/server/plugins.conf.d/nodes/importer/http.conf deleted file mode 100644 index 7a73a41bfd..0000000000 --- a/nodes/child/etc/pulp/server/plugins.conf.d/nodes/importer/http.conf +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file diff --git a/nodes/child/pulp_node/__init__.py b/nodes/child/pulp_node/__init__.py deleted file mode 100644 index b36383a610..0000000000 --- a/nodes/child/pulp_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/nodes/child/pulp_node/handlers/__init__.py b/nodes/child/pulp_node/handlers/__init__.py deleted file mode 100644 index 32466f8d26..0000000000 --- a/nodes/child/pulp_node/handlers/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/nodes/child/pulp_node/handlers/handler.py b/nodes/child/pulp_node/handlers/handler.py deleted file mode 100644 index e74916826f..0000000000 --- a/nodes/child/pulp_node/handlers/handler.py +++ /dev/null @@ -1,163 +0,0 @@ -from logging import getLogger - -from pulp.agent.lib.handler import ContentHandler -from pulp.agent.lib.report import ContentReport - -from pulp_node import constants -from pulp_node import resources -from pulp_node.error import GetBindingsError -from pulp_node.handlers.model import RepositoryBinding -from pulp_node.handlers.reports import HandlerProgress, SummaryReport -from pulp_node.handlers.strategies import find_strategy, Request - - -log = getLogger(__name__) - - -def parent_bindings(options): - """ - Get the parent API bindings based on handler options. - :param options: Update options. - :type options: dict - :return: A configured parent API bindings. - :rtype: pulp.bindings.bindings.Bindings - """ - settings = options[constants.PARENT_SETTINGS] - host = settings[constants.HOST] - port = settings[constants.PORT] - return resources.parent_bindings(host, port) - - -class NodeHandler(ContentHandler): - - def update(self, conduit, units, options): - """ - Update the specified content units. Each unit must be of - type 'node'. Updates the entire child node. - - Report format: - succeeded: - details: { - errors: [ - { error_id: , - details: {} - }, - ] - repositories: [ - { repo_id: , - action: , - units: { - added: , - updated: , - removed: - } - }, - ] - } - - :param conduit: A handler conduit. - :type conduit: pulp.agent.lib.conduit.Conduit - :param units: A list of content unit_keys. - :type units: list - :param options: Unit update options. - :type options: dict - :return: An update report. - :rtype: ContentReport - """ - handler_report = ContentReport() - summary_report = SummaryReport() - progress_report = HandlerProgress(conduit) - pulp_bindings = parent_bindings(options) - - try: - bindings = RepositoryBinding.fetch_all(pulp_bindings, conduit.consumer_id) - except GetBindingsError, ne: - log.error(ne) - summary_report.errors.append(ne) - handler_report.set_failed(summary_report.dict()) - return handler_report - - strategy_name = options.setdefault(constants.STRATEGY_KEYWORD, constants.MIRROR_STRATEGY) - request = Request( - conduit=conduit, - progress=progress_report, - summary=summary_report, - bindings=bindings, - scope=constants.NODE_SCOPE, - options=options) - strategy = find_strategy(strategy_name)() - strategy.synchronize(request) - - for ne in summary_report.errors: - log.error(ne) - - if summary_report.succeeded(): - handler_report.set_succeeded(summary_report.dict()) - else: - handler_report.set_failed(summary_report.dict()) - return handler_report - - -class RepositoryHandler(ContentHandler): - - def update(self, conduit, units, options): - """ - Update the specified content units. Each unit must be - of type 'repository'. Updates only the repositories specified in - the unit_key by repo_id. - - Report format: - succeeded: - details: { - errors: [ - { error_id: , - details: {} - }, - ] - repositories: [ - { repo_id: , - action: , - units: { - added: , - updated: , - removed: - } - }, - ] - } - - :param conduit: A handler conduit. - :type conduit: pulp.agent.lib.conduit.Conduit - :param units: A list of content unit_keys. - :type units: list - :param options: Unit update options. - :type options: dict - :return: An update report. - :rtype: ContentReport - """ - summary_report = SummaryReport() - progress_report = HandlerProgress(conduit) - repo_ids = [key['repo_id'] for key in units if key] - pulp_bindings = parent_bindings(options) - bindings = RepositoryBinding.fetch(pulp_bindings, conduit.consumer_id, repo_ids) - - strategy_name = options.setdefault(constants.STRATEGY_KEYWORD, constants.MIRROR_STRATEGY) - request = Request( - conduit=conduit, - progress=progress_report, - summary=summary_report, - bindings=bindings, - scope=constants.REPOSITORY_SCOPE, - options=options) - strategy = find_strategy(strategy_name)() - strategy.synchronize(request) - - for ne in summary_report.errors: - log.error(ne) - - handler_report = ContentReport() - if summary_report.succeeded(): - handler_report.set_succeeded(summary_report.dict()) - else: - handler_report.set_failed(summary_report.dict()) - return handler_report diff --git a/nodes/child/pulp_node/handlers/model.py b/nodes/child/pulp_node/handlers/model.py deleted file mode 100644 index cf81e52b89..0000000000 --- a/nodes/child/pulp_node/handlers/model.py +++ /dev/null @@ -1,532 +0,0 @@ -""" -Provides classes representing the database objects contained in either -the parent or child pulp server. Parent objects are read-only and are used for querying, -comparison and merging to child objects. Child objects are used for querying, comparison -and merging from remote objects. Unlike parent objects, child objects are also used to apply -changes to the child database and to trigger repository synchronization. These objects cover -repositories and their associated plugins. Content units are not represented here. That is -the responsibility of the nodes importers. -""" - -import httplib -from logging import getLogger - -from pulp.bindings.exceptions import NotFoundException -from pulp.common.bundle import Bundle -from pulp.common.plugins import importer_constants - -from pulp_node import constants -from pulp_node import resources -from pulp_node.error import PurgeOrphansError, RepoSyncRestError, GetBindingsError -from pulp_node.poller import TaskPoller - - -log = getLogger(__name__) - - -def subdict(adict, *keylist): - """ - Get a subset dictionary. - @param adict: A dictionary to subset. - :type adict: dict - :param keylist: A list of keys to be included in the subset. - :type keylist: list - :return: The subset dictionary. - :rtype: dict. - """ - return dict([t for t in adict.items() if t[0] in keylist]) - - -class Entity(object): - """ - Model entity base. - """ - - @staticmethod - def purge_orphans(): - """ - Purge orphaned units within the inventory. - """ - bindings = resources.pulp_bindings() - http = bindings.content_orphan.remove_all() - if http.response_code != httplib.ACCEPTED: - raise PurgeOrphansError(http.response_code) - - -class Repository(Entity): - """ - Represents a repository database object. - :ivar repo_id: Repository ID. - :type repo_id: str - :ivar details: Repository details as modeled by bind payload. - :type details: dict - """ - - def __init__(self, repo_id, details=None): - """ - :param repo_id: The repository ID. - :type repo_id: str - :param details: The repository details defined in the bind payload. - :type details: dict - :return: - """ - self.repo_id = repo_id - self.details = details or {} - - @staticmethod - def fetch_all(): - """ - Fetch all repositories from the inventory. - :return: A list of: Repository - :rtype: list - """ - repositories = [] - bindings = resources.pulp_bindings() - for repository in bindings.repo_search.search(): - repo_id = repository['id'] - details = { - 'repository': repository, - 'distributors': [] - } - r = Repository(repo_id, details) - repositories.append(r) - return repositories - - @staticmethod - def fetch(repo_id): - """ - Fetch a specific repository from the inventory. - :param repo_id: Repository ID. - :type repo_id: str - :return: The fetched repository. - :rtype: Repository - """ - details = {} - bindings = resources.pulp_bindings() - try: - http = bindings.repo.repository(repo_id) - details['repository'] = http.response_body - http = bindings.repo_distributor.distributors(repo_id) - details['distributors'] = http.response_body - http = bindings.repo_importer.importers(repo_id) - details['importers'] = http.response_body - return Repository(repo_id, details) - except NotFoundException: - return None - - @property - def basic_properties(self): - """ - Get basic mutable properties within the bind payload details. - :return: A dict of basic properties. - :rtype: dict - """ - return subdict(self.details['repository'], 'display_name', 'description', 'notes', - 'scratchpad') - - @property - def distributors(self): - """ - Get the list of distributors defined in the bind payload details. - :return: A list of distributors. - :rtype: list - """ - return self.details['distributors'] - - @property - def importers(self): - """ - Get the list of importers defined in the bind payload details. - :return: A list of importers. - :rtype: list - """ - return self.details['importers'] - - def add(self): - """ - Add the repository and associated plugins. - """ - # repository - bindings = resources.pulp_bindings() - bindings.repo.create( - self.repo_id, - self.basic_properties['display_name'], - self.basic_properties['description'], - self.basic_properties['notes']) - bindings.repo.update(self.repo_id, {'scratchpad': self.basic_properties['scratchpad']}) - # distributors - for details in self.distributors: - dist_id = details['id'] - dist = Distributor(self.repo_id, dist_id, details) - dist.add() - # importers - for details in self.importers: - imp_id = details['id'] - importer = Importer(self.repo_id, imp_id, details) - importer.add() - log.info('Repository: %s, added', self.repo_id) - - def update(self, delta): - """ - Update this repository. - :param delta: The properties that need to be updated. - :type delta: dict - """ - bindings = resources.pulp_bindings() - bindings.repo.update(self.repo_id, delta) - log.info('Repository: %s, updated', self.repo_id) - - def delete(self): - """ - Delete this repository. - """ - bindings = resources.pulp_bindings() - bindings.repo.delete(self.repo_id) - log.info('Repository: %s, deleted', self.repo_id) - - def merge(self, repository): - """ - Merge another repository. - 1. Determine the delta and update the repository properties. - 2. Merge importers - 3. Merge distributors - :param repository: Another repository. - :type repository: Repository - """ - delta = {} - for k, v in repository.basic_properties.items(): - if self.basic_properties.get(k) != v: - self.basic_properties[k] = v - delta[k] = v - if delta: - self.update(delta) - self.merge_importers(repository) - self.merge_distributors(repository) - - def merge_importers(self, repository): - """ - Merge importers. - - Delete importers associated to this repository but not - associated with the other repository. - - Merge importers associated with this repository AND associated - with the other repository. - - Add importers associated with the other repository but NOT associated - with this repository. - :param repository: Another repository. - :type repository: Repository - """ - self.delete_importers(repository) - for details in repository.importers: - imp_id = details['id'] - importer_in = Importer(self.repo_id, imp_id, details) - importer = Importer.fetch(self.repo_id, imp_id) - if importer: - importer.merge(importer_in) - else: - importer_in.add() - - def delete_importers(self, repository): - """ - Delete importers associated with this repository but not - associated with the other repository. - :param repository: Another repository. - :type repository: Repository - """ - wanted_ids = [d['id'] for d in repository.importers] - for details in self.importers: - imp_id = details['id'] - if imp_id not in wanted_ids: - importer = Importer(self.repo_id, imp_id, {}) - importer.delete() - - def merge_distributors(self, repository): - """ - Merge distributors. - - Merge distributors associated with this repository AND - associated with the other repository. - - Add distributors associated with the other repository but - NOT associated with this repository. - :param repository: Another repository. - :type repository: Repository - """ - for details in repository.distributors: - dist_id = details['id'] - distributor_in = Distributor(self.repo_id, dist_id, details) - distributor = Distributor.fetch(self.repo_id, dist_id) - if distributor: - distributor.merge(distributor_in) - else: - distributor_in.add() - - def run_synchronization(self, progress, cancelled, options): - """ - Run a repo_sync() on this repository. - :param progress: A progress report. - :type progress: pulp_node.progress.RepositoryProgress - :param options: node synchronization options. - :type options: dict - :return: The task result. - """ - bindings = resources.pulp_bindings() - poller = TaskPoller(bindings) - max_download = options.get( - constants.MAX_DOWNLOAD_CONCURRENCY_KEYWORD, - constants.DEFAULT_DOWNLOAD_CONCURRENCY) - node_certificate = options[constants.PARENT_SETTINGS][constants.NODE_CERTIFICATE] - key, certificate = Bundle.split(node_certificate) - configuration = { - importer_constants.KEY_MAX_DOWNLOADS: max_download, - importer_constants.KEY_MAX_SPEED: options.get(constants.MAX_DOWNLOAD_BANDWIDTH_KEYWORD), - importer_constants.KEY_SSL_CLIENT_KEY: key, - importer_constants.KEY_SSL_CLIENT_CERT: certificate, - importer_constants.KEY_SSL_VALIDATION: False, - } - http = bindings.repo_actions.sync(self.repo_id, configuration) - if http.response_code != httplib.ACCEPTED: - raise RepoSyncRestError(self.repo_id, http.response_code) - # The repo sync is returned with a single sync task in the Call Report - task = http.response_body.spawned_tasks[0] - result = poller.join(task.task_id, progress, cancelled) - if cancelled(): - self._cancel_synchronization(task) - return result - - def _cancel_synchronization(self, task): - """ - Cancel a task associated with a repository synchronization. - :param task: A running task. - :type task: pulp.bindings.responses.Task - """ - bindings = resources.pulp_bindings() - http = bindings.tasks.cancel_task(task.task_id) - if http.response_code == httplib.ACCEPTED: - log.info('Task [%s] canceled', task.task_id) - else: - log.error('Task [%s] cancellation failed http=%s', task.task_id, http.response_code) - - def __str__(self): - return 'repository: %s' % self.repo_id - - -class Distributor(Entity): - """ - Represents a repository-distributor association. - :ivar repo_id: Repository ID. - :type repo_id: str - :param dist_id: Distributor ID. - :type dist_id: str - :ivar details: Distributor details as modeled in the bind payload. - :type details: dict - """ - - @staticmethod - def fetch(repo_id, dist_id): - """ - Fetch the repository-distributor from the inventory. - :param repo_id: The repository ID. - :type repo_id: str - :param dist_id: A distributor ID. - :type dist_id: str - :return: The fetched distributor. - :rtype: Distributor - """ - try: - bindings = resources.pulp_bindings() - http = bindings.repo_distributor.distributor(repo_id, dist_id) - details = http.response_body - return Distributor(repo_id, dist_id, details) - except NotFoundException: - return None - - def __init__(self, repo_id, dist_id, details): - """ - :param repo_id: Repository ID. - :type repo_id: str - :param dist_id: Distributor ID. - :type dist_id: str - :param details: Distributor details as modeled in the bind payload. - :type details: dict - """ - self.repo_id = repo_id - self.dist_id = dist_id - self.details = subdict(details, 'config', 'auto_publish', 'distributor_type_id') - - def add(self): - """ - Add this repository-distributor to the inventory. - """ - bindings = resources.pulp_bindings() - bindings.repo_distributor.create( - self.repo_id, - self.details['distributor_type_id'], - self.details['config'], - self.details['auto_publish'], - self.dist_id) - log.info('Distributor: %s/%s, added', self.repo_id, self.dist_id) - - def update(self, configuration): - """ - Update this repository-distributor in the inventory. - :param configuration: The updated configuration. - :type configuration: dict - """ - bindings = resources.pulp_bindings() - bindings.repo_distributor.update(self.repo_id, self.dist_id, configuration) - log.info('Distributor: %s/%s, updated', self.repo_id, self.dist_id) - - def delete(self): - """ - Delete this distributor. - """ - bindings = resources.pulp_bindings() - bindings.repo_distributor.delete(self.repo_id, self.dist_id) - log.info('Distributor: %s/%s, deleted', self.repo_id, self.dist_id) - - def merge(self, distributor): - """ - Merge the distributor configuration from another distributor. - :param distributor: Another distributor. - :type distributor: Distributor - """ - key = 'config' - configuration = distributor.details[key] - if self.details[key] != configuration: - self.update(configuration) - - def __str__(self): - return 'distributor: %s.%s' % (self.repo_id, self.dist_id) - - -class Importer(Entity): - """ - Represents a repository-importer association. - :ivar repo_id: Repository ID. - :type repo_id: str - :param imp_id: Importer ID. - :type imp_id: str - :ivar details: Importer details as modeled in the bind payload. - :type details: dict - """ - - @staticmethod - def fetch(repo_id, imp_id): - """ - Fetch the repository-importer from the inventory. - :return: The fetched importer. - :rtype: Importer - """ - try: - bindings = resources.pulp_bindings() - http = bindings.repo_importer.importer(repo_id, imp_id) - details = http.response_body - return Importer(repo_id, imp_id, details) - except NotFoundException: - return None - - def __init__(self, repo_id, imp_id, details): - """ - :param repo_id: Repository ID. - :type repo_id: str - :param imp_id: Importer ID. - :type imp_id: str - :param details: Importer details as modeled in the bind payload. - :type details: dict - """ - self.repo_id = repo_id - self.imp_id = imp_id - self.details = details - - def add(self): - """ - Add this importer to the inventory. - """ - conf = self.details['config'] - bindings = resources.pulp_bindings() - bindings.repo_importer.create(self.repo_id, self.imp_id, conf) - log.info('Importer %s/%s, added', self.repo_id, self.imp_id) - - def update(self, configuration): - """ - Update this importer. - :param configuration: The updated configuration. - :type configuration: dict - """ - bindings = resources.pulp_bindings() - bindings.repo_importer.update(self.repo_id, self.imp_id, configuration) - log.info('Importer: %s/%s, updated', self.repo_id, self.imp_id) - - def delete(self): - """ - Delete this importer. - """ - bindings = resources.pulp_bindings() - bindings.repo_importer.delete(self.repo_id, self.imp_id) - log.info('Importer: %s/%s, deleted', self.repo_id, self.imp_id) - - def merge(self, importer): - """ - Merge this importer configuration from another importer. - :param importer: Another importer. - :type importer: Importer - """ - self.update(importer.details['config']) - - def __str__(self): - return 'importer: %s.%s' % (self.repo_id, self.imp_id) - - -class RepositoryBinding(Entity): - """ - Represents a parent node bindings to a repository. - """ - - @staticmethod - def fetch_all(bindings, node_id): - """ - Fetch a list of ALL bind payloads for this consumer. - :param bindings: A pulp API object. - :type bindings: pulp.bindings.bindings.Bindings - :param node_id: The node ID. - :type node_id: str - :return: List of bind payloads. - :rtype: list - """ - http = bindings.bind.find_by_id(node_id) - if http.response_code == httplib.OK: - return RepositoryBinding.filtered(http.response_body) - else: - raise GetBindingsError(http.response_code) - - @staticmethod - def fetch(bindings, node_id, repo_ids): - """ - Fetch a list of bind payloads for the specified list of repository ID. - :param bindings: A pulp API object. - :type bindings: pulp.bindings.bindings.Bindings - :param node_id: The node ID. - :type node_id: str - :param repo_ids: A list of repository IDs. - :type repo_ids: list - :return: List of bind payloads. - :rtype: list - """ - binds = [] - for repo_id in repo_ids: - http = bindings.bind.find_by_id(node_id, repo_id) - if http.response_code == httplib.OK: - binds.extend(RepositoryBinding.filtered(http.response_body)) - else: - raise GetBindingsError(http.response_code) - return binds - - @staticmethod - def filtered(binds): - """ - Get a filtered list of binds. - - Includes only the nodes_ distributors. - :param binds: A list of bind payloads. - :type binds: list - :return: The filtered list of bind payloads. - :rtype: list - """ - return [b for b in binds if b['type_id'] in constants.ALL_DISTRIBUTORS] diff --git a/nodes/child/pulp_node/handlers/reports.py b/nodes/child/pulp_node/handlers/reports.py deleted file mode 100644 index d1b7d1cf2f..0000000000 --- a/nodes/child/pulp_node/handlers/reports.py +++ /dev/null @@ -1,148 +0,0 @@ -from pulp_node.error import ErrorList -from pulp_node.reports import RepositoryReport, RepositoryProgress - - -class SummaryReport(object): - """ - Node synchronization summary report. - :ivar errors: A list of error messages. - :type errors: list - :ivar repository: A dictionary of RepositoryReport keyed by repo_id. - :type repository: dict - """ - - def __init__(self): - self.errors = ErrorList() - self.repository = {} - - def setup(self, bindings): - """ - Setup (prime) the report using the specified bindings. - A RepositoryReport is created for each repository referenced in the bindings. - :param bindings: - :return: - """ - for bind in bindings: - repo_id = bind['repo_id'] - self.repository[repo_id] = RepositoryReport(repo_id) - - def succeeded(self): - """ - Get whether the update succeeded (or not). - :return: True if succeeded. - :rtype: bool - """ - return not self.failed() - - def failed(self): - """ - Get whether the update failed (or not). - :return: True if failed. - :rtype: bool - """ - return len(self.errors) > 0 - - def dict(self): - """ - Dictionary representation. - :return: A dictionary representation. - :rtype: dict - """ - return dict( - errors=[e.dict() for e in self.errors], - repositories=[r.dict() for r in self.repository.values()]) - - def __getitem__(self, repo_id): - return self.repository[repo_id] - - def __setitem__(self, repo_id, report): - self.repository[repo_id] = report - - -# --- progress reporting ---------------------------------------------------- - - -class HandlerProgress(object): - """ - The nodes handler progress report. - :ivar conduit: A handler conduit. - :type conduit: pulp.agent.lib.conduit.Conduit - :ivar state: The current state of the synchronization. - :type state: str - :ivar progress: A list of RepositoryProgress reports. - :type progress: list - """ - - PENDING = 'pending' - STARTED = 'in-progress' - FINISHED = 'finished' - - def __init__(self, conduit): - """ - :param conduit: A handler conduit. - :type conduit: pulp.agent.lib.conduit.Conduit - """ - self.conduit = conduit - self.state = self.PENDING - self.progress = [] - - def started(self, bindings): - """ - Indicate the handler synchronization has started. - State set to: STARTED. - :param bindings: List of bindings used to populate the report. - :type bindings: list - """ - self.state = self.STARTED - for bind in bindings: - repo_id = bind['repo_id'] - p = RepositoryProgress(repo_id, self) - self.progress.append(p) - self._updated() - - def finished(self): - """ - Indicate the handler synchronization has finished. - State set to: FINISHED. - """ - self.state = self.FINISHED - self._updated() - - def find_report(self, repo_id): - """ - Find a repository report by ID. - :param repo_id: A repository ID. - :type repo_id: str - :return The report if found. - :rtype RepositoryProgress - :raise ValueError - """ - for p in self.progress: - if p.repo_id == repo_id: - return p - raise ValueError(repo_id) - - def updated(self, report): - """ - Update the progress associated with a specific repository by repo_id. - :param report: The update repository progress report. - :type report: RepositoryProgress - """ - for i, p in enumerate(self.progress): - if p.repo_id == report.repo_id: - self.progress[i] = report - self._updated() - break - - def _updated(self): - """ - Notification that the report has been updated. - Reported using the conduit. - """ - self.conduit.update_progress(self.dict()) - - def dict(self): - return dict( - state=self.state, - progress=[r.dict() for r in self.progress] - ) diff --git a/nodes/child/pulp_node/handlers/strategies.py b/nodes/child/pulp_node/handlers/strategies.py deleted file mode 100644 index b88b3de474..0000000000 --- a/nodes/child/pulp_node/handlers/strategies.py +++ /dev/null @@ -1,269 +0,0 @@ -from gettext import gettext as _ -from logging import getLogger -from operator import itemgetter - -from pulp_node import constants -from pulp_node.error import NodeError, CaughtException -from pulp_node.handlers import model -from pulp_node.handlers.validation import Validator -from pulp_node.handlers.reports import RepositoryReport - - -log = getLogger(__name__) - - -STRATEGY_UNSUPPORTED = _('Handler strategy "%(s)s" not supported') - - -class Request(object): - """ - Represents a specific request to synchronize a child node. - It contains the resources needed by the strategy to complete the request - and maintains the state of the request. - :ivar conduit: A handler conduit. - :type conduit: pulp.agent.lib.conduit.Conduit - :ivar progress: A progress report. - :type progress: HandlerProgress - :ivar summary: The summary report. - :type summary: SummaryReport - :ivar bindings: A list of consumer binding payloads. - :type bindings: list - :ivar scope: Specifies the scope of the request (node|repository). - :type scope: str - :ivar options: synchronization options. - :type options: dict - """ - - def __init__(self, conduit, progress, summary, bindings, scope, options): - """ - :param conduit: A handler conduit. - :type conduit: pulp.agent.lib.conduit.Conduit - :param progress: A progress report. - :type progress: HandlerProgress - :param summary: The summary report. - :type summary: SummaryReport - :param bindings: A list of consumer binding payloads. - :type bindings: list - :param scope: The request scope (node|repository) - :type scope: str - :param options: synchronization options. - :type options: dict - """ - self.conduit = conduit - self.progress = progress - self.summary = summary - self.bindings = sorted(bindings, key=itemgetter('repo_id')) - self.scope = scope - self.options = options - summary.setup(self.bindings) - - def cancelled(self): - """ - Get whether the request has been cancelled. - :return: True if cancelled. - :rtype: bool - """ - return self.conduit.cancelled() - - def started(self): - """ - Processing of the request has started. - """ - self.progress.started(self.bindings) - - def finished(self): - """ - Processing of the request has finished. - """ - self.progress.finished() - - -# --- abstract strategy ----------------------------------------------------------------- - - -class HandlerStrategy(object): - """ - Provides strategies for synchronizing repositories between pulp servers. - """ - - def synchronize(self, request): - """ - Synchronize child repositories based on bindings. - Subclasses must not override this method. - """ - request.started() - - try: - # validation - validator = Validator(request.summary) - validator.validate(request.bindings) - if request.summary.failed(): - return - - # synchronization implemented by subclasses - self._synchronize(request) - - # purge orphans - if request.options.get(constants.PURGE_ORPHANS_KEYWORD): - model.Repository.purge_orphans() - except NodeError, ne: - request.summary.errors.append(ne) - except Exception, e: - log.exception('synchronization failed') - error = CaughtException(e) - request.summary.errors.append(error) - finally: - request.finished() - - def _synchronize(self, request): - """ - Specific strategies defined by subclasses. - :param request: A synchronization request. - :type request: SyncRequest - """ - raise NotImplementedError() - - def _merge_repositories(self, request): - """ - Add or update repositories based on bindings. - - Merge repositories found in BOTH parent and child. - - Add repositories found in the parent but NOT in the child. - :param request: A synchronization request. - :type request: SyncRequest - """ - for bind in request.bindings: - try: - repo_id = bind['repo_id'] - details = bind['details'] - if request.cancelled(): - request.summary[repo_id].action = RepositoryReport.CANCELLED - continue - parent = model.Repository(repo_id, details) - child = model.Repository.fetch(repo_id) - progress = request.progress.find_report(repo_id) - progress.begin_merging() - if child: - request.summary[repo_id].action = RepositoryReport.MERGED - child.merge(parent) - else: - child = model.Repository(repo_id, parent.details) - request.summary[repo_id].action = RepositoryReport.ADDED - child.add() - self._synchronize_repository(request, repo_id) - except NodeError, ne: - request.summary.errors.append(ne) - except Exception, e: - log.exception(repo_id) - error = CaughtException(e, repo_id) - request.summary.errors.append(error) - - def _synchronize_repository(self, request, repo_id): - """ - Run synchronization on a repository by ID. - :param request: A synchronization request. - :type request: SyncRequest - :param repo_id: A repository ID. - :type repo_id: str - """ - progress = request.progress.find_report(repo_id) - skip = request.options.get(constants.SKIP_CONTENT_UPDATE_KEYWORD, False) - if skip: - progress.finished() - return - repo = model.Repository(repo_id) - importer_report = repo.run_synchronization(progress, request.cancelled, request.options) - if request.cancelled(): - request.summary[repo_id].action = RepositoryReport.CANCELLED - return - progress.finished() - details = importer_report['details'] - for _dict in details['errors']: - ne = NodeError(None) - ne.load(_dict) - request.summary.errors.append(ne) - _report = request.summary[repo_id] - _report.units.added = importer_report['added_count'] - _report.units.updated = importer_report['updated_count'] - _report.units.removed = importer_report['removed_count'] - _report.sources = details['sources'] - - def _delete_repositories(self, request): - """ - Delete repositories found in the child but NOT in the parent. - :param request: A synchronization request. - :type request: SyncRequest - """ - repositories_on_parent = [b['repo_id'] for b in request.bindings] - repositories_on_child = [r.repo_id for r in model.Repository.fetch_all()] - for repo_id in sorted(repositories_on_child): - if request.cancelled(): - request.summary[repo_id] = RepositoryReport(repo_id, RepositoryReport.CANCELLED) - continue - try: - if repo_id not in repositories_on_parent: - request.summary[repo_id] = RepositoryReport(repo_id, RepositoryReport.DELETED) - repo = model.Repository(repo_id) - repo.delete() - except NodeError, ne: - request.summary.errors.append(ne) - except Exception, e: - log.exception(repo_id) - error = CaughtException(e, repo_id) - request.summary.errors.append(error) - - -class Mirror(HandlerStrategy): - - def _synchronize(self, request): - """ - Synchronize repositories. - - Add/Merge bound repositories as needed. - - Synchronize all bound repositories. - - Purge unbound repositories (scope=node only). - :param request: A synchronization request. - :type request: SyncRequest - """ - self._merge_repositories(request) - if request.scope == constants.NODE_SCOPE: - self._delete_repositories(request) - - -class Additive(HandlerStrategy): - - def _synchronize(self, request): - """ - Synchronize repositories. - - Add/Merge bound repositories as needed. - - Synchronize all bound repositories. - :param request: A synchronization request. - :type request: SyncRequest - """ - self._merge_repositories(request) - - -STRATEGIES = { - constants.MIRROR_STRATEGY: Mirror, - constants.ADDITIVE_STRATEGY: Additive, -} - - -class StrategyUnsupported(Exception): - - def __init__(self, name): - msg = STRATEGY_UNSUPPORTED % {'s': name} - Exception.__init__(self, msg) - - -def find_strategy(name): - """ - Find a strategy (class) by name. - :param name: A strategy name. - :type name: str - :return: A strategy class. - :rtype: callable - :raise: StrategyUnsupported on not found. - """ - try: - return STRATEGIES[name] - except KeyError: - raise StrategyUnsupported(name) diff --git a/nodes/child/pulp_node/handlers/validation.py b/nodes/child/pulp_node/handlers/validation.py deleted file mode 100644 index 13f0516610..0000000000 --- a/nodes/child/pulp_node/handlers/validation.py +++ /dev/null @@ -1,95 +0,0 @@ -import httplib - -from pulp_node import resources -from pulp_node.error import ImporterNotInstalled, DistributorNotInstalled - - -TYPE_ID = 'id' -REPO_ID = 'id' -DISTRIBUTOR_TYPE_ID = 'distributor_type_id' -IMPORTER_TYPE_ID = 'importer_type_id' -IMPORTERS = 'importers' -DISTRIBUTORS = 'distributors' -REPOSITORY = 'repository' -DETAILS = 'details' - - -class Validator(object): - """ - Validate that a child server is in the proper state to be synchronized. - """ - - def __init__(self, report): - """ - :param report: A strategy reported used for error reporting. - :type report: pulp_node.handlers.reports.StrategyReport - :return: - """ - self.report = report - - def validate(self, bindings): - """ - Validate that the child node is suitable for synchronization. - :param bindings: A list of binding payloads. - :type bindings: list - :return: - """ - self.report.errors.extend(self._validate_db_versions()) - self.report.errors.extend(self._validate_plugins(bindings)) - - def _validate_db_versions(self): - """ - Validate that the database versions are compatible. - """ - # Future - return [] - - def _validate_plugins(self, bindings): - """ - Validate that all plugins referenced in the bindings are installed. - :param bindings: A list of binding payloads. - :type bindings: list - """ - errors = [] - child = ChildServer() - for binding in bindings: - details = binding[DETAILS] - repo_id = details[REPOSITORY][REPO_ID] - for plugin in details[IMPORTERS]: - type_id = plugin[IMPORTER_TYPE_ID] - if not child.has_importer(type_id): - errors.append(ImporterNotInstalled(repo_id, type_id)) - for plugin in details[DISTRIBUTORS]: - type_id = plugin[DISTRIBUTOR_TYPE_ID] - if not child.has_distributor(type_id): - errors.append(DistributorNotInstalled(repo_id, type_id)) - return errors - - -class ChildServer(object): - - def __init__(self): - self.importers = self._importers() - self.distributors = self._distributors() - - def has_importer(self, type_id): - return type_id in self.importers - - def has_distributor(self, type_id): - return type_id in self.distributors - - def _importers(self): - bindings = resources.pulp_bindings() - http = bindings.server_info.get_importers() - if http.response_code == httplib.OK: - return set([p[TYPE_ID] for p in http.response_body]) - else: - raise Exception('get importers failed:%d', http.response_code) - - def _distributors(self): - bindings = resources.pulp_bindings() - http = bindings.server_info.get_distributors() - if http.response_code == httplib.OK: - return set([p[TYPE_ID] for p in http.response_body]) - else: - raise Exception('get distributors failed:%d', http.response_code) diff --git a/nodes/child/pulp_node/importers/__init__.py b/nodes/child/pulp_node/importers/__init__.py deleted file mode 100644 index 32466f8d26..0000000000 --- a/nodes/child/pulp_node/importers/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/nodes/child/pulp_node/importers/download.py b/nodes/child/pulp_node/importers/download.py deleted file mode 100644 index 68162de3be..0000000000 --- a/nodes/child/pulp_node/importers/download.py +++ /dev/null @@ -1,108 +0,0 @@ -import os -import tarfile - -from pulp.server.content.sources.event import Listener -from pulp.server.content.sources.model import Request -from pulp_node import constants -from pulp_node import pathlib -from pulp_node.error import UnitDownloadError - - -STORAGE_PATH = constants.STORAGE_PATH -UNIT_REF = 'unit_ref' - - -def untar_dir(path, storage_path): - """ - Replaces the tarball at the specified path with the extracted directory tree. - :param path: The absolute path to a tarball. - :type path: str - :param storage_path: The path into which the content is extracted. - :type storage_path: str - :raise IOError: on i/o errors. - """ - try: - fp = tarfile.open(path) - try: - fp.extractall(path=storage_path) - finally: - fp.close() - finally: - if os.path.exists(path): - os.unlink(path) - - -class ContentDownloadListener(Listener): - """ - The content unit download event listener. - Listens for status changes to unit download requests and calls into the importer - strategy object based on whether the download succeeded or failed. If the download - succeeded, the importer strategy is called to add the associated content unit (in the DB). - """ - - @staticmethod - def create_request(url, destination, unit, unit_ref): - """ - Create a content container download request that is compatible with the listener. - The destination directory is created as needed. - :param url: The download URL. - :type url: str - :param destination: The absolute path to where the file is to be downloaded. - :type destination: str - :param unit: A published content unit. - :type unit: dict - :param unit_ref: A reference to the unit association. - :type unit_ref: pulp_node.manifest.UnitRef. - :return: A download request. - :rtype: Request - """ - data = { - STORAGE_PATH: unit[constants.STORAGE_PATH], - UNIT_REF: unit_ref - } - dir_path = os.path.dirname(destination) - pathlib.mkdir(dir_path) - request = Request(unit[constants.TYPE_ID], unit[constants.UNIT_KEY], url, destination) - request.data = data - return request - - def __init__(self, strategy, request): - """ - :param strategy: An importer strategy object. - :type strategy: pulp_node.importer.strategy.ImporterStrategy. - :param request: The nodes synchronization request. - :type request: pulp_node.importers.strategies.SyncRequest. - """ - super(self.__class__, self).__init__() - self._strategy = strategy - self.request = request - self.error_list = [] - - def on_succeeded(self, request): - """ - A specific download (request) has succeeded. - 1. Fetch the content unit using the reference. - 2. Update the storage_path on the unit. - 3. Add the unit. - 4. Extract downloaded tarballs as needed. - :param request: The download request that succeeded. - :type request: Request - """ - storage_path = request.data[STORAGE_PATH] - unit_ref = request.data[UNIT_REF] - unit = unit_ref.fetch() - unit[constants.STORAGE_PATH] = storage_path - self._strategy.add_unit(self.request, unit) - if unit.get(constants.TARBALL_PATH): - untar_dir(request.destination, storage_path) - - def on_failed(self, request): - """ - A specific download (request) has failed. - Append download request errors to our list of errors. - :param request: The download request that failed. - :type request: Request - """ - for msg in request.errors: - error = UnitDownloadError(request.url, self.request.repo_id, msg) - self.error_list.append(error) diff --git a/nodes/child/pulp_node/importers/http/__init__.py b/nodes/child/pulp_node/importers/http/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/nodes/child/pulp_node/importers/http/importer.py b/nodes/child/pulp_node/importers/http/importer.py deleted file mode 100644 index 8cc7a1c176..0000000000 --- a/nodes/child/pulp_node/importers/http/importer.py +++ /dev/null @@ -1,164 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -from logging import getLogger -from gettext import gettext as _ -from threading import Event - -from nectar.downloaders.threaded import HTTPThreadedDownloader as Downloader - -from pulp.server.compat import json -from pulp.plugins.importer import Importer -from pulp.plugins.util.nectar_config import importer_config_to_nectar_config - -from pulp_node import constants -from pulp_node.error import CaughtException -from pulp_node.reports import RepositoryProgress -from pulp_node.importers.reports import SummaryReport, ProgressListener -from pulp_node.importers.strategies import find_strategy, Request - - -log = getLogger(__name__) - - -# --- constants ------------------------------------------------------------------------- - -PROPERTY_MISSING = _('Missing required configuration property: %(p)s') -STRATEGY_UNSUPPORTED = _('Strategy %(s)s not supported') - -CONFIGURATION_PATH = '/etc/pulp/server/plugins.conf.d/nodes/importer/http.conf' - - -# --- plugin loading -------------------------------------------------------------------- - -def entry_point(): - """ - Entry point that pulp platform uses to load the importer. - :return: importer class and its configuration - :rtype: Importer, dict - """ - with open(CONFIGURATION_PATH) as fp: - return NodesHttpImporter, json.load(fp) - - -# --- plugin ---------------------------------------------------------------------------- - - -class NodesHttpImporter(Importer): - """ - The nodes importer is used to synchronize repository content. - :ivar cancel_event: Event used to signal that the last method called has been - canceled by another thread. - :type cancel_event: Event - """ - - @classmethod - def metadata(cls): - return { - 'id': constants.HTTP_IMPORTER, - 'display_name': 'Pulp Nodes HTTP Importer', - 'types': ['node', 'repository'] - } - - def __init__(self): - self.cancel_event = Event() - - def validate_config(self, repo, config): - """ - Validate the configuration. - :param repo: A repository object. - :type repo: pulp.plugins.model.Repository - :param config: The importer configuration to validate. - :param config: pulp.plugins.config.PluginCallConfiguration - - :return: A tuple of: (is_valid, reason): - is_valid : (bool) True when config if deemed valid. - reason: (str) The reason of the validation failure. - :rtype: tuple - """ - errors = [] - - for key in (constants.MANIFEST_URL_KEYWORD, - constants.STRATEGY_KEYWORD): - value = config.get(key) - if not value: - msg = PROPERTY_MISSING % dict(p=key) - errors.append(msg) - - strategy = config.get(constants.STRATEGY_KEYWORD, constants.DEFAULT_STRATEGY) - if strategy not in constants.STRATEGIES: - msg = STRATEGY_UNSUPPORTED % dict(s=strategy) - errors.append(msg) - - valid = not bool(errors) - return valid, errors - - def sync_repo(self, repo, conduit, config): - """ - Synchronize the content of the specified repository. - The implementation is delegated to the strategy object which - is selected based on the 'strategy' option passed specified in - the configuration. - :param repo: A repository object. - :type repo: pulp.plugins.model.Repository - :param conduit: Provides access to relevant Pulp functionality. - :param config: pulp.server.conduits.repo_sync.RepoSyncConduit - :return: A report describing the result. - :rtype: pulp.server.plugins.model.SyncReport - """ - summary_report = SummaryReport() - downloader = None - - try: - downloader = self._downloader(config) - strategy_name = config.get(constants.STRATEGY_KEYWORD, constants.DEFAULT_STRATEGY) - progress_report = RepositoryProgress(repo.id, ProgressListener(conduit)) - request = Request( - self.cancel_event, - conduit=conduit, - config=config, - downloader=downloader, - progress=progress_report, - summary=summary_report, - repo=repo) - strategy = find_strategy(strategy_name)() - strategy.synchronize(request) - except Exception, e: - summary_report.errors.append(CaughtException(e, repo.id)) - - summary_report.errors.update(repo_id=repo.id) - report = conduit.build_success_report({}, summary_report.dict()) - return report - - def cancel_sync_repo(self, call_request, call_report): - """ - Cancel an in-progress repository synchronization. - :param call_request: - :param call_report: - """ - self.cancel_event.set() - - def _downloader(self, config): - """ - Get a configured downloader. - The integration between the importer configuration and the - download package happens here. The https downloader may be - used for both http and https so always chosen for simplicity. - :param config: The importer configuration. - :param config: pulp.plugins.config.PluginCallConfiguration - :return: A configured downloader - :rtype: nectar.downloaders.base.Downloader - """ - configuration = importer_config_to_nectar_config(config.flatten()) - downloader = Downloader(configuration) - return downloader diff --git a/nodes/child/pulp_node/importers/inventory.py b/nodes/child/pulp_node/importers/inventory.py deleted file mode 100644 index e02d7847a1..0000000000 --- a/nodes/child/pulp_node/importers/inventory.py +++ /dev/null @@ -1,101 +0,0 @@ -from pulp_node import constants - - -class UniqueKey(object): - """ - A unique unit key consisting of a unit's type_id & unit_key. - The unit key is sorted to ensure consistency. - :ivar uid: The unique ID. - :type uid: A tuple of: (type_id, unit_key) - """ - - def __init__(self, unit): - """ - :param unit: A content unit. - :type unit: dict - """ - type_id = unit['type_id'] - unit_key = tuple(sorted(unit['unit_key'].items())) - self.uid = (type_id, unit_key) - - def __hash__(self): - return hash(self.uid) - - def __eq__(self, other): - return self.uid == other.uid - - def __ne__(self, other): - return self.uid != other.uid - - -class UnitInventory(object): - """ - The unit inventory contains both the parent and child inventory - of content units associated with a specific repository. Each is contained - within a dictionary keyed by {UnitKey} to ensure uniqueness. - """ - - @staticmethod - def _import_parent_units(units): - _units = {} - for unit, ref in units: - unit.pop('metadata', None) - key = UniqueKey(unit) - _units[key] = (unit, ref) - return _units - - @staticmethod - def _import_child_units(units): - _units = {} - for unit in units: - unit.pop('metadata', None) - key = UniqueKey(unit) - _units[key] = unit - return _units - - def __init__(self, base_URL, parent_units, child_units): - """ - :param base_URL: The base URL for downloading parent units. - :param parent_units: The content units in the parent node. - :type parent_units: iterable - :param child_units: The content units in the child node. - :type child_units: iterable - """ - self.base_URL = base_URL - self.parent_units = self._import_parent_units(parent_units) - self.child_units = self._import_child_units(child_units) - - def units_on_parent_only(self): - """ - Listing of units contained in the parent inventory - but not contained in the child inventory. - :return: List of (unit, ref). - :rtype: list - """ - return [r for k, r in self.parent_units.items() if k not in self.child_units] - - def units_on_child_only(self): - """ - Listing of units contained in the child inventory - but not contained in the parent inventory. - :return: List of units that need to be purged. - :rtype: list - """ - return [u for k, u in self.child_units.items() if k not in self.parent_units] - - def updated_units(self): - """ - Listing of units updated on the parent. - :return: List of (unit, ref). - :rtype: list - """ - updated = [] - for key, (unit, ref) in self.parent_units.items(): - child_unit = self.child_units.get(key) - if child_unit is None: - continue - parent_last_updated = unit.get(constants.LAST_UPDATED, 0) - child_last_updated = child_unit.get(constants.LAST_UPDATED, 0) - if parent_last_updated > child_last_updated: - updated.append((unit, ref)) - return updated diff --git a/nodes/child/pulp_node/importers/reports.py b/nodes/child/pulp_node/importers/reports.py deleted file mode 100644 index 27268d80b0..0000000000 --- a/nodes/child/pulp_node/importers/reports.py +++ /dev/null @@ -1,61 +0,0 @@ -from pulp.server.content.sources.model import DownloadReport - -from pulp_node.error import ErrorList - - -def key_and_repr(units): - """ - Convert to list of unit_key and exception tuple into a list of - tuple containing the unit_key and string representation of the - exception. This could just be done inline but more descriptive - to wrap in a method. - :param units: List of: (Unit, Exception) - :type units: list - :return: List of: (dict, str) - :rtype: list - """ - return [(u[0].unit_key, repr(u[1])) for u in units] - - -class SummaryReport(object): - """ - A report that provides both summary and details regarding the importing - of content units associated with a repository. - :ivar errors: List of errors. - :type errors: ErrorList - :ivar sources: The content sources container statistics. - :type sources: DownloadReport - """ - - def __init__(self): - self.errors = ErrorList() - self.sources = DownloadReport() - - def dict(self): - """ - Dictionary representation. - :return: A dictionary representation. - :rtype: dict - """ - return dict(errors=[e.dict() for e in self.errors], sources=self.sources.dict()) - - -class ProgressListener(object): - """ - Progress listener provides integration with plugin progress reporting facility. - :ivar conduit: The importer conduit. - :type conduit: pulp.server.conduits.repo_sync.RepoSyncConduit - """ - - def __init__(self, conduit): - """ - :param conduit: The importer conduit. - :type conduit: pulp.server.conduits.repo_sync.RepoSyncConduit - """ - self.conduit = conduit - - def updated(self, report): - """ - Send progress report using the conduit when the report is updated. - """ - self.conduit.set_progress(report.dict()) diff --git a/nodes/child/pulp_node/importers/strategies.py b/nodes/child/pulp_node/importers/strategies.py deleted file mode 100644 index bd53d2a8fd..0000000000 --- a/nodes/child/pulp_node/importers/strategies.py +++ /dev/null @@ -1,436 +0,0 @@ -""" -Provides classes that implement unit synchronization strategies. -Nodes importer plugins delegate synchronization to one of -the strategies provided here. -""" - -import os -import errno - -from gettext import gettext as _ -from logging import getLogger -from urlparse import urlparse, ParseResult - -from pulp.plugins.model import Unit, AssociatedUnit -from pulp.server.config import config as pulp_conf -from pulp.server.content.sources.container import ContentContainer - -from pulp_node import constants -from pulp_node import pathlib -from pulp_node.conduit import NodesConduit -from pulp_node.manifest import Manifest, RemoteManifest -from pulp_node.importers.inventory import UnitInventory -from pulp_node.importers.download import ContentDownloadListener -from pulp_node.error import (NodeError, GetChildUnitsError, GetParentUnitsError, AddUnitError, - DeleteUnitError, InvalidManifestError, CaughtException) - - -_log = getLogger(__name__) - - -STRATEGY_UNSUPPORTED = _('Importer strategy "%(s)s" not supported') - - -class Request(object): - """ - Represents a specific request to synchronize a repository on a child node. - It contains the resources needed by the strategy to complete the request - and maintains the state of the request. - :ivar conduit: Provides access to relevant Pulp functionality. - :type conduit: pulp.server.conduits.repo_sync.RepoSyncConduit - :ivar config: The plugin configuration. - :type config: pulp.server.plugins.config.PluginCallConfiguration - :ivar downloader: A fully configured file downloader. - :type downloader: nectar.downloaders.base.Downloader - :ivar progress: A progress reporting object. - :type progress: pulp_node.importers.reports.RepositoryProgress - :ivar summary: A summary report. - :type summary: pulp_node.importers.reports.SummaryReport - :ivar repo_id: The ID of a repository to synchronize. - :type repo_id: str - :ivar working_dir: The absolute path to a directory to be used as temporary storage. - :type working_dir: str - """ - - def __init__(self, cancel_event, conduit, config, downloader, progress, summary, repo): - """ - :param cancel_event: Event used to signal that the synchronization has been - canceled by another thread. - :type cancel_event: threading.Event - :param conduit: Provides access to relevant Pulp functionality. - :type conduit: pulp.server.conduits.repo_sync.RepoSyncConduit - :param config: The plugin configuration. - :type config: pulp.server.plugins.config.PluginCallConfiguration - :param downloader: A fully configured file downloader. - :type downloader: nectar.downloaders.base.Downloader - :param progress: A progress reporting object. - :type progress: pulp_node.importers.reports.RepositoryProgress - :param summary: A summary report. - :type summary: pulp_node.importers.reports.SummaryReport - :param repo: The repository to synchronize. - :type repo: pulp.server.plugins.model.Repository - """ - self.cancel_event = cancel_event - self.conduit = conduit - self.config = config - self.downloader = downloader - self.progress = progress - self.summary = summary - self.repo_id = repo.id - self.working_dir = repo.working_dir - - def started(self): - """ - Processing the request has started. - """ - self.progress.begin_importing() - - def cancelled(self): - """ - Get whether the request has been cancelled. - :return: True if cancelled. - :rtype: bool - """ - return self.cancel_event.isSet() - - -# --- abstract strategy ---------------------------------------------------------------- - - -class ImporterStrategy(object): - """ - This object provides the transport independent content unit - synchronization strategies used by nodes importer plugins. - """ - - def synchronize(self, request): - """ - Synchronize the content units associated with the specified repository. - :param request: A synchronization request. - :type request: SyncRequest - """ - request.started() - - try: - self._synchronize(request) - except NodeError, ne: - request.summary.errors.append(ne) - except Exception, e: - _log.exception(request.repo_id) - request.summary.errors.append(CaughtException(e, request.repo_id)) - - def _synchronize(self, request): - """ - Specific strategies defined by subclasses. - :param request: A synchronization request. - :type request: SyncRequest - """ - raise NotImplementedError() - - def add_unit(self, request, unit): - """ - Add the specified unit to the child inventory using the conduit. - The conduit will automatically associate the unit to the repository - to which it's pre-configured. - :param request: A synchronization request. - :type request: SyncRequest - :param unit: The unit to be added. - :type unit: dict - """ - try: - new_unit = Unit( - type_id=unit['type_id'], - unit_key=unit['unit_key'], - metadata=unit['metadata'], - storage_path=unit['storage_path']) - request.conduit.save_unit(new_unit) - request.progress.unit_added(details=new_unit.storage_path) - except Exception: - _log.exception(unit['unit_id']) - request.summary.errors.append(AddUnitError(request.repo_id)) - - # --- protected --------------------------------------------------------------------- - - def _unit_inventory(self, request): - """ - Build the unit inventory. - :param request: A synchronization request. - :type request: SyncRequest - :return: The built inventory. - :rtype: UnitInventory - """ - # fetch child units - try: - conduit = NodesConduit() - child_units = conduit.get_units(request.repo_id) - except NodeError: - raise - except Exception: - _log.exception(request.repo_id) - raise GetChildUnitsError(request.repo_id) - - # fetch parent units - try: - request.progress.begin_manifest_download() - url = request.config.get(constants.MANIFEST_URL_KEYWORD) - manifest = Manifest(request.working_dir) - try: - manifest.read() - except IOError, e: - if e.errno == errno.ENOENT: - pass - except ValueError: - # json decoding failed - pass - fetched_manifest = RemoteManifest(url, request.downloader, request.working_dir) - fetched_manifest.fetch() - if manifest != fetched_manifest or \ - not manifest.is_valid() or not manifest.has_valid_units(): - fetched_manifest.write() - fetched_manifest.fetch_units() - manifest = fetched_manifest - if not manifest.is_valid(): - raise InvalidManifestError() - except NodeError: - raise - except Exception: - _log.exception(request.repo_id) - raise GetParentUnitsError(request.repo_id) - - # build the inventory - parent_units = manifest.get_units() - base_URL = manifest.publishing_details[constants.BASE_URL] - inventory = UnitInventory(base_URL, parent_units, child_units) - return inventory - - def _reset_storage_path(self, unit): - """ - Reset the storage_path using the storage_dir defined in - server.conf and the relative_path injected when the unit was published. - :param unit: A published unit. - :type unit: dict - :return: The re-oriented storage path. - :rtype: str - """ - storage_path = unit.get(constants.STORAGE_PATH) - if not storage_path: - return - storage_dir = pulp_conf.get('server', 'storage_dir') - relative_path = unit[constants.RELATIVE_PATH] - storage_path = pathlib.join(storage_dir, relative_path) - unit[constants.STORAGE_PATH] = storage_path - - def _add_units(self, request, unit_inventory): - """ - Determine the list of units contained in the parent inventory - but are not contained in the child inventory and add them. - For each unit, this is performed in the following steps: - 1. Download the file (if defined) associated with the unit. - 2. Add the unit to the child inventory. - 3. Associate the unit to the repository. - The unit is added only: - 1. If no file is associated with unit. - 2. The file associated with the unit is successfully downloaded. - For units with files, the unit is added to the inventory as part of the - unit download manager callback. - :param request: A synchronization request. - :type request: SyncRequest - :param unit_inventory: The inventory of both parent and child content units. - :type unit_inventory: UnitInventory - """ - download_list = [] - units = unit_inventory.units_on_parent_only() - request.progress.begin_adding_units(len(units)) - listener = ContentDownloadListener(self, request) - for unit, unit_ref in units: - if request.cancelled(): - return - self._reset_storage_path(unit) - if not self._needs_download(unit): - # unit has no file associated - self.add_unit(request, unit_ref.fetch()) - continue - unit_url, destination = self._url_and_destination(unit_inventory.base_URL, unit) - _request = listener.create_request(unit_url, destination, unit, unit_ref) - download_list.append(_request) - if request.cancelled(): - return - container = ContentContainer() - request.summary.sources = container.download(request.downloader, download_list, listener) - request.summary.errors.extend(listener.error_list) - - def _update_units(self, request, unit_inventory): - """ - Update units that have been updated on the parent since - added or last updated in the child inventory. - :param request: A synchronization request. - :type request: SyncRequest - :param unit_inventory: The inventory of both parent and child content units. - :type unit_inventory: UnitInventory - """ - download_list = [] - units = unit_inventory.updated_units() - listener = ContentDownloadListener(self, request) - for unit, unit_ref in units: - storage_path = unit[constants.STORAGE_PATH] - if storage_path: - self._reset_storage_path(unit) - unit_url, destination = self._url_and_destination(unit_inventory.base_URL, unit) - _request = listener.create_request(unit_url, destination, unit, unit_ref) - download_list.append(_request) - else: - unit = unit_ref.fetch() - self.add_unit(request, unit) - if not download_list: - return - container = ContentContainer() - request.summary.sources = container.download( - request.downloader, - download_list, - listener) - request.summary.errors.extend(listener.error_list) - - def _url_and_destination(self, base_url, unit): - """ - Get the download URL and download destination. - :param base_url: The base URL. - :type base_url: str - :param unit: A content unit. - :type unit: dict - :return: (url, destination) - :rtype: tuple(2) - """ - storage_path = unit[constants.STORAGE_PATH] - tar_path = unit.get(constants.TARBALL_PATH) - if not tar_path: - # The pulp/nodes/content endpoint provides all content. - # This replaced the publishing of individual links for each unit. - parsed = urlparse(base_url) - relative_path = unit[constants.RELATIVE_PATH] - path = pathlib.join(constants.CONTENT_PATH, pathlib.quote(relative_path)) - base_url = ParseResult( - scheme=parsed.scheme, - netloc=parsed.netloc, - path=path, - params=parsed.params, - query=parsed.query, - fragment=parsed.fragment) - return base_url.geturl(), storage_path - else: - return pathlib.url_join(base_url, pathlib.quote(tar_path)),\ - pathlib.join(os.path.dirname(storage_path), os.path.basename(tar_path)) - - def _needs_download(self, unit): - """ - Get whether the unit has an associated file that needs to be downloaded. - :param unit: A content unit. - :type unit: dict - :return: True if has associated file that needs to be downloaded. - :rtype: bool - """ - storage_path = unit.get(constants.STORAGE_PATH) - if storage_path: - if not os.path.exists(storage_path): - return True - if os.path.getsize(storage_path) != unit[constants.FILE_SIZE]: - return True - return False - - def _delete_units(self, request, unit_inventory): - """ - Determine the list of units contained in the child inventory - but are not contained in the parent inventory and un-associate them. - :param request: A synchronization request. - :type request: SyncRequest - :param unit_inventory: The inventory of both parent and child content units. - :type unit_inventory: UnitInventory - """ - for unit in unit_inventory.units_on_child_only(): - if request.cancelled(): - return - try: - _unit = AssociatedUnit( - type_id=unit['type_id'], - unit_key=unit['unit_key'], - metadata={}, - storage_path=None, - created=None, - updated=None) - _unit.id = unit['unit_id'] - request.conduit.remove_unit(_unit) - except Exception: - _log.exception(unit['unit_id']) - request.summary.errors.append(DeleteUnitError(request.repo_id)) - - -class Mirror(ImporterStrategy): - """ - The *mirror* strategy is used to ensure that the content units associated - with a child repository exactly matches the units associated with the same - repository in the parent. Maintains an exact mirror. - """ - - def _synchronize(self, request): - """ - Performs the following steps: - 1. Read the (parent) manifest. - 2. Fetch the child units associated with the repository. - 3. Add missing units. - 4. Delete units specified in the child but not in the parent. - :param request: A synchronization request. - :type request: SyncRequest - """ - unit_inventory = self._unit_inventory(request) - self._add_units(request, unit_inventory) - self._update_units(request, unit_inventory) - self._delete_units(request, unit_inventory) - - -class Additive(ImporterStrategy): - """ - The I{additive} strategy is used to ensure that the content units associated - with a child repository contains all of the units associated with the same - repository in the parent. However, any units contained in the child inventory - that are not contained in the parent inventory are permitted to remain. - """ - - def _synchronize(self, request): - """ - Performs the following steps: - 1. Read the (parent) manifest. - 2. Fetch the child units associated with the repository. - 3. Add missing units. - :param request: A synchronization request. - :type request: SyncRequest - """ - unit_inventory = self._unit_inventory(request) - self._add_units(request, unit_inventory) - self._update_units(request, unit_inventory) - - -STRATEGIES = { - constants.MIRROR_STRATEGY: Mirror, - constants.ADDITIVE_STRATEGY: Additive, -} - - -class StrategyUnsupported(Exception): - - def __init__(self, name): - msg = STRATEGY_UNSUPPORTED % {'s': name} - Exception.__init__(self, msg) - - -def find_strategy(name): - """ - Find a strategy (class) by name. - :param name: A strategy name. - :type name: str - :return: A strategy class. - :rtype: callable - :raise: StrategyUnsupported when not found. - """ - try: - return STRATEGIES[name] - except KeyError: - raise StrategyUnsupported(name) diff --git a/nodes/child/pulp_node/importers/types/nodes.json b/nodes/child/pulp_node/importers/types/nodes.json deleted file mode 100644 index 8414817096..0000000000 --- a/nodes/child/pulp_node/importers/types/nodes.json +++ /dev/null @@ -1,16 +0,0 @@ -{"types":[ - { - "id":"repository", - "display_name":"Repository", - "description":"A Pulp Repository", - "unit_key":["repo_id"], - "search_indexes":["repo_id"] - }, - { - "id":"node", - "display_name":"Pulp Node", - "description":"A Pulp Node", - "unit_key":["node_id"], - "search_indexes":["node_id"] - } -]} \ No newline at end of file diff --git a/nodes/child/setup.py b/nodes/child/setup.py deleted file mode 100755 index fc83dd1674..0000000000 --- a/nodes/child/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='pulp_node_child', - version='2.11a1', - license='GPLv2+', - packages=find_packages(), - author='Pulp Team', - author_email='pulp-list@redhat.com', - entry_points={ - 'pulp.importers': [ - 'importer = pulp_node.importers.http.importer:entry_point', - ], - }, -) diff --git a/nodes/common/__init__.py b/nodes/common/__init__.py deleted file mode 100644 index 32466f8d26..0000000000 --- a/nodes/common/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/nodes/common/bin/pulp-gen-nodes-certificate b/nodes/common/bin/pulp-gen-nodes-certificate deleted file mode 100755 index 8a9eb412b6..0000000000 --- a/nodes/common/bin/pulp-gen-nodes-certificate +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public License, -# version 2 (GPLv2). There is NO WARRANTY for this software, express or -# implied, including the implied warranties of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 -# along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. -# -# Red Hat trademarks are not licensed under GPLv2. No permission is -# granted to use or replicate Red Hat trademarks that are incorporated -# in this software or its documentation. -# -set -e - -READ_PULP_CONF=\ -$(cat << END -from pulp.server.config import config as pulp_conf -print pulp_conf.get('security', 'cakey') -print pulp_conf.get('security', 'cacert') -END -) - -READ_NODE_CONF=\ -$(cat << END -from pulp_node.config import read_config -node_conf = read_config() -print node_conf.main.node_certificate -END -) - -PULP_CONF=(`python -c "$READ_PULP_CONF"`) -NODE_CONF=(`python -c "$READ_NODE_CONF"`) - - -NODE_CRT=${NODE_CONF[0]} -CA_KEY=${PULP_CONF[0]} -CA_CRT=${PULP_CONF[1]} -BASE='nodes' -# This temporary folder will contain the certificate signing request. -TMP="$(mktemp -d)" -CN=`hostname` -ORG="PULP" -ORG_UNIT="NODES" - -mkdir -p `dirname $NODE_CRT` - -# create client key -OLD_UMASK="$(umask)" -umask 0027 -openssl genrsa -out $NODE_CRT 4096 &> /dev/null -chgrp apache $NODE_CRT -umask $OLD_UMASK - -# create signing request for client -openssl req \ - -new \ - -key $NODE_CRT \ - -out $TMP/$BASE.req \ - -subj "/CN=$CN/O=$ORG/OU=$ORG_UNIT" &> /dev/null - -# sign server request w/ CA key and gen x.509 cert. -openssl x509 \ - -req \ - -in $TMP/$BASE.req \ - -out $TMP/$BASE.crt \ - -sha1 \ - -CA $CA_CRT \ - -CAkey $CA_KEY \ - -CAcreateserial \ - -set_serial $RANDOM \ - -days 3650 &> /dev/null - -# bundle -cat $TMP/$BASE.crt >> $NODE_CRT - -# clean -rm $TMP/$BASE.req -rm $TMP/$BASE.crt -rmdir $TMP diff --git a/nodes/common/etc/pulp/nodes.conf b/nodes/common/etc/pulp/nodes.conf deleted file mode 100644 index 01f0aaa970..0000000000 --- a/nodes/common/etc/pulp/nodes.conf +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt - -# The settings in this file are all commented by default, and the commented settings show the -# default values that a Pulp Node will choose if not specified here. - -# The main node configuration -# ca_path - This is a path to a file of concatenated trusted CA certificates, or to a -# directory of trusted CA certificates (with openssl-style hashed symlinks, one -# certificate per file). -# node_certificate - The absolute path to the node SSL certificate -# verify_ssl - Set this to False to configure Nodes not to verify that the Pulp server's SSL -# cert is signed by a trusted authority. -# WARNING: Setting this to False is not safe if you wish to use Pulp with real -# passwords or any other private information. It is strongly recommended -# to keep this set to True, and to use certificates that are signed by a -# trusted authority on the web server that serves Pulp. - -[main] -# ca_path: /etc/pki/tls/certs/ca-bundle.crt -# node_certificate: /etc/pki/pulp/nodes/node.crt -# verify_ssl: True - - -# The oauth configuration used to authenticate to this node -# user_id - The pulp user ID - -[oauth] -# user_id: admin - - -# The oauth configuration used to authenticate to the parent node -# key - The oauth key -# secret - The oauth secret -# user_id - The pulp user ID - -[parent_oauth] -# key: -# secret: -# user_id: admin diff --git a/nodes/common/pulp_node/__init__.py b/nodes/common/pulp_node/__init__.py deleted file mode 100644 index b36383a610..0000000000 --- a/nodes/common/pulp_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/nodes/common/pulp_node/conduit.py b/nodes/common/pulp_node/conduit.py deleted file mode 100644 index fd2dbeed2a..0000000000 --- a/nodes/common/pulp_node/conduit.py +++ /dev/null @@ -1,120 +0,0 @@ -from pulp.plugins.types.database import type_units_collection -from pulp.plugins.util.misc import paginate -from pulp.server.controllers.units import get_unit_key_fields_for_type -from pulp.server.db.model.repository import RepoContentUnit -from pulp.server.config import config as pulp_conf - - -class NodesConduit(object): - - @staticmethod - def get_units(repo_id): - """ - Get all units associated with a repository. - :param repo_id: The repository ID used to query the units. - :type repo_id: str - :return: unit iterator - :rtype: UnitsIterator - """ - unit_ids = {} - associations = {} - collection = RepoContentUnit.get_collection() - for association in collection.find({'repo_id': repo_id}): - unit_id = association['unit_id'] - type_id = association['unit_type_id'] - associations[unit_id] = association - id_list = unit_ids.setdefault(type_id, []) - id_list.append(unit_id) - return UnitsIterator(associations, unit_ids) - - -class UnitsIterator(object): - """ - Provides a memory efficient iterator of associated content units. - """ - - @staticmethod - def associated_unit(association, unit): - """ - Create a dictionary that is a composite of a unit association and the unit. - - :param association: A unit association DB record. - :type association: dict - :param unit: A DB unit record. - :type unit: dict - :return: A composite of the unit association and the unit. - :rtype: dict - """ - unit_key = {} - unit_id = unit.pop('_id') - type_id = association['unit_type_id'] - for key in get_unit_key_fields_for_type(type_id): - unit_key[key] = unit.pop(key, None) - storage_dir = pulp_conf.get('server', 'storage_dir') - storage_path = unit.pop('_storage_path', None) - last_updated = unit.pop('_last_updated', 0.0) - if storage_path: - relative_path = storage_path[len(storage_dir):].lstrip('/') - else: - relative_path = None - return dict( - unit_id=unit_id, - type_id=type_id, - unit_key=unit_key, - storage_path=storage_path, - relative_path=relative_path, - last_updated=last_updated, - metadata=unit) - - @staticmethod - def open_cursors(unit_ids): - """ - Get a generator of unit cursors. - - :param unit_ids: A dictionary of unit_ids keyed by type_id. - :type unit_ids: dict - :return: A list of open cursors. - :rtype: generator - """ - for type_id, id_list in unit_ids.items(): - for page in paginate(id_list): - query = {'_id': {'$in': page}} - collection = type_units_collection(type_id) - cursor = collection.find(query) - yield cursor - - def get_units(self, associations, unit_ids): - """ - Get units generator. - - :param associations: A dictionary of unit associates keyed by type_id. - :type associations: dict - :param unit_ids: A dictionary of unit_ids keyed by type_id. - :type unit_ids: dict - :return: A composite association and unit. - :rtype: generator - """ - for cursor in UnitsIterator.open_cursors(unit_ids): - for unit in cursor: - unit_id = unit['_id'] - association = associations[unit_id] - yield self.associated_unit(association, unit) - - def __init__(self, associations, unit_ids): - """ - :param associations: A dictionary of unit associates keyed by type_id. - :type associations: dict - :param unit_ids: A dictionary of unit_ids keyed by type_id. - :type unit_ids: dict - """ - self.length = len(associations) - self.unit_generator = self.get_units(associations, unit_ids) - - def next(self): - return self.unit_generator.next() - - def __iter__(self): - return self - - def __len__(self): - return self.length diff --git a/nodes/common/pulp_node/config.py b/nodes/common/pulp_node/config.py deleted file mode 100644 index 5edd221eb9..0000000000 --- a/nodes/common/pulp_node/config.py +++ /dev/null @@ -1,49 +0,0 @@ -from pulp.common.config import ANY, BOOL, Config, REQUIRED - - -NODE_CONFIGURATION_PATH = '/etc/pulp/nodes.conf' - -DEFAULT = { - 'main': { - 'ca_path': '/etc/pki/tls/certs/ca-bundle.crt', - 'node_certificate': '/etc/pki/pulp/nodes/node.crt', - 'verify_ssl': 'true', - }, - 'oauth': { - 'user_id': 'admin', - }, - 'parent_oauth': { - 'key': '', - 'secret': '', - 'user_id': 'admin', - }, -} - -SCHEMA = ( - ('main', REQUIRED, - (('ca_path', REQUIRED, ANY), - ('node_certificate', REQUIRED, ANY), - ('verify_ssl', REQUIRED, BOOL))), - ('oauth', REQUIRED, - (('user_id', REQUIRED, ANY),)), - ('parent_oauth', REQUIRED, - (('key', REQUIRED, ANY), - ('secret', REQUIRED, ANY), - ('user_id', REQUIRED, ANY),)), -) - - -def read_config(path=NODE_CONFIGURATION_PATH, validate=True): - """ - Get the node configuration object. - The node configuration is overridden using values from the pulp - consumer.conf and defaulted using server.conf as appropriate. - :param path: The optional path to the configuration. - :return: The configuration object. - :rtype: pulp.common.config.Graph - """ - config = Config(DEFAULT) - config.update(Config(path)) - if validate: - config.validate(SCHEMA) - return config.graph() diff --git a/nodes/common/pulp_node/constants.py b/nodes/common/pulp_node/constants.py deleted file mode 100644 index 136834a50a..0000000000 --- a/nodes/common/pulp_node/constants.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - - -# --- plugins ---------------------------------------------------------------- - - -HTTP_DISTRIBUTOR = 'nodes_http_distributor' -HTTP_IMPORTER = 'nodes_http_importer' - -ALL_IMPORTERS = [HTTP_IMPORTER] -ALL_DISTRIBUTORS = [HTTP_DISTRIBUTOR] - - -# --- strategies ------------------------------------------------------------- - -ADDITIVE_STRATEGY = 'additive' -MIRROR_STRATEGY = 'mirror' -STRATEGIES = [ADDITIVE_STRATEGY, MIRROR_STRATEGY] -DEFAULT_STRATEGY = ADDITIVE_STRATEGY - -NODE_SCOPE = 'node' -REPOSITORY_SCOPE = 'repository' -SCOPES = [NODE_SCOPE, REPOSITORY_SCOPE] - - -# --- keywords --------------------------------------------------------------- - -STRATEGY_KEYWORD = 'strategy' -PROTOCOL_KEYWORD = 'protocol' -MANIFEST_URL_KEYWORD = 'manifest_url' -PURGE_ORPHANS_KEYWORD = 'purge_orphans' - -MAX_DOWNLOAD_BANDWIDTH_KEYWORD = 'max_download_bandwidth' -MAX_DOWNLOAD_CONCURRENCY_KEYWORD = 'max_download_concurrency' - -SKIP_CONTENT_UPDATE_KEYWORD = 'skip_content_update' - - -# --- unit/publishing -------------------------------------------------------- - -TYPE_ID = 'type_id' -UNIT_KEY = 'unit_key' -BASE_URL = 'base_url' -STORAGE_PATH = 'storage_path' -RELATIVE_PATH = 'relative_path' -FILE_SIZE = 'size' -TARBALL_PATH = 'tgz_path' -LAST_UPDATED = 'last_updated' - -# The URL endpoint linked to /var/lib/pulp/content. -# This replaced publishing links to each unit file individually. -CONTENT_PATH = 'pulp/nodes/' - - -# --- consumer notes --------------------------------------------------------- - -NODE_NOTE_KEY = '_child-node' -STRATEGY_NOTE_KEY = '_node-update-strategy' - - -# --- settings --------------------------------------------------------------- - -DEFAULT_DOWNLOAD_CONCURRENCY = 20 - - -# --- profiling -------------------------------------------------------------- - -PROFILER_ID = 'node' -PARENT_SETTINGS = 'parent_settings' -NODE_CERTIFICATE = 'node_certificate' -HOST = 'host' -PORT = 'port' diff --git a/nodes/common/pulp_node/error.py b/nodes/common/pulp_node/error.py deleted file mode 100644 index 65aef236eb..0000000000 --- a/nodes/common/pulp_node/error.py +++ /dev/null @@ -1,222 +0,0 @@ -from gettext import gettext as _ - - -class NodeError(Exception): - - def __init__(self, error_id, **details): - self.error_id = error_id - self.details = details - - def load(self, _dict): - if isinstance(_dict, dict): - self.__dict__.update(_dict) - else: - raise ValueError(_dict) - - def dict(self): - return self.__dict__ - - def __eq__(self, other): - return self.error_id == other.error_id and self.details == other.details - - -class CaughtException(NodeError): - - ERROR_ID = 'exception' - DESCRIPTION = _('An unexpected error occurred. repo_id=%(repo_id)s') - - def __init__(self, exception, repo_id=None): - super(CaughtException, self).__init__( - self.ERROR_ID, message=str(exception), repo_id=repo_id) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class PurgeOrphansError(NodeError): - - ERROR_ID = 'rest.child.orphans.purge' - DESCRIPTION = _('Purge orphans failed with http code [%(http_code)s].') - - def __init__(self, http_code): - super(PurgeOrphansError, self).__init__(self.ERROR_ID, http_code=http_code) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class RepoSyncRestError(NodeError): - - ERROR_ID = 'rest.child.repository.synchronization' - DESCRIPTION = _('Repository synchronization failed with http code [%(http_code)s].') - - def __init__(self, repo_id, http_code): - super(RepoSyncRestError, self).__init__(self.ERROR_ID, repo_id=repo_id, http_code=http_code) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class GetBindingsError(NodeError): - - ERROR_ID = 'rest.parent.bindings.get' - DESCRIPTION = _('Get bindings from the parent failed with http code [%(http_code)s].') - - def __init__(self, http_code): - super(GetBindingsError, self).__init__(self.ERROR_ID, http_code=http_code) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class GetChildUnitsError(NodeError): - - ERROR_ID = 'rest.child.units.get' - DESCRIPTION = _('Get units in repository [%(repo_id)s] from the child failed.') - - def __init__(self, repo_id): - super(GetChildUnitsError, self).__init__(self.ERROR_ID, repo_id=repo_id) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class GetParentUnitsError(NodeError): - - ERROR_ID = 'rest.parent.units.get' - DESCRIPTION = _('An error occurred while downloading units from the parent for ' - 'repository [%(repo_id)s. The cause may be that the repository has not ' - 'been published') - - def __init__(self, repo_id): - super(GetParentUnitsError, self).__init__(self.ERROR_ID, repo_id=repo_id) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class ImporterNotInstalled(NodeError): - - ERROR_ID = 'plugins.child.importer.missing' - DESCRIPTION = _('The [%(type_id)s] importer is associated with a repository [%(repo_id)s] ' - 'on the parent but is not installed on the child. The plugin providing this ' - 'importer needs to be installed and loaded by restarting httpd.') - - def __init__(self, repo_id, type_id): - super(ImporterNotInstalled, self).__init__(self.ERROR_ID, repo_id=repo_id, type_id=type_id) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class DistributorNotInstalled(NodeError): - - ERROR_ID = 'plugins.child.distributor.missing' - DESCRIPTION = _('The [%(type_id)s] distributor is associated with a repository [%(repo_id)s] ' - 'on the parent but is not installed on the child. The plugin providing this ' - 'distributor needs to be installed and loaded by restarting httpd.') - - def __init__(self, repo_id, type_id): - super(DistributorNotInstalled, self).__init__(self.ERROR_ID, repo_id=repo_id, - type_id=type_id) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class ManifestDownloadError(NodeError): - - ERROR_ID = 'download.parent.manifest' - DESCRIPTION = _('Received error [%(message)s] while downloading the manifest at ' - 'URL [%(url)s]. The cause could be that the repository has not been published.') - - def __init__(self, url, message): - super(ManifestDownloadError, self).__init__(self.ERROR_ID, url=url, message=message) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class InvalidManifestError(NodeError): - - ERROR_ID = 'manifest.not-valid' - DESCRIPTION = _('Published manifest not valid . ' - 'The cause is most likely a pulp-nodes software version mismatch between the ' - 'parent and child nodes. Or, the software has been updated and the repository' - '(manifest) needs to be republished.') - - def __init__(self): - super(InvalidManifestError, self).__init__(self.ERROR_ID) - - def __str__(self): - return self.DESCRIPTION - - -class UnitDownloadError(NodeError): - - ERROR_ID = 'download.unit' - DESCRIPTION = _('Received error [%(message)s] while downloading a unit file at ' - 'URL [%(url)s] for repository [%(repo_id)s]. The cause could be that the ' - 'repository has not been published.') - - def __init__(self, url, repo_id, message): - super(UnitDownloadError, self).__init__( - self.ERROR_ID, url=url, repo_id=repo_id, message=message) - - -class AddUnitError(NodeError): - - ERROR_ID = 'child.unit.add' - DESCRIPTION = _('Adding a unit associated with repository [%(repo_id)s] failed.') - - def __init__(self, repo_id): - super(AddUnitError, self).__init__(self.ERROR_ID, repo_id=repo_id) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class DeleteUnitError(NodeError): - - ERROR_ID = 'child.unit.delete' - DESCRIPTION = _('Deleting a unit associated with repository [%(repo_id)s] failed.') - - def __init__(self, repo_id): - super(DeleteUnitError, self).__init__(self.ERROR_ID, repo_id=repo_id) - - def __str__(self): - return self.DESCRIPTION % self.details - - -class ErrorList(list): - - def append(self, error): - """ - Append the error. - - Duplicates ignored. - - Increment the 'count' on Summary Errors. - :param error: An error to append. - :type error: NodeError - """ - if not isinstance(error, NodeError): - raise ValueError(error) - if error not in self: - super(ErrorList, self).append(error) - - def extend(self, iterable): - """ - Extend the list using append(). - :param iterable: An iterable containing errors. - :type iterable: iterable - """ - for e in iterable: - self.append(e) - - def update(self, **details): - """ - Update the details of all contained errors. - :param details: A details dictionary. - :type details: dict - """ - for e in self: - e.details.update(details) diff --git a/nodes/common/pulp_node/extension.py b/nodes/common/pulp_node/extension.py deleted file mode 100644 index c78dcde289..0000000000 --- a/nodes/common/pulp_node/extension.py +++ /dev/null @@ -1,73 +0,0 @@ -from gettext import gettext as _ - -from pulp.bindings.exceptions import NotFoundException - -from pulp_node import constants - - -SECTION_NAME = 'node' -SECTION_DESCRIPTION = _('pulp nodes related commands') - - -def ensure_node_section(cli): - """ - Ensures that the root section of node-related commands exists in the CLI, - creating it using constants from this module if it does not. - :param cli: CLI instance being configured - :type cli: pulp.client.extensions.core.PulpCli - """ - section = cli.find_section(SECTION_NAME) - if section is None: - section = cli.create_section(SECTION_NAME, SECTION_DESCRIPTION) - return section - - -def node_activated(context, node_id): - """ - Get whether a node has been activated. - :param context: A client context. - :type context: pulp.client.extensions.core.ClientContext - :param node_id: The ID of the node being checked. - :param node_id: str - :return: True if activated. - :rtype: bool - """ - try: - http = context.server.consumer.consumer(node_id) - consumer = http.response_body - notes = consumer['notes'] - return notes.get(constants.NODE_NOTE_KEY, False) - except NotFoundException: - return False - - -def repository_enabled(context, repo_id): - """ - Get whether a repository is enabled. - :param context: A client context. - :type context: pulp.client.extensions.core.ClientContext - :param repo_id: The ID of the repository being checked. - :param repo_id: str - :return: True if enabled. - :rtype: bool - """ - try: - http = context.server.repo_distributor.distributors(repo_id) - for dist in http.response_body: - if dist['distributor_type_id'] in constants.ALL_DISTRIBUTORS: - return True - return False - except NotFoundException: - return False - - -def missing_resources(exception): - """ - Generator use to get missing resources specified by an exception. - :param exception: A NotFoundException exception. - :type exception: pulp.bindings.exceptions.NotFoundException - :return: generator of: (id, type) - :rtype: tuple - """ - for _type, _id in exception.extra_data['resources'].items(): - yield _id, _type diff --git a/nodes/common/pulp_node/manifest.py b/nodes/common/pulp_node/manifest.py deleted file mode 100644 index 7d052bab4b..0000000000 --- a/nodes/common/pulp_node/manifest.py +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -""" -Provides classes for managing the content unit manifest. -The manifest is a json encoded file that defines content units -associated with repository. The units themselves are stored in a separate -json encoded file. For performance reasons, the unit files are compressed. -""" - -import os -import gzip -import errno - -from logging import getLogger - -from nectar.request import DownloadRequest -from nectar.listener import AggregatingEventListener - -from pulp.server.compat import json - -from pulp_node import pathlib -from pulp_node.error import ManifestDownloadError - - -log = getLogger(__name__) - - -# --- constants ------------------------------------------------------------------------- - -MANIFEST_VERSION = 2 -MANIFEST_FILE_NAME = 'manifest.json' -UNITS_FILE_NAME = 'units.json.gz' - -ID = 'id' -VERSION = 'version' -PUBLISHING_DETAILS = 'publishing_details' -PATH = 'path' -UNITS = 'units' -UNITS_PATH = 'path' -UNITS_TOTAL = 'total' -UNITS_SIZE = 'size' - - -# --- utils ----------------------------------------------------------------------------- - -def unzip(path, destination, bfrlen=65535): - """ - Unzip the file at the specified path. - :param path: The path to the file to be unzipped. - :type path: str - :param destination: The destination path. - :type destination: str - :param bfrlen: The buffer size in bytes. - :type bfrlen: int - :raise IOError: on any i/o error. - """ - fp_in = gzip.open(path) - try: - with open(destination, 'w+') as fp_out: - while True: - bfr = fp_in.read(bfrlen) - if bfr: - fp_out.write(bfr) - else: - break - finally: - fp_in.close() - - -# --- manifest -------------------------------------------------------------------------- - - -class Manifest(object): - """ - Provides structured access to the information contained within the manifest - and access to the content units associated with the manifest through - and iterator to ensure a small memory footprint. - :ivar id: The unique manifest ID. - :type id: str - :ivar total_units: The number of units in the units file. - :type total_units: int - :param publishing_details: Details of how units have been published. - :type publishing_details: dict - """ - - def __init__(self, path, manifest_id=None): - """ - :param path: The path to either a directory containing the standard - named manifest or the absolute path to the manifest. - :type path: str - :param manifest_id: An optional manifest ID. - :type manifest_id: str - """ - self.id = manifest_id - self.version = MANIFEST_VERSION - self.units = {UNITS_PATH: None, UNITS_TOTAL: 0, UNITS_SIZE: 0} - self.publishing_details = {} - if os.path.isdir(path): - path = pathlib.join(path, MANIFEST_FILE_NAME) - self.path = path - - def write(self): - """ - Write the manifest to a json encoded file at the specified path. - :raise IOError: on I/O errors. - :raise ValueError: on json encoding errors - """ - state = { - ID: self.id, - VERSION: self.version, - UNITS: self.units, - PUBLISHING_DETAILS: self.publishing_details - } - with open(self.path, 'w+') as fp: - json.dump(state, fp, indent=2) - - def read(self, path=None): - """ - Read the manifest file at the specified path. - The manifest is updated using the contents of the read json document. - :param path: An optional path to the manifest. - :type path: str - :raise IOError: on I/O errors. - :raise ValueError: on json decoding errors - """ - with open(path or self.path) as fp: - d = json.load(fp) - self.id = d.get(ID) - self.version = d.get(VERSION, 0) - self.units = d.get(UNITS, {UNITS_PATH: None, UNITS_TOTAL: 0, UNITS_SIZE: 0}) - self.publishing_details = d.get(PUBLISHING_DETAILS, {}) - - def get_units(self): - """ - Get the content units referenced in the manifest. - :return: An iterator used to read downloaded content units. - :rtype: iterable - :raise IOError: on I/O errors. - :raise ValueError: json decoding errors - """ - total = self.units[UNITS_TOTAL] - if total: - path = self.units_path() - path = self.unzip_units(path) - return UnitIterator(path, total) - else: - return [] - - def units_published(self, unit_writer): - """ - Update the manifest publishing information. - :param unit_writer: A writer used to publish the units. - :type unit_writer: UnitWriter - """ - self.units[UNITS_TOTAL] = unit_writer.total_units - self.units[UNITS_SIZE] = unit_writer.bytes_written - - def published(self, details): - """ - Update the publishing details. - :param details: Publishing details. - :type details: dict - """ - self.publishing_details.update(details) - - def is_valid(self): - """ - Get whether the manifest is valid. - To start with, compare the version number. - :return: True if valid. - :rtype: bool - """ - try: - return self.version == MANIFEST_VERSION - except AttributeError: - return False - - def has_valid_units(self): - """ - Validate the associated units file by comparing the size of the - units file to units_size in the manifest. - :return: True if valid. - :rtype: bool - """ - try: - path = self.units_path() - size = os.path.getsize(path) - return size == self.units[UNITS_SIZE] - except OSError, e: - if e.errno != errno.ENOENT: - raise - return False - - def unzip_units(self, path): - """ - Uncompress the unit file at the specified path and update - the path and size stored in the manifest. - :param path: The path to a units file. - :type path: str - :return: The path to the unzipped file. - :rtype:str - :raise IOError: on any i/o error. - """ - if not path.endswith('.gz'): - return path - if not self.has_valid_units(): - return path - destination = path[:-3] - unzip(path, destination) - self.units[UNITS_PATH] = destination - self.units[UNITS_SIZE] = os.path.getsize(destination) - os.unlink(path) - self.write() - return destination - - def units_path(self): - """ - Get the absolute path to the associated units file. - The path is - """ - return self.units[UNITS_PATH] or pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME) - - def __eq__(self, other): - if isinstance(other, Manifest): - return self.id == other.id - else: - return False - - def __ne__(self, other): - return not self == other - - -class RemoteManifest(Manifest): - """ - Represents a remote manifest. - """ - - def __init__(self, url, downloader, path): - """ - :param url: The URL to the remote manifest. - :type url: str - :param downloader: The downloader used for fetch methods. - :type downloader: nectar.downloaders.base.Downloader - :param path: An absolute path to a file or directory. - :type path: str - """ - Manifest.__init__(self, path) - self.url = str(url) - self.downloader = downloader - - def fetch(self): - """ - Fetch the manifest file using the specified URL. - :raise ManifestDownloadError: on downloading errors. - :raise HTTPError: on URL errors. - :raise ValueError: on json decoding errors - """ - listener = AggregatingEventListener() - destination = os.path.join(os.path.dirname(self.path), '.' + MANIFEST_FILE_NAME) - request = DownloadRequest(self.url, destination) - self.downloader.event_listener = listener - self.downloader.download([request]) - if listener.failed_reports: - report = listener.failed_reports[0] - raise ManifestDownloadError(self.url, report.error_msg) - self.read(destination) - - def fetch_units(self): - """ - Fetch the units file referenced in the manifest. - :raise ManifestDownloadError: on downloading errors. - :raise HTTPError: on URL errors. - :raise ValueError: on json decoding errors - """ - base_url = self.url.rsplit('/', 1)[0] - url = pathlib.join(base_url, UNITS_FILE_NAME) - destination = pathlib.join(os.path.dirname(self.path), UNITS_FILE_NAME) - request = DownloadRequest(str(url), destination) - listener = AggregatingEventListener() - self.downloader.event_listener = listener - self.downloader.download([request]) - if listener.failed_reports: - report = listener.failed_reports[0] - raise ManifestDownloadError(self.url, report.error_msg) - - -class UnitWriter(object): - """ - Writes json encoded content units to a file. - This approach is 30x faster than opening, appending, and closing for each unit. - :ivar path: The absolute path to a file or directory. When a directory is specified, - the standard file name is appended. - :type path: str - :ivar fp: The file pointer used to write units to the file. - :type fp: A python file object. - :ivar total_units: Tracks the total number of units written. - :type total_units: int - :ivar bytes_written: The total number of bytes written. - :type bytes_written: int - """ - - def __init__(self, path): - """ - :param path: The absolute path to a file or directory. - When a directory is specified, the standard file name is appended. - :type path: str - :raise IOError: on I/O errors - """ - if os.path.isdir(path): - path = pathlib.join(path, UNITS_FILE_NAME) - self.path = path - self.fp = gzip.open(path, 'wb') - self.total_units = 0 - self.bytes_written = 0 - - @property - def closed(self): - """ - Determines if the file is closed or not. This exists because - the gzip API changed drastically from python 2.6 to 2.7. - :return: True if the file is closed. - :rtype: bool - """ - try: - # python 2.7 - return self.fp.closed - except AttributeError: - # python 2.6 - return self.fp.fileobj.closed - - def add(self, unit): - """ - Add (write) the specified unit to the file as a json encoded string. - :param unit: A content unit. - :type unit: dict - :raise IOError: on I/O errors. - :raise ValueError: json encoding errors - """ - self.total_units += 1 - json_unit = json.dumps(unit) - self.fp.write(json_unit) - self.fp.write('\n') - - def close(self): - """ - Close and compress the associated file. This method is idempotent. - :return: The number of units written. - :rtype: int - """ - if not self.closed: - self.fp.close() - self.bytes_written = os.path.getsize(self.path) - return self.total_units - - def __enter__(self): - return self - - def __exit__(self, *unused): - self.close() - return False - - -class UnitIterator: - """ - Used to iterate content units inventory file associated with a manifest. - The file contains (1) json encoded unit per line. The total number - of units in the file is reported by __len__(). - """ - - @staticmethod - def get_units(path): - with open(path) as fp: - while True: - begin = fp.tell() - json_unit = fp.readline() - end = fp.tell() - if json_unit: - unit = json.loads(json_unit) - length = (end - begin) - ref = UnitRef(path, begin, length) - yield (unit, ref) - else: - break - - def __init__(self, path, total_units): - """ - :param path: The absolute path to the units file to be iterated. - :type path: str - :param total_units: The number of units contained in the units file. - :type total_units: int - """ - self.unit_generator = UnitIterator.get_units(path) - self.total_units = total_units - - def next(self): - return self.unit_generator.next() - - def __iter__(self): - return self - - def __len__(self): - return self.total_units - - -class UnitRef(object): - """ - Reference to a unit within the downloaded units file. - :ivar path: The absolute path to the units file. - :type path: str - :ivar offset: The offset for a specific unit with the file. - :type offset: int - :ivar length: The length of a specific unit within the file. - :type length: int - """ - - def __init__(self, path, offset, length): - """ - :param path: The absolute path to the units file. - :type path: str - :param offset: The offset for a specific unit with the file. - :type offset: int - :param length: The length of a specific unit within the file. - :type length: int - """ - self.path = path - self.offset = offset - self.length = length - - def fetch(self): - """ - Fetch referenced content unit from the units file. - :return: The json decoded unit. - :rtype: dict - :raise IOError: on I/O errors. - :raise ValueError: json decoding errors - """ - with open(self.path) as fp: - fp.seek(self.offset) - json_unit = fp.read(self.length) - return json.loads(json_unit) diff --git a/nodes/common/pulp_node/pathlib.py b/nodes/common/pulp_node/pathlib.py deleted file mode 100644 index d81439480e..0000000000 --- a/nodes/common/pulp_node/pathlib.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -""" -Contains convenience functions for dealing with both filesystem and URL -related paths. Basically wrappers around os.path and urllib that compensates -for undesirable behaviors or modifies the behavior in ways that reduce code -duplication with in the nodes project. -""" - -import os -import urllib -import errno - - -def mkdir(path): - try: - os.makedirs(path) - except OSError, e: - if e.errno != errno.EEXIST: - raise e - - -def join(base, *paths): - path = os.path.join(*[p.lstrip('/') for p in paths]) - return os.path.join(base.rstrip('/'), path) - - -def url_join(base, *paths): - base = base.split('://', 1) - base = '://'.join((base[0], base[1].rstrip('/'))) - path = os.path.join(*[p.lstrip('/') for p in paths]) - return '/'.join((base, path)) - - -def quote(path): - return urllib.quote(path) diff --git a/nodes/common/pulp_node/poller.py b/nodes/common/pulp_node/poller.py deleted file mode 100644 index 9d16d238c2..0000000000 --- a/nodes/common/pulp_node/poller.py +++ /dev/null @@ -1,103 +0,0 @@ -import httplib -from gettext import gettext as _ -from time import sleep - -from pulp.common.constants import CALL_COMPLETE_STATES, CALL_ERRORED_STATE - - -FETCH_TASK_FAILED = _('Fetch task %(t)s, failed: http=%(c)s') -TASK_FAILED = _('Task %(t)s, failed: state=%(s)s') - - -class TaskFailed(Exception): - pass - - -class PollingFailed(Exception): - pass - - -class TaskPoller(object): - """ - The task poller is used to poll a running task by ID. - :ivar binding: A pulp API binding. - :type binding: pulp_node.handlers.model.PulpBinding - :ivar delay: The delay in seconds between each poll. - :type delay: int - """ - - DELAY = 1 - - def __init__(self, binding, delay=DELAY): - """ - :param binding: A pulp API binding. - :type binding: pulp_node.handlers.model.PulpBinding - :param delay: The delay in seconds between each poll. - :type delay: int - """ - self.binding = binding - self.delay = delay - - def join(self, task_id, progress, cancelled): - """ - Begin polling the specified task. - This call blocks until the task has completed. - :param task_id: A task ID. - :type task_id: str - :param progress: A progress reporting object. - :type progress: pulp_node.progress.RepositoryProgress - :param cancelled: A function used to get whether polling has been cancelled. - :type cancelled: callable - :return: The task result. - :raise PollingFailed: On failure to fetch the task. - :raise TaskFailed: On indication that the task being polled has failed. - """ - poll = True - task_result = None - last_hash = 0 - - while poll: - if cancelled(): - poll = False - continue - - sleep(self.delay) - - http = self.binding.tasks.get_task(task_id) - if http.response_code != httplib.OK: - msg = FETCH_TASK_FAILED % {'t': task_id, 'c': http.response_code} - raise PollingFailed(msg) - - task = http.response_body - - if task.state == CALL_ERRORED_STATE: - msg = TASK_FAILED % {'t': task_id, 's': task.state} - raise TaskFailed(msg, task.exception, task.traceback) - - last_hash = self._report_progress(progress, task, last_hash) - - if task.state in CALL_COMPLETE_STATES: - task_result = task.result - poll = False - - return task_result - - def _report_progress(self, progress, task, last_hash): - """ - Update the progress report only if the progress in the task has changed. - :param progress: A progress reporting object. - :type progress: pulp_node.progress.RepositoryProgress - :param task: A task - :type task: Task - :param last_hash: The hash of the last reported progress. - :type last_hash: int - :return The new hash. - :rtype: int - """ - _hash = hash(repr(task.progress_report)) - if _hash != last_hash: - if task.progress_report: - reported = task.progress_report.values()[0] - progress.__dict__.update(reported) - progress.updated() - return _hash diff --git a/nodes/common/pulp_node/reports.py b/nodes/common/pulp_node/reports.py deleted file mode 100644 index 2c73f2c97a..0000000000 --- a/nodes/common/pulp_node/reports.py +++ /dev/null @@ -1,167 +0,0 @@ -class RepositoryReport(object): - """ - Repository merge report. - :ivar repo_id: The repository ID. - :type repo_id: str - :ivar action: The action taken on the repository. - :param action: str - :ivar units: A content unit report. - :type units: UnitReport - :ivar sources: The content sources container statistics. - :type sources: dict - """ - - # actions - PENDING = 'pending' - CANCELLED = 'cancelled' - ADDED = 'added' - MERGED = 'merged' - DELETED = 'deleted' - - def __init__(self, repo_id, action=PENDING): - """ - :param repo_id: The repository ID. - :type repo_id: str - :param action: The action taken on the repository. - :param action: str - """ - self.repo_id = repo_id - self.action = action - self.units = UnitReport() - self.sources = {} - - def dict(self): - """ - Dictionary representation. - :return: A dictionary representation. - :rtype: dict - """ - return dict( - repo_id=self.repo_id, - action=self.action, - units=self.units.dict(), - sources=self.sources) - - -class UnitReport(object): - """ - Content unit synchronization summary report. - :ivar added: Count of units added. - :type added: int - :ivar updated: Count of units updated. - :type updated: int - :ivar removed: Count of units removed. - :type removed: int - """ - - def __init__(self): - self.added = 0 - self.updated = 0 - self.removed = 0 - - def dict(self): - """ - Dictionary representation. - :return: A dictionary representation. - :rtype: dict - """ - return self.__dict__ - - -# --- progress reporting ---------------------------------------------------- - - -class RepositoryProgress(object): - """ - Tracks the progress of a repository in the pulp nodes synchronization process. - """ - - PENDING = 'pending' - MERGING = 'merging' - IMPORTING = 'import_started' - DOWNLOADING_MANIFEST = 'downloading_manifest' - ADDING_UNITS = 'adding_units' - FINISHED = 'import_finished' - - def __init__(self, repo_id, listener=None): - """ - :param repo_id: A repository ID. - :type repo_id: str - :param listener: Progress change listener. An object that supports - a method updated() with this object as a parameter. The listener is - is notified each time the report has changed. - :type listener: object - """ - self.repo_id = repo_id - self.listener = listener - self.state = self.PENDING - self.unit_add = dict(total=0, completed=0, details=None) - - def begin_merging(self): - """ - Update the report to reflect that merging has started. - Set state=MERGING. - """ - self.state = self.MERGING - self.updated() - - def begin_importing(self): - """ - Update the report to reflect that the importer has stared synchronizing. - Set state=IMPORTING. - """ - self.state = self.IMPORTING - self.updated() - - def begin_manifest_download(self): - """ - Update the report to reflect that the importer has started downloading the manifest. - Set state=DOWNLOADING_MANIFEST. - """ - self.state = self.DOWNLOADING_MANIFEST - self.updated() - - def begin_adding_units(self, total): - """ - Update the report to reflect that the importer has stared adding/downloading units. - Set state=ADDING_UNITS. Updates the total number of units expected to be added. - :param total: The expected total number of units to be added. - :type total: int - """ - self.state = self.ADDING_UNITS - self.unit_add['total'] = total - self.updated() - - def unit_added(self, added=1, details=None): - """ - Update the report to reflect that one or more units have been added. - :param added: The number of units added since last reported. - :type added: int - :param details: Details (optional) about the unit added. - :type details: object - """ - self.unit_add['completed'] += added - self.unit_add['details'] = details - self.updated() - - def finished(self): - """ - Update the report to reflect that the synchronization has finished. - Set state=FINISHED. - """ - self.state = self.FINISHED - self.updated() - - def updated(self): - """ - The report has changed. Notify the listener. - """ - if self.listener is not None: - self.listener.updated(self) - - def dict(self): - return dict( - repo_id=self.repo_id, - state=self.state, - unit_add=self.unit_add - ) diff --git a/nodes/common/pulp_node/resources.py b/nodes/common/pulp_node/resources.py deleted file mode 100644 index cdf841aa98..0000000000 --- a/nodes/common/pulp_node/resources.py +++ /dev/null @@ -1,59 +0,0 @@ -from pulp.common.config import parse_bool -from pulp.server.config import config as pulp_conf -from pulp.bindings.server import PulpConnection -from pulp.bindings.bindings import Bindings - -from pulp_node.config import read_config - - -def parent_bindings(host, port=443): - """ - Get a pulp bindings object for the parent node. - :param host: The hostname of IP of the parent server. - :type host: str - :param port: The TCP port number. - :type port: int - :return: A pulp bindings object. - :rtype: Bindings - """ - node_conf = read_config() - oauth = node_conf.parent_oauth - verify_ssl = parse_bool(node_conf.main.verify_ssl) - ca_path = node_conf.main.ca_path - connection = PulpConnection( - host=host, - port=port, - oauth_key=oauth.key, - oauth_secret=oauth.secret, - oauth_user=oauth.user_id, - verify_ssl=verify_ssl, - ca_path=ca_path) - bindings = Bindings(connection) - return bindings - - -def pulp_bindings(): - """ - Get a pulp bindings object for this node. - Properties defined in the pulp server configuration are used - when not defined in the node configuration. - :return: A pulp bindings object. - :rtype: Bindings - """ - node_conf = read_config() - oauth = node_conf.oauth - verify_ssl = parse_bool(node_conf.main.verify_ssl) - ca_path = node_conf.main.ca_path - host = pulp_conf.get('server', 'server_name') - key = pulp_conf.get('oauth', 'oauth_key') - secret = pulp_conf.get('oauth', 'oauth_secret') - connection = PulpConnection( - host=host, - port=443, - oauth_key=key, - oauth_secret=secret, - oauth_user=oauth.user_id, - verify_ssl=verify_ssl, - ca_path=ca_path) - bindings = Bindings(connection) - return bindings diff --git a/nodes/common/setup.py b/nodes/common/setup.py deleted file mode 100755 index 21e934bd65..0000000000 --- a/nodes/common/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='pulp_node_common', - version='2.11a1', - license='GPLv2+', - packages=find_packages(), - author='Pulp Team', - author_email='pulp-list@redhat.com', -) diff --git a/nodes/extensions/admin/pulp_node/__init__.py b/nodes/extensions/admin/pulp_node/__init__.py deleted file mode 100644 index b36383a610..0000000000 --- a/nodes/extensions/admin/pulp_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/nodes/extensions/admin/pulp_node/extensions/__init__.py b/nodes/extensions/admin/pulp_node/extensions/__init__.py deleted file mode 100644 index b36383a610..0000000000 --- a/nodes/extensions/admin/pulp_node/extensions/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/nodes/extensions/admin/pulp_node/extensions/admin/__init__.py b/nodes/extensions/admin/pulp_node/extensions/admin/__init__.py deleted file mode 100644 index 32466f8d26..0000000000 --- a/nodes/extensions/admin/pulp_node/extensions/admin/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/nodes/extensions/admin/pulp_node/extensions/admin/commands.py b/nodes/extensions/admin/pulp_node/extensions/admin/commands.py deleted file mode 100644 index 2bfc045068..0000000000 --- a/nodes/extensions/admin/pulp_node/extensions/admin/commands.py +++ /dev/null @@ -1,510 +0,0 @@ -import os - -from gettext import gettext as _ - -from pulp.bindings.exceptions import NotFoundException -from pulp.client.arg_utils import convert_boolean_arguments -from pulp.client.extensions.decorator import priority -from pulp.client.extensions.extensions import PulpCliCommand, PulpCliOption -from pulp.client.commands.polling import PollingCommand -from pulp.client.commands.consumer.query import ConsumerListCommand -from pulp.client.commands.options import OPTION_REPO_ID, OPTION_CONSUMER_ID -from pulp.client.commands.repo.cudl import ListRepositoriesCommand - -from pulp_node import constants -from pulp_node.extension import (missing_resources, node_activated, repository_enabled, - ensure_node_section) -from pulp_node.extensions.admin import sync_schedules -from pulp_node.extensions.admin.options import (NODE_ID_OPTION, MAX_BANDWIDTH_OPTION, - MAX_CONCURRENCY_OPTION) -from pulp_node.extensions.admin.rendering import ProgressTracker, UpdateRenderer - - -NODE = _('Node') -CONSUMER = _('Consumer') -REPOSITORY = _('Repository') - - -REPO_NAME = 'repo' -ACTIVATE_NAME = 'activate' -DEACTIVATE_NAME = 'deactivate' -ENABLE_NAME = 'enable' -DISABLE_NAME = 'disable' -SYNC_NAME = 'sync' -PUBLISH_NAME = 'publish' -BIND_NAME = 'bind' -UNBIND_NAME = 'unbind' -UPDATE_NAME = 'run' -SCHEDULES_NAME = 'schedules' - - -NODE_LIST_DESC = _('list child nodes') -REPO_LIST_DESC = _('list node enabled repositories') -ACTIVATE_DESC = _('activate a consumer as a child node') -DEACTIVATE_DESC = _('deactivate a child node') -BIND_DESC = _('bind a child node to a repository') -UNBIND_DESC = _('removes the binding between a child node and a repository') -UPDATE_DESC = _('triggers an immediate synchronization of a child node') -ENABLE_DESC = _('enables binding to a repository by a child node') -DISABLE_DESC = _('disables binding to a repository by a child node') -REPO_DESC = _('repository related commands') -AUTO_PUBLISH_DESC = _('if "true", the nodes information will be automatically published each ' - 'time the repository is synchronized; defaults to "true"') -SYNC_DESC = _('child node synchronization commands') -PUBLISH_DESC = _('publishing commands') -STRATEGY_DESC = _('synchronization strategy (mirror|additive) default is additive') -SCHEDULES_DESC = _('manage node sync schedules') - - -NODE_LIST_TITLE = _('Child Nodes') -REPO_LIST_TITLE = _('Enabled Repositories') - - -AUTO_PUBLISH_OPTION = PulpCliOption('--auto-publish', AUTO_PUBLISH_DESC, required=False, - default='true') - -STRATEGY_OPTION = \ - PulpCliOption('--strategy', STRATEGY_DESC, required=False, default=constants.ADDITIVE_STRATEGY) - -# --- messages --------------------------------------------------------------- - -REPO_ENABLED = _('Repository enabled.') -REPO_DISABLED = _('Repository disabled.') -PUBLISH_SUCCEEDED = _('Publish succeeded.') -PUBLISH_FAILED = _('Publish failed. See: pulp log for details.') -NODE_ACTIVATED = _('Consumer activated as child node.') -NODE_DEACTIVATED = _('Child node deactivated.') -BIND_SUCCEEDED = _('Node bind succeeded.') -UNBIND_SUCCEEDED = _('Node unbind succeeded') -ALREADY_ENABLED = _('Repository already enabled. Nothing done.') -FAILED_NOT_ENABLED = _('Repository not enabled. See: the \'node repo enable\' command.') -NOT_BOUND_NOTHING_DONE = _('Node not bound to repository. No action performed.') -NOT_ACTIVATED_ERROR = _( - '%(t)s [ %(id)s ] not activated as a node. See: the \'node activate\' command.') -NOT_ACTIVATED_NOTHING_DONE = _('%(t)s is not activated as a node. No action performed.') -NOT_ENABLED_NOTHING_DONE = _('%(t)s not enabled. No action performed.') -STRATEGY_NOT_SUPPORTED = _('Strategy [ %(n)s ] not supported. Must be one of: %(s)s') -RESOURCE_MISSING_ERROR = _('%(t)s [ %(id)s ] not found on the server.') -ALREADY_ACTIVATED_NOTHING_DONE = _('%(n)s already activated as child node. No action performed.') - -BIND_WARNING = \ - _('Note: Repository [ %(r)s ] will be included in node synchronization.') -UNBIND_WARNING = \ - _('Warning: Repository [ %(r)s ] will NOT be included in node synchronization') - -ENABLE_WARNING = \ - _('Note: Repository [ %(r)s ] will not be available for node synchronization until published.' - ' See: the \'node repo publish\' command.') - -AUTO_PUBLISH_WARNING = \ - _('Warning: enabling with auto-publish may degrade repository synchronization performance.') - - -# --- extension loading ------------------------------------------------------ - -@priority() -def initialize(context): - """ - :type context: pulp.client.extensions.core.ClientContext - """ - node_section = ensure_node_section(context.cli) - node_section.add_command(NodeListCommand(context)) - node_section.add_command(NodeActivateCommand(context)) - node_section.add_command(NodeDeactivateCommand(context)) - node_section.add_command(NodeBindCommand(context)) - node_section.add_command(NodeUnbindCommand(context)) - - repo_section = node_section.create_subsection(REPO_NAME, REPO_DESC) - repo_section.add_command(NodeRepoEnableCommand(context)) - repo_section.add_command(NodeRepoDisableCommand(context)) - repo_section.add_command(NodeListRepositoriesCommand(context)) - repo_section.add_command(NodeRepoPublishCommand(context)) - - sync_section = node_section.create_subsection(SYNC_NAME, SYNC_DESC) - sync_section.add_command(NodeUpdateCommand(context)) - - schedules_section = sync_section.create_subsection(SCHEDULES_NAME, SCHEDULES_DESC) - schedules_section.add_command(sync_schedules.NodeCreateScheduleCommand(context)) - schedules_section.add_command(sync_schedules.NodeDeleteScheduleCommand(context)) - schedules_section.add_command(sync_schedules.NodeUpdateScheduleCommand(context)) - schedules_section.add_command(sync_schedules.NodeListScheduleCommand(context)) - schedules_section.add_command(sync_schedules.NodeNextRunCommand(context)) - - -# --- listing ---------------------------------------------------------------- - -class NodeListCommand(ConsumerListCommand): - - STRATEGY_FIELD = 'update_strategy' - - _ALL_FIELDS = ConsumerListCommand._ALL_FIELDS[0:-1] \ - + [STRATEGY_FIELD] + ConsumerListCommand._ALL_FIELDS[-1:] - - def __init__(self, context): - super(NodeListCommand, self).__init__(context, description=NODE_LIST_DESC) - - def get_title(self): - return NODE_LIST_TITLE - - def get_consumer_list(self, kwargs): - nodes = [] - for consumer in super(NodeListCommand, self).get_consumer_list(kwargs): - notes = consumer['notes'] - if not notes.get(constants.NODE_NOTE_KEY): - continue - consumer[self.STRATEGY_FIELD] = \ - notes.get(constants.STRATEGY_NOTE_KEY, constants.DEFAULT_STRATEGY) - nodes.append(consumer) - return nodes - - def format_bindings(self, consumer): - formatted = {} - key = 'bindings' - for binding in consumer.get(key, []): - repo_id = binding['repo_id'] - type_id = binding['type_id'] - if type_id not in constants.ALL_DISTRIBUTORS: - # nodes only - continue - strategy = binding['binding_config'].get('strategy', constants.DEFAULT_STRATEGY) - repo_ids = formatted.get(strategy) - if repo_ids is None: - repo_ids = [] - formatted[strategy] = repo_ids - repo_ids.append(repo_id) - consumer[key] = formatted - - -class NodeListRepositoriesCommand(ListRepositoriesCommand): - - def __init__(self, context): - super(NodeListRepositoriesCommand, self).__init__( - context, - description=REPO_LIST_DESC, - repos_title=REPO_LIST_TITLE) - - def get_repositories(self, query_params, **kwargs): - enabled = [] - _super = super(NodeListRepositoriesCommand, self) - repositories = _super.get_repositories(query_params, **kwargs) - for repository in repositories: - repo_id = repository['id'] - http = self.context.server.repo_distributor.distributors(repo_id) - for dist in http.response_body: - if dist['distributor_type_id'] in constants.ALL_DISTRIBUTORS: - enabled.append(repository) - return enabled - - -# --- publishing ------------------------------------------------------------- - -class NodeRepoPublishCommand(PollingCommand): - - def __init__(self, context): - super(NodeRepoPublishCommand, self).__init__(PUBLISH_NAME, PUBLISH_DESC, self.run, context) - self.add_option(OPTION_REPO_ID) - - def run(self, **kwargs): - repo_id = kwargs[OPTION_REPO_ID.keyword] - - if not repository_enabled(self.context, repo_id): - msg = FAILED_NOT_ENABLED - self.context.prompt.render_failure_message(msg) - return - - try: - http = self.context.server.repo_actions.publish(repo_id, constants.HTTP_DISTRIBUTOR, {}) - task = http.response_body - self.poll([task], kwargs) - except NotFoundException, e: - for _id, _type in missing_resources(e): - if _type == 'repo_id': - msg = RESOURCE_MISSING_ERROR % dict(t=REPOSITORY, id=_id) - self.context.prompt.render_failure_message(msg) - else: - raise - return os.EX_DATAERR - - def succeeded(self, task): - self.context.prompt.render_success_message(PUBLISH_SUCCEEDED) - - def failed(self, task): - self.context.prompt.render_failure_message(PUBLISH_FAILED) - - -# --- activation ------------------------------------------------------------- - -class NodeActivateCommand(PulpCliCommand): - - def __init__(self, context): - super(NodeActivateCommand, self).__init__(ACTIVATE_NAME, ACTIVATE_DESC, self.run) - self.add_option(OPTION_CONSUMER_ID) - self.add_option(STRATEGY_OPTION) - self.context = context - - def run(self, **kwargs): - - consumer_id = kwargs[OPTION_CONSUMER_ID.keyword] - strategy = kwargs[STRATEGY_OPTION.keyword] - delta = {'notes': {constants.NODE_NOTE_KEY: True, constants.STRATEGY_NOTE_KEY: strategy}} - - if node_activated(self.context, consumer_id): - msg = ALREADY_ACTIVATED_NOTHING_DONE % dict(n=CONSUMER) - self.context.prompt.render_success_message(msg) - return - - if strategy not in constants.STRATEGIES: - msg = STRATEGY_NOT_SUPPORTED % dict(n=strategy, s=constants.STRATEGIES) - self.context.prompt.render_failure_message(msg) - return os.EX_DATAERR - - try: - self.context.server.consumer.update(consumer_id, delta) - self.context.prompt.render_success_message(NODE_ACTIVATED) - except NotFoundException, e: - for _id, _type in missing_resources(e): - if _type == 'consumer': - msg = RESOURCE_MISSING_ERROR % dict(t=CONSUMER, id=_id) - self.context.prompt.render_failure_message(msg) - else: - raise - return os.EX_DATAERR - - -class NodeDeactivateCommand(PulpCliCommand): - - def __init__(self, context): - super(NodeDeactivateCommand, self).__init__(DEACTIVATE_NAME, DEACTIVATE_DESC, self.run) - self.add_option(NODE_ID_OPTION) - self.context = context - - def run(self, **kwargs): - - consumer_id = kwargs[NODE_ID_OPTION.keyword] - delta = {'notes': {constants.NODE_NOTE_KEY: None, constants.STRATEGY_NOTE_KEY: None}} - - if not node_activated(self.context, consumer_id): - msg = NOT_ACTIVATED_NOTHING_DONE % dict(t=CONSUMER) - self.context.prompt.render_success_message(msg) - return - - try: - self.context.server.consumer.update(consumer_id, delta) - self.context.prompt.render_success_message(NODE_DEACTIVATED) - except NotFoundException, e: - for _id, _type in missing_resources(e): - if _type == 'consumer': - msg = RESOURCE_MISSING_ERROR % dict(t=CONSUMER, id=_id) - self.context.prompt.render_failure_message(msg) - else: - raise - return os.EX_DATAERR - - -# --- enable ----------------------------------------------------------------- - -class NodeRepoEnableCommand(PulpCliCommand): - - def __init__(self, context): - super(NodeRepoEnableCommand, self).__init__(ENABLE_NAME, ENABLE_DESC, self.run) - self.add_option(OPTION_REPO_ID) - self.add_option(AUTO_PUBLISH_OPTION) - self.context = context - - def run(self, **kwargs): - - convert_boolean_arguments([AUTO_PUBLISH_OPTION.keyword], kwargs) - - repo_id = kwargs[OPTION_REPO_ID.keyword] - auto_publish = kwargs[AUTO_PUBLISH_OPTION.keyword] - binding = self.context.server.repo_distributor - - if repository_enabled(self.context, repo_id): - msg = ALREADY_ENABLED - self.context.prompt.render_success_message(msg) - return - - try: - binding.create( - repo_id, - constants.HTTP_DISTRIBUTOR, - {}, - auto_publish, - constants.HTTP_DISTRIBUTOR) - self.context.prompt.render_success_message(REPO_ENABLED) - self.context.prompt.render_warning_message(ENABLE_WARNING % dict(r=repo_id)) - if auto_publish: - self.context.prompt.render_warning_message(AUTO_PUBLISH_WARNING) - except NotFoundException, e: - for _id, _type in missing_resources(e): - if _type == 'repository': - msg = RESOURCE_MISSING_ERROR % dict(t=REPOSITORY, id=_id) - self.context.prompt.render_failure_message(msg) - else: - raise - return os.EX_DATAERR - - -class NodeRepoDisableCommand(PulpCliCommand): - - def __init__(self, context): - super(NodeRepoDisableCommand, self).__init__(DISABLE_NAME, DISABLE_DESC, self.run) - self.add_option(OPTION_REPO_ID) - self.context = context - - def run(self, **kwargs): - - repo_id = kwargs[OPTION_REPO_ID.keyword] - - try: - self.context.server.repo_distributor.delete(repo_id, constants.HTTP_DISTRIBUTOR) - self.context.prompt.render_success_message(REPO_DISABLED) - except NotFoundException, e: - for _id, _type in missing_resources(e): - if _type == 'repository': - msg = RESOURCE_MISSING_ERROR % dict(t=REPOSITORY, id=_id) - self.context.prompt.render_failure_message(msg) - continue - if _type == 'distributor': - msg = NOT_ENABLED_NOTHING_DONE % dict(t=REPOSITORY) - self.context.prompt.render_failure_message(msg) - continue - raise - return os.EX_DATAERR - - -class BindingCommand(PulpCliCommand): - - def missing_resources(self, prompt, exception): - unhandled = [] - for _id, _type in missing_resources(exception): - if _type == 'consumer_id': - msg = RESOURCE_MISSING_ERROR % dict(t=NODE, id=_id) - prompt.render_failure_message(msg) - continue - if _type == 'repo_id': - msg = RESOURCE_MISSING_ERROR % dict(t=REPOSITORY, id=_id) - prompt.render_failure_message(msg) - continue - unhandled.append((_id, _type)) - return unhandled - - -class NodeBindCommand(BindingCommand): - - def __init__(self, context): - super(NodeBindCommand, self).__init__(BIND_NAME, BIND_DESC, self.run) - self.add_option(OPTION_REPO_ID) - self.add_option(NODE_ID_OPTION) - self.add_option(STRATEGY_OPTION) - self.context = context - - def run(self, **kwargs): - - repo_id = kwargs[OPTION_REPO_ID.keyword] - node_id = kwargs[NODE_ID_OPTION.keyword] - dist_id = constants.HTTP_DISTRIBUTOR - strategy = kwargs[STRATEGY_OPTION.keyword] - binding_config = {constants.STRATEGY_KEYWORD: strategy} - - if not node_activated(self.context, node_id): - msg = NOT_ACTIVATED_ERROR % dict(t=CONSUMER, id=node_id) - self.context.prompt.render_failure_message(msg) - return os.EX_USAGE - - if not repository_enabled(self.context, repo_id): - msg = FAILED_NOT_ENABLED - self.context.prompt.render_failure_message(msg) - return os.EX_USAGE - - if strategy not in constants.STRATEGIES: - msg = STRATEGY_NOT_SUPPORTED % dict(n=strategy, s=constants.STRATEGIES) - self.context.prompt.render_failure_message(msg) - return os.EX_DATAERR - - self.context.server.bind.bind( - node_id, - repo_id, - dist_id, - notify_agent=False, - binding_config=binding_config) - self.context.prompt.render_success_message(BIND_SUCCEEDED) - warning = BIND_WARNING % dict(r=repo_id) - self.context.prompt.render_warning_message(warning) - - -class NodeUnbindCommand(BindingCommand): - - def __init__(self, context): - super(NodeUnbindCommand, self).__init__(UNBIND_NAME, UNBIND_DESC, self.run) - self.add_option(OPTION_REPO_ID) - self.add_option(NODE_ID_OPTION) - self.context = context - - def run(self, **kwargs): - - repo_id = kwargs[OPTION_REPO_ID.keyword] - node_id = kwargs[NODE_ID_OPTION.keyword] - dist_id = constants.HTTP_DISTRIBUTOR - - try: - self.context.server.bind.unbind(node_id, repo_id, dist_id) - self.context.prompt.render_success_message(UNBIND_SUCCEEDED) - warning = UNBIND_WARNING % dict(r=repo_id) - self.context.prompt.render_warning_message(warning) - except NotFoundException, e: - unhandled = self.missing_resources(self.context.prompt, e) - for _id, _type in unhandled: - if _type == 'bind_id': - msg = NOT_BOUND_NOTHING_DONE - self.context.prompt.render_failure_message(msg) - else: - raise - return os.EX_DATAERR - - -class NodeUpdateCommand(PollingCommand): - - def __init__(self, context): - super(NodeUpdateCommand, self).__init__(UPDATE_NAME, UPDATE_DESC, self.run, context) - self.add_option(NODE_ID_OPTION) - self.add_option(MAX_CONCURRENCY_OPTION) - self.add_option(MAX_BANDWIDTH_OPTION) - self.tracker = ProgressTracker(self.context.prompt) - - def run(self, **kwargs): - node_id = kwargs[NODE_ID_OPTION.keyword] - max_bandwidth = kwargs[MAX_BANDWIDTH_OPTION.keyword] - max_concurrency = kwargs[MAX_CONCURRENCY_OPTION.keyword] - units = [dict(type_id='node', unit_key=None)] - options = { - constants.MAX_DOWNLOAD_BANDWIDTH_KEYWORD: max_bandwidth, - constants.MAX_DOWNLOAD_CONCURRENCY_KEYWORD: max_concurrency, - } - - if not node_activated(self.context, node_id): - msg = NOT_ACTIVATED_ERROR % dict(t=CONSUMER, id=node_id) - self.context.prompt.render_failure_message(msg) - return os.EX_USAGE - - try: - http = self.context.server.consumer_content.update(node_id, units=units, - options=options) - task = http.response_body - self.poll([task], kwargs) - except NotFoundException, e: - for _id, _type in missing_resources(e): - if _type == 'consumer': - msg = RESOURCE_MISSING_ERROR % dict(t=NODE, id=_id) - self.context.prompt.render_failure_message(msg) - else: - raise - return os.EX_DATAERR - - def progress(self, task, spinner): - self.tracker.display(task.progress_report) - - def succeeded(self, task): - report = task.result['details'].values()[0] - r = UpdateRenderer(self.context.prompt, report) - r.render() diff --git a/nodes/extensions/admin/pulp_node/extensions/admin/options.py b/nodes/extensions/admin/pulp_node/extensions/admin/options.py deleted file mode 100644 index 4ba2241cca..0000000000 --- a/nodes/extensions/admin/pulp_node/extensions/admin/options.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -""" -Contains options and flags common across nodes commands. -""" - -from gettext import gettext as _ - -from pulp.client.commands.options import DESC_ID -from pulp.client.extensions.extensions import PulpCliOption -from pulp.client.validators import id_validator_allow_dots -from pulp.client.parsers import pulp_parse_optional_positive_int - - -# --- descriptions ----------------------------------------------------------- - -MAX_BANDWIDTH_DESC = _('maximum bandwidth used per download in bytes/sec') -MAX_CONCURRENCY_DESC = _('maximum number of downloads permitted to run concurrently') - - -# --- options ---------------------------------------------------------------- - -NODE_ID_OPTION = PulpCliOption( - '--node-id', DESC_ID, required=True, validate_func=id_validator_allow_dots) - -MAX_BANDWIDTH_OPTION = PulpCliOption( - '--max-speed', MAX_BANDWIDTH_DESC, required=False, parse_func=pulp_parse_optional_positive_int) - -MAX_CONCURRENCY_OPTION = PulpCliOption( - '--max-downloads', MAX_CONCURRENCY_DESC, required=False, - parse_func=pulp_parse_optional_positive_int) diff --git a/nodes/extensions/admin/pulp_node/extensions/admin/rendering.py b/nodes/extensions/admin/pulp_node/extensions/admin/rendering.py deleted file mode 100644 index eb7f55cc95..0000000000 --- a/nodes/extensions/admin/pulp_node/extensions/admin/rendering.py +++ /dev/null @@ -1,262 +0,0 @@ -from gettext import gettext as _ -from operator import itemgetter - -from pulp.common.constants import PRIMARY_ID - -from pulp_node import error -from pulp_node.reports import RepositoryProgress, RepositoryReport - - -REPOSITORY_FIELD = _('(%(n)d/%(t)d) Repository: %(id)s') -STEP_FIELD = _('Step: %(s)s') -ADD_UNIT_FIELD = _('(%(n)d/%(t)d) Add unit: %(d)s') - -PROGRESS_STATES = { - RepositoryProgress.PENDING: _('Pending'), - RepositoryProgress.MERGING: _('Merging Repository'), - RepositoryProgress.DOWNLOADING_MANIFEST: _('Downloading Manifest'), - RepositoryProgress.ADDING_UNITS: _('Adding Units'), - RepositoryProgress.IMPORTING: _('Importing'), - RepositoryProgress.FINISHED: _('Finished') -} - -ACTIONS = { - RepositoryReport.PENDING: _('Pending'), - RepositoryReport.CANCELLED: _('Cancelled'), - RepositoryReport.ADDED: _('Added'), - RepositoryReport.MERGED: _('Merged'), - RepositoryReport.DELETED: _('Removed') -} - -NODE_ERRORS = { - error.CaughtException.ERROR_ID: error.CaughtException.DESCRIPTION, - error.PurgeOrphansError.ERROR_ID: error.PurgeOrphansError.DESCRIPTION, - error.RepoSyncRestError.ERROR_ID: error.RepoSyncRestError.DESCRIPTION, - error.GetBindingsError.ERROR_ID: error.GetBindingsError.DESCRIPTION, - error.GetChildUnitsError.ERROR_ID: error.GetChildUnitsError.DESCRIPTION, - error.GetParentUnitsError.ERROR_ID: error.GetParentUnitsError.DESCRIPTION, - error.ImporterNotInstalled.ERROR_ID: error.ImporterNotInstalled.DESCRIPTION, - error.DistributorNotInstalled.ERROR_ID: error.DistributorNotInstalled.DESCRIPTION, - error.ManifestDownloadError.ERROR_ID: error.ManifestDownloadError.DESCRIPTION, - error.UnitDownloadError.ERROR_ID: error.UnitDownloadError.DESCRIPTION, - error.AddUnitError.ERROR_ID: error.AddUnitError.DESCRIPTION, - error.DeleteUnitError.ERROR_ID: error.DeleteUnitError.DESCRIPTION, - error.InvalidManifestError.ERROR_ID: error.InvalidManifestError.DESCRIPTION, -} - -SYNC_TITLE = _('Child Node Synchronization') -SUCCEEDED_MSG = _('Synchronization succeeded') -FAILED_MSG = _('Error occurred during synchronization, check the child node logs for details') -REPORTED_ERRORS = _('The following [%(n)d] errors were reported') -REPOSITORIES_FAILED = _('The following repositories had errors') - -PARENT_NODE = _('Parent Node') - - -class ProgressTracker: - """ - Track and render repository progress reports. - Each report has the following format: - {repo_id: , - state: , - unit_add: { - total: , - completed: , - details: ) - } - """ - - @staticmethod - def _find(repo_id, reports): - """ - Find a report by repo_id. - :param repo_id: A repository ID. - :type repo_id: str - :param reports: List of repository progress reports. - :type reports: list - :return: The found report or None - :rtype: dict - """ - for r in reports: - if r['repo_id'] == repo_id: - return r - - @staticmethod - def _render(report, progress_bar): - """ - Render the specified report and progress bar. - :param report: A repository progress report. - :type report: dict - :param progress_bar: An okaara progress bar. - :type progress_bar: okaara.progress.ProgressBar - """ - state = report['state'] - unit_add = report['unit_add'] - total = unit_add['total'] - completed = unit_add['completed'] - details = unit_add['details'] or '' - - # just display the file part of paths because the directory - # is not interesting and clutters up the display. - - if '/' in details: - details = details.rsplit('/', 1)[1] - - # message part of the progress bar - - if state == RepositoryProgress.ADDING_UNITS: - message = '\n'.join( - (STEP_FIELD % {'s': PROGRESS_STATES[state]}, - ADD_UNIT_FIELD % {'n': completed, 't': total, 'd': details})) - else: - message = None - - # prevent divide by zero and make sure the progress bar and - # make sure the progress bar shows complete when the report is finished. - - if total < 1: - if state == RepositoryProgress.FINISHED: - progress_bar.render(1, 1) - else: - progress_bar.render(0.01, 1) - else: - progress_bar.render(completed, total, message) - - return progress_bar - - def __init__(self, prompt): - """ - :param prompt: An okaara prompt. - :type prompt: okaara.prompt.Prompt - """ - self.prompt = prompt - self.snapshot = [] - - def display(self, report): - """ - Display the specified progress report. - :param report: A serialized pulp_node.reports.RepositoryProgress. - :type report: dict - """ - if report is None: - # nothing to render - return - reports = report.get('progress') - if reports is None: - # nothing to render - return - - # On the 2nd+ report, update the last in-progress report. - - if self.snapshot: - report, progress_bar = self.snapshot[-1] - repo_id = report['repo_id'] - report = self._find(repo_id, reports) - self._render(report, progress_bar) - - # The latency in polling can causes gaps in the reported progress. - # This includes the gap between never having processed a report and receiving - # the 1st report. Here, we get caught up in all cases. - - in_progress = [r for r in reports if r['state'] != RepositoryProgress.PENDING] - for i in range(len(self.snapshot), len(in_progress)): - r = in_progress[i] - progress_bar = self.prompt.create_progress_bar() - self.snapshot.append((r, progress_bar)) - repo_id = r['repo_id'] - self.prompt.write('\n') - self.prompt.write(REPOSITORY_FIELD % {'n': i + 1, 't': len(reports), 'id': repo_id}) - self._render(r, progress_bar) - - -class UpdateRenderer(object): - """ - Render the node synchronization summary with the following format: - succeeded: - details: { - errors: [ - { error_id: , - details: {} - }, - ] - repositories: [ - { repo_id: , - action: , - units: { - added: , - updated: , - removed: - } - }, - ] - } - """ - - def __init__(self, prompt, report): - self.prompt = prompt - self.succeeded = report['succeeded'] - self.details = report['details'] - self.errors = self.details.get('errors', []) - self.message = self.details.get('message') - self.repositories = self.details.get('repositories', {}) - - def render(self): - if self.message: - # An unexpected exception has been reported - self.prompt.render_failure_message(self.message) - self.prompt.render_failure_message(FAILED_MSG) - return - - documents = [] - for repo_report in sorted(self.repositories, key=itemgetter('repo_id')): - sources = repo_report['sources'] - downloads = sources.get('downloads', {}) - for source_id, stat in downloads.items(): - if source_id == PRIMARY_ID: - source_id = PARENT_NODE - stat['source_id'] = source_id - sources['downloads'] = downloads.values() - document = { - 'repository': { - 'id': repo_report['repo_id'], - 'action': ACTIONS[repo_report['action']], - 'units': repo_report['units'], - 'content_sources': sources - }, - } - documents.append(document) - - self.prompt.write('\n\n') - - if self.succeeded: - self.prompt.render_success_message(SUCCEEDED_MSG) - else: - self.prompt.render_failure_message(FAILED_MSG) - - self.prompt.render_title(SYNC_TITLE) - self.prompt.render_document_list(documents) - - if not self.succeeded: - n = 1 - self.prompt.render_title(REPORTED_ERRORS % dict(n=len(self.errors))) - for ne in self.errors: - repo_id = ne['error_id'] - details = ne['details'] - details['n'] = n - description = NODE_ERRORS.get(repo_id, str(ne)) - self.prompt.write('- %.2d: %s\n' % (n, description % details)) - n += 1 - - failed_repositories = self.failed_repositories() - if failed_repositories: - self.prompt.render_title(REPOSITORIES_FAILED) - for repo_id in failed_repositories: - self.prompt.write('- %s' % repo_id) - - def failed_repositories(self): - failed = set() - for e in self.errors: - repo_id = e['details'].get('repo_id') - if repo_id: - failed.add(repo_id) - return sorted(failed) diff --git a/nodes/extensions/admin/pulp_node/extensions/admin/sync_schedules.py b/nodes/extensions/admin/pulp_node/extensions/admin/sync_schedules.py deleted file mode 100644 index d197ac0294..0000000000 --- a/nodes/extensions/admin/pulp_node/extensions/admin/sync_schedules.py +++ /dev/null @@ -1,96 +0,0 @@ -from gettext import gettext as _ - -from pulp.client.commands.schedule import ( - DeleteScheduleCommand, ListScheduleCommand, CreateScheduleCommand, - UpdateScheduleCommand, NextRunCommand, ScheduleStrategy) - -from pulp_node import constants -from pulp_node.extensions.admin.options import (NODE_ID_OPTION, MAX_BANDWIDTH_OPTION, - MAX_CONCURRENCY_OPTION) - - -DESC_LIST = _('list scheduled sync operations') -DESC_CREATE = _('adds a new scheduled sync operation') -DESC_DELETE = _('delete a sync schedule') -DESC_UPDATE = _('updates an existing schedule') -DESC_NEXT_RUN = _('displays the next scheduled sync run for a child node') - -# A node sync is considered an update operation on the REST API -SYNC_OPERATION = 'update' - - -class NodeListScheduleCommand(ListScheduleCommand): - def __init__(self, context): - strategy = NodeSyncScheduleStrategy(context) - super(self.__class__, self).__init__(context, strategy, description=DESC_LIST) - self.add_option(NODE_ID_OPTION) - - -class NodeCreateScheduleCommand(CreateScheduleCommand): - def __init__(self, context): - strategy = NodeSyncScheduleStrategy(context) - super(self.__class__, self).__init__(context, strategy, description=DESC_CREATE) - self.add_option(NODE_ID_OPTION) - self.add_option(MAX_BANDWIDTH_OPTION) - self.add_option(MAX_CONCURRENCY_OPTION) - - -class NodeDeleteScheduleCommand(DeleteScheduleCommand): - def __init__(self, context): - strategy = NodeSyncScheduleStrategy(context) - super(self.__class__, self).__init__(context, strategy, description=DESC_DELETE) - self.add_option(NODE_ID_OPTION) - - -class NodeUpdateScheduleCommand(UpdateScheduleCommand): - def __init__(self, context): - strategy = NodeSyncScheduleStrategy(context) - super(self.__class__, self).__init__(context, strategy, description=DESC_UPDATE) - self.add_option(NODE_ID_OPTION) - - -class NodeNextRunCommand(NextRunCommand): - def __init__(self, context): - strategy = NodeSyncScheduleStrategy(context) - super(self.__class__, self).__init__(context, strategy, description=DESC_NEXT_RUN) - self.add_option(NODE_ID_OPTION) - - -class NodeSyncScheduleStrategy(ScheduleStrategy): - - # See super class for method documentation - - def __init__(self, context): - super(self.__class__, self).__init__() - self.context = context - self.api = context.server.consumer_content_schedules - - def create_schedule(self, schedule, failure_threshold, enabled, kwargs): - node_id = kwargs[NODE_ID_OPTION.keyword] - max_bandwidth = kwargs[MAX_BANDWIDTH_OPTION.keyword] - max_concurrency = kwargs[MAX_CONCURRENCY_OPTION.keyword] - units = [dict(type_id='node', unit_key=None)] - options = { - constants.MAX_DOWNLOAD_BANDWIDTH_KEYWORD: max_bandwidth, - constants.MAX_DOWNLOAD_CONCURRENCY_KEYWORD: max_concurrency, - } - return self.api.add_schedule( - SYNC_OPERATION, - node_id, - schedule, - units, - failure_threshold, - enabled, - options) - - def delete_schedule(self, schedule_id, kwargs): - node_id = kwargs[NODE_ID_OPTION.keyword] - return self.api.delete_schedule(SYNC_OPERATION, node_id, schedule_id) - - def retrieve_schedules(self, kwargs): - node_id = kwargs[NODE_ID_OPTION.keyword] - return self.api.list_schedules(SYNC_OPERATION, node_id) - - def update_schedule(self, schedule_id, **kwargs): - node_id = kwargs.pop(NODE_ID_OPTION.keyword) - return self.api.update_schedule(SYNC_OPERATION, node_id, schedule_id, **kwargs) diff --git a/nodes/extensions/admin/setup.py b/nodes/extensions/admin/setup.py deleted file mode 100644 index cf0bb5886f..0000000000 --- a/nodes/extensions/admin/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='pulp_node_admin_extensions', - version='2.11a1', - license='GPLv2+', - packages=find_packages(exclude=['test', 'test.*']), - author='Pulp Team', - author_email='pulp-list@redhat.com', - entry_points={ - 'pulp.extensions.admin': [ - 'repo_admin = pulp_node.extensions.admin.commands:initialize', - ] - } -) diff --git a/nodes/extensions/consumer/pulp_node/__init__.py b/nodes/extensions/consumer/pulp_node/__init__.py deleted file mode 100644 index b36383a610..0000000000 --- a/nodes/extensions/consumer/pulp_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/nodes/extensions/consumer/pulp_node/extensions/__init__.py b/nodes/extensions/consumer/pulp_node/extensions/__init__.py deleted file mode 100644 index b36383a610..0000000000 --- a/nodes/extensions/consumer/pulp_node/extensions/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/nodes/extensions/consumer/pulp_node/extensions/consumer/__init__.py b/nodes/extensions/consumer/pulp_node/extensions/consumer/__init__.py deleted file mode 100644 index 32466f8d26..0000000000 --- a/nodes/extensions/consumer/pulp_node/extensions/consumer/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/nodes/extensions/consumer/pulp_node/extensions/consumer/commands.py b/nodes/extensions/consumer/pulp_node/extensions/consumer/commands.py deleted file mode 100644 index b57c210a40..0000000000 --- a/nodes/extensions/consumer/pulp_node/extensions/consumer/commands.py +++ /dev/null @@ -1,231 +0,0 @@ -import os - -from gettext import gettext as _ - -from pulp.bindings.exceptions import NotFoundException - -from pulp.client.extensions.decorator import priority -from pulp.client.extensions.extensions import PulpCliOption, PulpCliCommand -from pulp.client.commands.options import OPTION_REPO_ID -from pulp.client.consumer_utils import load_consumer_id - -from pulp_node import constants -from pulp_node.extension import ensure_node_section, node_activated, repository_enabled -from pulp_node.extension import missing_resources - - -# --- resources -------------------------------------------------------------- - -NODE = _('Node') -REPOSITORY = _('Repository') - - -# --- names ------------------------------------------------------------------ - -ACTIVATE_NAME = 'activate' -DEACTIVATE_NAME = 'deactivate' -BIND_NAME = 'bind' -UNBIND_NAME = 'unbind' - - -# --- descriptions ----------------------------------------------------------- - -BIND_DESC = _('bind this node to a repository') -UNBIND_DESC = _('remove the binding between this node and a repository') -ACTIVATE_DESC = _('activate a consumer as a child node') -DEACTIVATE_DESC = _('deactivate a child node') -STRATEGY_DESC = _('synchronization strategy (mirror|additive) default is additive') - - -# --- messages --------------------------------------------------------------- - -NODE_ACTIVATED = _('Consumer activated as child node') -NODE_DEACTIVATED = _('Child node deactivated') -BIND_SUCCEEDED = _('Node bind succeeded.') -UNBIND_SUCCEEDED = _('Node unbind succeeded') -BIND_FAILED_NOT_ENABLED = _('Repository not enabled. See: \'node repo enable\' command.') -NOT_BOUND_NOTHING_DONE = _('Node not bound to repository. No action performed.') -NOT_ACTIVATED_NOTHING_DONE = _('This consumer is not activated as a node. No action performed.') -NOT_ACTIVATED_ERROR = _('This consumer is not activated as a node. See: \'node activate\' command.') -STRATEGY_NOT_SUPPORTED = _('Strategy [ %(n)s ] not supported. Must be one of: %(s)s') -RESOURCE_MISSING_ERROR = _('%(t)s [ %(id)s ] not found on the server.') -NOT_REGISTERED_MESSAGE = _('This consumer is not registered.') -ALREADY_ACTIVATED_NOTHING_DONE = _('This consumer already activated. No action performed.') - -BIND_WARNING = \ - _('Note: Repository [ %(r)s ] will be included in node synchronization.') -UNBIND_WARNING = \ - _('Warning: Repository [ %(r)s ] will NOT be included in node synchronization') - - -# --- options ---------------------------------------------------------------- - -STRATEGY_OPTION = PulpCliOption('--strategy', STRATEGY_DESC, required=False, - default=constants.ADDITIVE_STRATEGY) - - -# --- extension loading ------------------------------------------------------ - -@priority() -def initialize(context): - """ - :type context: pulp.client.extensions.core.ClientContext - """ - node_section = ensure_node_section(context.cli) - node_section.add_command(NodeActivateCommand(context)) - node_section.add_command(NodeDeactivateCommand(context)) - node_section.add_command(NodeBindCommand(context)) - node_section.add_command(NodeUnbindCommand(context)) - - -# --- activation ------------------------------------------------------------- - -class NodeActivateCommand(PulpCliCommand): - - def __init__(self, context): - super(NodeActivateCommand, self).__init__(ACTIVATE_NAME, ACTIVATE_DESC, self.run) - self.add_option(STRATEGY_OPTION) - self.context = context - - def run(self, **kwargs): - - consumer_id = load_consumer_id(self.context) - strategy = kwargs[STRATEGY_OPTION.keyword] - delta = {'notes': {constants.NODE_NOTE_KEY: True, constants.STRATEGY_NOTE_KEY: strategy}} - - if node_activated(self.context, consumer_id): - self.context.prompt.render_success_message(ALREADY_ACTIVATED_NOTHING_DONE) - return - - if strategy not in constants.STRATEGIES: - msg = STRATEGY_NOT_SUPPORTED % dict(n=strategy, s=constants.STRATEGIES) - self.context.prompt.render_failure_message(msg) - return os.EX_DATAERR - - try: - self.context.server.consumer.update(consumer_id, delta) - self.context.prompt.render_success_message(NODE_ACTIVATED) - except NotFoundException, e: - for _id, _type in missing_resources(e): - if _type == 'consumer': - self.context.prompt.render_failure_message(NOT_REGISTERED_MESSAGE) - else: - raise - return os.EX_DATAERR - - -class NodeDeactivateCommand(PulpCliCommand): - - def __init__(self, context): - super(NodeDeactivateCommand, self).__init__(DEACTIVATE_NAME, DEACTIVATE_DESC, self.run) - self.context = context - - def run(self, **kwargs): - - consumer_id = load_consumer_id(self.context) - delta = {'notes': {constants.NODE_NOTE_KEY: None, constants.STRATEGY_NOTE_KEY: None}} - - if not node_activated(self.context, consumer_id): - self.context.prompt.render_success_message(NOT_ACTIVATED_NOTHING_DONE) - return - - try: - self.context.server.consumer.update(consumer_id, delta) - self.context.prompt.render_success_message(NODE_DEACTIVATED) - except NotFoundException, e: - for _id, _type in missing_resources(e): - if _type == 'consumer': - self.context.prompt.render_failure_message(NOT_REGISTERED_MESSAGE) - else: - raise - return os.EX_DATAERR - - -# --- bind ------------------------------------------------------------------- - -class BindingCommand(PulpCliCommand): - - def missing_resources(self, prompt, exception): - unhandled = [] - for _id, _type in missing_resources(exception): - if _type == 'consumer_id': - msg = RESOURCE_MISSING_ERROR % dict(t=NODE, id=_id) - prompt.render_failure_message(msg) - continue - if _type == 'repo_id': - msg = RESOURCE_MISSING_ERROR % dict(t=REPOSITORY, id=_id) - prompt.render_failure_message(msg) - continue - unhandled.append((_id, _type)) - return unhandled - - -class NodeBindCommand(BindingCommand): - - def __init__(self, context): - super(NodeBindCommand, self).__init__(BIND_NAME, BIND_DESC, self.run) - self.add_option(OPTION_REPO_ID) - self.add_option(STRATEGY_OPTION) - self.context = context - - def run(self, **kwargs): - - repo_id = kwargs[OPTION_REPO_ID.keyword] - node_id = load_consumer_id(self.context) - dist_id = constants.HTTP_DISTRIBUTOR - strategy = kwargs[STRATEGY_OPTION.keyword] - binding_config = {constants.STRATEGY_KEYWORD: strategy} - - if not node_activated(self.context, node_id): - msg = NOT_ACTIVATED_ERROR - self.context.prompt.render_failure_message(msg) - return os.EX_USAGE - - if not repository_enabled(self.context, repo_id): - msg = BIND_FAILED_NOT_ENABLED - self.context.prompt.render_failure_message(msg) - return os.EX_USAGE - - if strategy not in constants.STRATEGIES: - msg = STRATEGY_NOT_SUPPORTED % dict(n=strategy, s=constants.STRATEGIES) - self.context.prompt.render_failure_message(msg) - return os.EX_DATAERR - - self.context.server.bind.bind( - node_id, - repo_id, - dist_id, - notify_agent=False, - binding_config=binding_config) - self.context.prompt.render_success_message(BIND_SUCCEEDED) - warning = BIND_WARNING % dict(r=repo_id) - self.context.prompt.render_warning_message(warning) - - -class NodeUnbindCommand(BindingCommand): - - def __init__(self, context): - super(NodeUnbindCommand, self).__init__(UNBIND_NAME, UNBIND_DESC, self.run) - self.add_option(OPTION_REPO_ID) - self.context = context - - def run(self, **kwargs): - - repo_id = kwargs[OPTION_REPO_ID.keyword] - node_id = load_consumer_id(self.context) - dist_id = constants.HTTP_DISTRIBUTOR - - try: - self.context.server.bind.unbind(node_id, repo_id, dist_id) - self.context.prompt.render_success_message(UNBIND_SUCCEEDED) - warning = UNBIND_WARNING % dict(r=repo_id) - self.context.prompt.render_warning_message(warning) - except NotFoundException, e: - unhandled = self.missing_resources(self.context.prompt, e) - for _id, _type in unhandled: - if _type == 'bind_id': - msg = NOT_BOUND_NOTHING_DONE - self.context.prompt.render_failure_message(msg) - else: - raise - return os.EX_DATAERR diff --git a/nodes/extensions/consumer/setup.py b/nodes/extensions/consumer/setup.py deleted file mode 100644 index 18b2764298..0000000000 --- a/nodes/extensions/consumer/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='pulp_node_consumer_extensions', - version='2.11a1', - license='GPLv2+', - packages=find_packages(exclude=['test', 'test.*']), - author='Pulp Team', - author_email='pulp-list@redhat.com', - entry_points={ - 'pulp.extensions.consumer': [ - 'repo_admin = pulp_node.extensions.consumer.commands:initialize', - ] - } -) diff --git a/nodes/parent/etc/httpd/conf.d/pulp_nodes.conf b/nodes/parent/etc/httpd/conf.d/pulp_nodes.conf deleted file mode 100644 index 295e0369ca..0000000000 --- a/nodes/parent/etc/httpd/conf.d/pulp_nodes.conf +++ /dev/null @@ -1,46 +0,0 @@ -# Apache configuration file for pulp web services and repositories -# -# Copyright © 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public License, -# version 2 (GPLv2). There is NO WARRANTY for this software, express or -# implied, including the implied warranties of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 -# along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. -# -# Red Hat trademarks are not licensed under GPLv2. No permission is -# granted to use or replicate Red Hat trademarks that are incorporated -# in this software or its documentation. - -# -- HTTP Repositories --------- - -Alias /pulp/nodes/http/repos /var/www/pulp/nodes/http/repos - - - Options FollowSymLinks Indexes - - -# -- HTTPS Repositories --------- - -Alias /pulp/nodes/https/repos /var/www/pulp/nodes/https/repos - - - Options FollowSymLinks Indexes - SSLRequireSSL - SSLVerifyClient require - SSLVerifyDepth 5 - SSLOptions +FakeBasicAuth - SSLRequire %{SSL_CLIENT_S_DN_O} eq "PULP" and %{SSL_CLIENT_S_DN_OU} eq "NODES" - - -Alias /pulp/nodes/content /var/www/pulp/nodes/content - - - Options FollowSymLinks Indexes - SSLRequireSSL - SSLVerifyClient require - SSLVerifyDepth 5 - SSLOptions +FakeBasicAuth - SSLRequire %{SSL_CLIENT_S_DN_O} eq "PULP" and %{SSL_CLIENT_S_DN_OU} eq "NODES" - diff --git a/nodes/parent/etc/pulp/server/plugins.conf.d/nodes/distributor/http.conf b/nodes/parent/etc/pulp/server/plugins.conf.d/nodes/distributor/http.conf deleted file mode 100644 index 854878c0c8..0000000000 --- a/nodes/parent/etc/pulp/server/plugins.conf.d/nodes/distributor/http.conf +++ /dev/null @@ -1,15 +0,0 @@ -{ - "protocol":"https", - "http":{ - "alias":[ - "/pulp/nodes/http/repos", - "/var/www/pulp/nodes/http/repos" - ] - }, - "https":{ - "alias":[ - "/pulp/nodes/https/repos", - "/var/www/pulp/nodes/https/repos" - ] - } -} diff --git a/nodes/parent/pulp_node/__init__.py b/nodes/parent/pulp_node/__init__.py deleted file mode 100644 index b36383a610..0000000000 --- a/nodes/parent/pulp_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/nodes/parent/pulp_node/distributors/__init__.py b/nodes/parent/pulp_node/distributors/__init__.py deleted file mode 100644 index 32466f8d26..0000000000 --- a/nodes/parent/pulp_node/distributors/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/nodes/parent/pulp_node/distributors/http/__init__.py b/nodes/parent/pulp_node/distributors/http/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/nodes/parent/pulp_node/distributors/http/distributor.py b/nodes/parent/pulp_node/distributors/http/distributor.py deleted file mode 100644 index c76cfff392..0000000000 --- a/nodes/parent/pulp_node/distributors/http/distributor.py +++ /dev/null @@ -1,259 +0,0 @@ -# -*- coding: utf-8 -*- - -from gettext import gettext as _ -from logging import getLogger -import os - -from pulp.plugins.distributor import Distributor -from pulp.server.db import model -from pulp.server.config import config as pulp_conf -from pulp.server.compat import json - -from pulp_node import constants -from pulp_node import pathlib -from pulp_node.conduit import NodesConduit -from pulp_node.distributors.http.publisher import HttpPublisher - - -_logger = getLogger(__name__) - -PROPERTY_MISSING = _('Missing required configuration property: %(p)s') -PROPERTY_INVALID = _('Property %(p)s must be: %(v)s') - -CONFIGURATION_PATH = '/etc/pulp/server/plugins.conf.d/nodes/distributor/http.conf' - - -def entry_point(): - """ - Entry point that pulp platform uses to load the distributor. - :return: distributor class and its configuration. - :rtype: Distributor, dict - """ - with open(CONFIGURATION_PATH) as fp: - return NodesHttpDistributor, json.load(fp) - - -class NodesHttpDistributor(Distributor): - """ - The (nodes) distributor - """ - - @classmethod - def metadata(cls): - return { - 'id': constants.HTTP_DISTRIBUTOR, - 'display_name': 'Pulp Nodes HTTP Distributor', - 'types': ['node'] - } - - def validate_config(self, repo, config, config_conduit): - """ - Layout: - { - protocol : (http|https|file), - http : { - alias : [url, directory] - }, - https : { - alias : [url, directory], - ssl (optional) : { - ca_cert : , - client_cert : - verify : - } - } - } - """ - key = constants.PROTOCOL_KEYWORD - protocol = config.get(key) - valid_protocols = ('http', 'https', 'file') - if not protocol: - return (False, PROPERTY_MISSING % {'p': key}) - if protocol not in valid_protocols: - return (False, PROPERTY_INVALID % {'p': key, 'v': valid_protocols}) - for key in ('http', 'https'): - section = config.get(key) - if not section: - return (False, PROPERTY_MISSING % {'p': key}) - key = (key, 'alias') - alias = section.get(key[1]) - if not alias: - return (False, PROPERTY_MISSING % {'p': '.'.join(key)}) - return (True, None) - - def publish_repo(self, repo, conduit, config): - """ - Publishes the given repository. - While this call may be implemented using multiple threads, its execution - from the Pulp server's standpoint should be synchronous. This call should - not return until the publish is complete. - - It is not expected that this call be atomic. Should an error occur, it - is not the responsibility of the distributor to rollback any changes - that have been made. - - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - :param conduit: provides access to relevant Pulp functionality - :type conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit - :param config: plugin configuration - :type config: pulp.plugins.config.PluginConfiguration - :return: report describing the publish run - :rtype: pulp.plugins.model.PublishReport - """ - nodes_conduit = NodesConduit() - units = nodes_conduit.get_units(repo.id) - with self.publisher(repo, config) as publisher: - publisher.publish(units) - publisher.commit() - details = dict(unit_count=len(units)) - return conduit.build_success_report('succeeded', details) - - def publisher(self, repo, config): - """ - Get a configured publisher. - :param repo: A repository. - :type repo: pulp.plugins.model.Repository - :param config: plugin configuration - :type config: pulp.plugins.config.PluginConfiguration - :return: The configured publisher. - """ - protocol = config.get(constants.PROTOCOL_KEYWORD) - host = pulp_conf.get('server', 'server_name') - section = config.get(protocol) - alias = section.get('alias') - base_url = '://'.join((protocol, host)) - repo_publish_dir = self._get_publish_dir(repo.id, config) - return HttpPublisher(base_url, alias, repo.id, repo_publish_dir) - - def cancel_publish_repo(self): - pass - - def create_consumer_payload(self, repo, config, binding_config): - """ - Called when a consumer binds to a repository using this distributor. - This call should return a dictionary describing all data the consumer - will need to access the repository. The contents will vary wildly - depending on the method the repository is published, but examples - of returned data includes authentication information, location of the - repository (e.g. URL), and data required to verify the contents - of the published repository. - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration - :param binding_config: The configuration stored on the binding. - :type binding_config: dict - :return: dictionary of relevant data - :rtype: dict - """ - payload = {} - self._add_repository(repo.id, payload) - self._add_importers(repo, config, binding_config or {}, payload) - self._add_distributors(repo.id, payload) - return payload - - def distributor_removed(self, repo, config): - """ - Called when a distributor of this type is removed from a repository. - - This will delete any published node data from the filesystem. - - :param repo: metadata describing the repository - :type repo: pulp.plugins.model.Repository - :param config: plugin config - :type config: pulp.plugins.config.PluginCallConfiguration - """ - _logger.debug(_('removing published node data for repo %s' % repo.id)) - repo_publish_path = self._get_publish_dir(repo.id, config) - os.system('rm -rf %s' % repo_publish_path) - - def _get_publish_dir(self, repo_id, config): - """ - generate the full path where the given repo should be published - - :param repo_id: unique ID for the repository - :type repo_id: str - :param config: plugin config - :type config: pulp.plugins.config.PluginCallConfiguration - - :return: full path to the directory where this repo's data - should be published - :rtype: str - """ - protocol = config.get(constants.PROTOCOL_KEYWORD) - section = config.get(protocol) - url, publish_path = section.get('alias') - return os.path.join(publish_path, repo_id) - - def _add_repository(self, repo_id, payload): - """ - Add repository information to the payload. - :param repo_id: The repository ID. - :type repo_id: str - :param payload: The repository payload - :type payload: dict - """ - repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id) - # Pseudo serialize the repository object so that it can be used by a node. - payload['repository'] = {'id': repo_obj.repo_id, 'display_name': repo_obj.display_name, - 'description': repo_obj.description, 'notes': repo_obj.notes, - 'scratchpad': repo_obj.scratchpad} - - def _add_importers(self, repo, config, binding_config, payload): - """ - Add the nodes importer. - :param repo: A repo object. - :type repo: pulp.plugins.model.Repository - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration - :param binding_config: The configuration stored on the binding. - :type binding_config: dict - :param payload: The bind payload. - :type payload: dict - """ - conf = self._importer_conf(repo, config, binding_config) - importer = { - 'id': constants.HTTP_IMPORTER, - 'importer_type_id': constants.HTTP_IMPORTER, - 'config': conf, - } - payload['importers'] = [importer] - - def _importer_conf(self, repo, config, binding_config): - """ - Build the nodes importer configuration. - :param repo: A repo object. - :type repo: pulp.plugins.model.Repository - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration - :param binding_config: The configuration stored on the binding. - :type binding_config: dict - :return: The importer configuration. - :rtype: dict - """ - publisher = self.publisher(repo, config) - manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path()) - strategy = binding_config.get(constants.STRATEGY_KEYWORD, constants.DEFAULT_STRATEGY) - configuration = { - constants.STRATEGY_KEYWORD: strategy, - constants.MANIFEST_URL_KEYWORD: manifest_url, - } - return configuration - - def _add_distributors(self, repo_id, payload): - """ - Add repository distributors information to the payload. - :param repo_id: The repository ID. - :type repo_id: str - :param payload: The distributor(s) payload - :type payload: dict - """ - distributors = [] - for dist in model.Distributor.objects(repo_id=repo_id): - if dist.distributor_type_id in constants.ALL_DISTRIBUTORS: - continue - serialized = model.Distributor.SERIALIZER(dist).data - serialized.pop('_href') - distributors.append(serialized) - payload['distributors'] = distributors diff --git a/nodes/parent/pulp_node/distributors/http/publisher.py b/nodes/parent/pulp_node/distributors/http/publisher.py deleted file mode 100644 index 24c07655d4..0000000000 --- a/nodes/parent/pulp_node/distributors/http/publisher.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -from pulp_node import constants -from pulp_node import pathlib -from pulp_node.distributors.publisher import FilePublisher -from pulp_node.manifest import Manifest, MANIFEST_FILE_NAME - - -class HttpPublisher(FilePublisher): - """ - An HTTP publisher. - :ivar repo_id: A repository ID. - :type repo_id: str - :ivar alias: The httpd alias (base_url, directory) - :type alias: tuple(2) - """ - - def __init__(self, base_url, alias, repo_id, publish_path): - """ - :param base_url: The base URL. - :type base_url: str - :param alias: The httpd alias (base_url, publish_dir) - :type alias: tuple(2) - :param repo_id: A repository ID. - :type repo_id: str - """ - self.base_url = base_url - self.alias = alias - self.repo_id = repo_id - FilePublisher.__init__(self, publish_path) - - def publish(self, units): - """ - Publish the specified units. - Writes the units.json file and symlinks each of the - files associated to the unit.storage_path. - :param units: A list of units to publish. - :type units: iterable - :return: The path to the written manifest. - :rtype: str - """ - manifest_path = super(self.__class__, self).publish(units) - manifest = Manifest(manifest_path) - manifest.read() - base_url = pathlib.url_join(self.base_url, self.alias[0], self.repo_id) - details = {constants.BASE_URL: base_url} - manifest.published(details) - manifest.write() - return manifest_path - - def manifest_path(self): - """ - Get the relative URL path to the manifest. - :return: The path component of the URL. - :rtype: str - """ - return pathlib.join(self.alias[0], self.repo_id, MANIFEST_FILE_NAME) diff --git a/nodes/parent/pulp_node/distributors/publisher.py b/nodes/parent/pulp_node/distributors/publisher.py deleted file mode 100644 index 71d9db7420..0000000000 --- a/nodes/parent/pulp_node/distributors/publisher.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. - -import os -import tarfile - -from uuid import uuid4 -from tempfile import mkdtemp -from logging import getLogger - -from pulp_node import constants -from pulp_node import pathlib -from pulp_node.manifest import Manifest, UnitWriter - - -log = getLogger(__name__) - - -# --- utils -------------------------------------------------------- - -def tar_path(path): - """ - Construct the tarball path. - :param path: A path - :type path: str - :return: The modified path - """ - return path + '.TGZ' - - -def tar_dir(dir_path, tar_path, bufsize=65535): - """ - Tar up the directory at the specified path. - :param dir_path: The absolute path to a directory. - :type dir_path: str - :param tar_path: The target path. - :type tar_path: str - :param bufsize: The buffer size to be used. - :type bufsize: int - :return: The path to the tarball - """ - tb = tarfile.open(tar_path, 'w', bufsize=bufsize) - try: - for name in os.listdir(dir_path): - path = os.path.join(dir_path, name) - tb.add(path, arcname=name) - return tar_path - finally: - tb.close() - - -# --- publisher ---------------------------------------------------- - - -class Publisher(object): - """ - The publisher does the heavy lifting for nodes distributor. - """ - - def publish(self, units): - """ - Publish the specified units. - Writes the units.json file and symlinks each of the files associated - to the unit's "storage_path". - :param units: A list of units to publish. - :type units: list - :return: The manifest and list of links created. - :rtype: tuple(2) - """ - raise NotImplementedError() - - def commit(self): - """ - Commit publishing. - Supports 2-stage publishing. - """ - pass - - -class FilePublisher(Publisher): - """ - The file-based publisher. - :ivar publish_dir: full path to the publish_dir directory for this repository. - :type publish_dir: str - :ivar tmp_dir: The absolute path to the temporary publishing directory. - :type tmp_dir: str - :ivar staged: A flag indicating that publishing has been staged and needs commit. - :type staged: bool - """ - - def __init__(self, publish_dir): - """ - :param publish_dir: The publishing root directory for this repository - :type publish_dir: str - """ - self.publish_dir = publish_dir - self.tmp_dir = None - self.staged = False - - def publish(self, units): - """ - Publish the specified units. - Writes the units.json file and symlinks each of the files associated - to the unit.storage_path. Publishing is staged in a temporary directory and - must use commit() to make the publishing permanent. - :param units: A list of units to publish. - :type units: iterable - :return: The absolute path to the manifest. - :rtype: str - """ - # make the parent dir and a temp dir within it - parent_path = os.path.normpath(os.path.join(self.publish_dir, '../')) - pathlib.mkdir(parent_path) - self.tmp_dir = mkdtemp(dir=parent_path) - - with UnitWriter(self.tmp_dir) as writer: - for unit in units: - self.publish_unit(unit) - writer.add(unit) - manifest_id = str(uuid4()) - manifest = Manifest(self.tmp_dir, manifest_id) - manifest.units_published(writer) - manifest.write() - self.staged = True - return manifest.path - - def publish_unit(self, unit): - """ - Publish the file associated with the unit into the publish directory. - :param unit: A content unit. - :type unit: dict - """ - storage_path = unit.get(constants.STORAGE_PATH) - if not storage_path: - # not all units have associated files. - return unit, None - unit[constants.FILE_SIZE] = os.path.getsize(storage_path) - if not os.path.isdir(storage_path): - # unit does not have multiple files - return - relative_path = unit[constants.RELATIVE_PATH] - published_path = pathlib.join(self.tmp_dir, relative_path) - pathlib.mkdir(os.path.dirname(published_path)) - tar_dir(storage_path, tar_path(published_path)) - unit[constants.TARBALL_PATH] = tar_path(relative_path) - - def commit(self): - """ - Commit publishing. - Move the tmp_dir to the publish_dir. - """ - if not self.staged: - # nothing to commit - return - os.system('rm -rf %s' % self.publish_dir) - os.rename(self.tmp_dir, self.publish_dir) - self.staged = False - - def unstage(self): - """ - Un-stage publishing. - """ - os.system('rm -rf %s' % self.tmp_dir) - self.staged = False - - def __enter__(self): - return self - - def __exit__(self, *unused): - self.unstage() diff --git a/nodes/parent/pulp_node/profilers/__init__.py b/nodes/parent/pulp_node/profilers/__init__.py deleted file mode 100644 index 32466f8d26..0000000000 --- a/nodes/parent/pulp_node/profilers/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2013 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public -# License as published by the Free Software Foundation; either version -# 2 of the License (GPLv2) or (at your option) any later version. -# There is NO WARRANTY for this software, express or implied, -# including the implied warranties of MERCHANTABILITY, -# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should -# have received a copy of GPLv2 along with this software; if not, see -# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. diff --git a/nodes/parent/pulp_node/profilers/nodes.py b/nodes/parent/pulp_node/profilers/nodes.py deleted file mode 100644 index 56655b4c31..0000000000 --- a/nodes/parent/pulp_node/profilers/nodes.py +++ /dev/null @@ -1,107 +0,0 @@ -from gettext import gettext as _ - -from pulp.plugins.profiler import Profiler -from pulp.server.config import config as pulp_conf -from pulp.server.managers import factory as managers - -from pulp_node import constants -from pulp_node.config import read_config - - -def entry_point(): - """ - Entry point that pulp platform uses to load the profiler. - :return: profiler class and its configuration. - :rtype: Profiler, dict - """ - return NodeProfiler, {} - - -class NodeProfiler(Profiler): - - @classmethod - def metadata(cls): - """ - Plugin metadata. - :return: The plugin metadata. - :rtype: dict - """ - return { - 'id': constants.PROFILER_ID, - 'display_name': _('Nodes Profiler'), - 'types': [constants.NODE_SCOPE, constants.REPOSITORY_SCOPE] - } - - @staticmethod - def _inject_parent_settings(options): - """ - Inject the parent settings into the options. - Add the pulp server host and port information to the options. - Used by the agent handler to make REST calls back to the parent. - :param options: An options dictionary. - :type options: dict - """ - port = 443 - host = pulp_conf.get('server', 'server_name') - node_conf = read_config() - path = node_conf.main.node_certificate - with open(path) as fp: - node_certificate = fp.read() - settings = { - constants.HOST: host, - constants.PORT: port, - constants.NODE_CERTIFICATE: node_certificate, - } - options[constants.PARENT_SETTINGS] = settings - - @staticmethod - def _inject_strategy(consumer_id, options): - """ - Inject the node-level synchronization strategy. - :param consumer_id: The consumer ID. - :type consumer_id: str - :param options: The update options. - :type options: dict - """ - manager = managers.consumer_manager() - consumer = manager.get_consumer(consumer_id) - strategy = consumer['notes'].get(constants.STRATEGY_NOTE_KEY) - options[constants.STRATEGY_KEYWORD] = strategy - - def update_units(self, consumer, units, options, config, conduit): - """ - Translate the specified content units to be updated. - The specified content units are intended to be updated on the - specified consumer. It is requested that the profiler translate - the units as needed. If any of the content units cannot be translated, - an exception should be raised by the profiler. The translation itself, - depends on the content unit type and is completely up to the Profiler. - Translation into an empty list is not considered an error condition and - will be interpreted by the caller as meaning that no content needs to be - updated. - - @see: Unit Translation examples in class documentation. - - :param consumer: A consumer. - :type consumer: pulp.plugins.model.Consumer - - :param units: A list of content units to be updated. - :type units: list of: { type_id:, unit_key: } - - :param options: Update options; based on unit type. - :type options: dict - - :param config: plugin configuration - :type config: pulp.plugins.config.PluginCallConfiguration - - :param conduit: provides access to relevant Pulp functionality - :type conduit: pulp.plugins.conduits.profiler.ProfilerConduit - - :return: The translated units - :rtype: list of: { type_id:, unit_key: } - - :raises: InvalidUnitsRequested - if one or more of the units cannot be updated - """ - self._inject_parent_settings(options) - self._inject_strategy(consumer.id, options) - return units diff --git a/nodes/parent/setup.py b/nodes/parent/setup.py deleted file mode 100755 index 5a9fbbd6be..0000000000 --- a/nodes/parent/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='pulp_node_parent', - version='2.11a1', - license='GPLv2+', - packages=find_packages(), - author='Pulp Team', - author_email='pulp-list@redhat.com', - entry_points={ - 'pulp.distributors': [ - 'distributor = pulp_node.distributors.http.distributor:entry_point', - ], - 'pulp.profilers': [ - 'profiler = pulp_node.profilers.nodes:entry_point' - ] - } -) diff --git a/nodes/test/nodes_tests/__init__.py b/nodes/test/nodes_tests/__init__.py deleted file mode 100644 index 848fb57b8c..0000000000 --- a/nodes/test/nodes_tests/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from pulp.devel.unit.server import base as devel_base -from pulp.devel.unit.server.base import block_load_conf - -block_load_conf() - - -def setup(): - """ - Set up the database connection for the tests to use. - """ - devel_base.start_database_connection() - - -def teardown(): - """ - Drop the test database. - """ - devel_base.drop_database() diff --git a/nodes/test/nodes_tests/base.py b/nodes/test/nodes_tests/base.py deleted file mode 100644 index b0db1b4faf..0000000000 --- a/nodes/test/nodes_tests/base.py +++ /dev/null @@ -1,104 +0,0 @@ -from ConfigParser import SafeConfigParser -from unittest import TestCase -import logging -import mock -import os -import shutil -import unittest - -import okaara -import pymongo -from pulp.bindings.bindings import Bindings -from pulp.bindings.server import PulpConnection -from pulp.client.extensions.core import PulpCli, ClientContext, PulpPrompt -from pulp.client.extensions.exceptions import ExceptionHandler -from pulp.common.config import Config -from pulp.server.async import celery_instance -from pulp.server.config import config as pulp_conf -from pulp.server.db import connection -from pulp.server.logs import start_logging, stop_logging -from pulp.server.managers import factory as managers -from pulp.server.managers.auth.cert.cert_generator import SerialNumber - - -SerialNumber.PATH = '/tmp/sn.dat' - - -class ServerTests(unittest.TestCase): - - TMP_ROOT = '/tmp/pulp/nodes' - - @classmethod - def setUpClass(cls): - # This will make Celery tasks run synchronously - celery_instance.celery.conf.CELERY_ALWAYS_EAGER = True - - if not os.path.exists(cls.TMP_ROOT): - os.makedirs(cls.TMP_ROOT) - stop_logging() - path = os.path.join( - os.path.abspath(os.path.dirname(__file__)), - 'data', - 'pulp.conf') - pulp_conf.read(path) - start_logging() - storage_dir = pulp_conf.get('server', 'storage_dir') - if not os.path.exists(storage_dir): - os.makedirs(storage_dir) - shutil.rmtree(storage_dir + '/*', ignore_errors=True) - managers.initialize() - - @classmethod - def tearDownClass(cls): - name = pulp_conf.get('database', 'name') - db = pymongo.database.Database(connection._CONNECTION, name) - for name in db.collection_names(): - if name[:7] == 'system.': - continue - db.drop_collection(name) - - -class ClientTests(TestCase): - - def setUp(self): - TestCase.setUp(self) - self.config = SafeConfigParser() - path = os.path.join( - os.path.abspath(os.path.dirname(__file__)), - 'data', - 'client.conf') - self.config = Config(path) - self.server_mock = mock.Mock() - self.pulp_connection = \ - PulpConnection('', server_wrapper=self.server_mock) - self.bindings = Bindings(self.pulp_connection) - self.recorder = okaara.prompt.Recorder() - self.prompt = PulpPrompt(enable_color=False, output=self.recorder, record_tags=True) - self.logger = logging.getLogger('pulp') - self.exception_handler = ExceptionHandler(self.prompt, self.config) - self.context = ClientContext( - self.bindings, - self.config, - self.logger, - self.prompt, - self.exception_handler) - self.cli = PulpCli(self.context) - self.context.cli = self.cli - - -class Response: - - def __init__(self, code, body): - self.response_code = code - self.response_body = body - - -class Task: - - def __init__(self, task_id=0): - self.task_id = task_id - - -class TaskResult: - def __init__(self, task_id): - self.spawned_tasks = [Task(task_id)] diff --git a/nodes/test/nodes_tests/data/client.conf b/nodes/test/nodes_tests/data/client.conf deleted file mode 100644 index e1180b0fe4..0000000000 --- a/nodes/test/nodes_tests/data/client.conf +++ /dev/null @@ -1,2 +0,0 @@ -[output] -poll_frequency_in_seconds = .5 diff --git a/nodes/test/nodes_tests/data/distribution.tar b/nodes/test/nodes_tests/data/distribution.tar deleted file mode 100644 index 67fe51ebe29fe81a8b6296bd75ba0d91af18c0a8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10240 zcmeH}TMoh?5JhJx-GK~`vYe(`KWdC7wcWpfnh+CWr;o&s%MVSNq2wH<(Dg&p9%$F5 zLi65H=-4}z_i-pTVyz8poN++^(y7Pd+`ycBm@4+=uaeEr0RXcscX0N4n&H zy7gmo+H0Xva9!Kn`)41-q1e9sYp*<2YOe+5^TL0q|K$IeW7+aK3ZCyj{2w^@H}Jou zgVJ1C{kNr= self.cancel_on - - -class TestResponse: - - def __init__(self, http_code, body=None): - self.response_code = http_code - self.response_body = body - - -class TestRepo: - - def __init__(self, repo_id): - self.repo_id = repo_id - - -REPO_ID = 'foo' -TYPE_ID = 'random_importer' -TASK_ID = 'test_task' - -NODE_CERTIFICATE = """ - -----BEGIN RSA PRIVATE KEY----- - PULPROCKSPULPROCKSPULPROCKS - -----END RSA PRIVATE KEY----- - -----BEGIN CERTIFICATE----- - PULPROCKSPULPROCKSPULPROCKS - -----END CERTIFICATE----- -""" - -PARENT_SETTINGS = { - constants.HOST: 'pulp.redhat.com', - constants.PORT: 443, - constants.NODE_CERTIFICATE: NODE_CERTIFICATE, -} - - -class TestBase(TestCase): - - def request(self, cancel_on=0): - conduit = TestConduit(cancel_on) - progress = HandlerProgress(conduit) - summary = SummaryReport() - request = strategies.Request( - conduit=conduit, - progress=progress, - summary=summary, - bindings=[dict(repo_id=REPO_ID, details={})], - scope=constants.NODE_SCOPE, - options={constants.PARENT_SETTINGS: PARENT_SETTINGS} - ) - return request - - def test_abstract(self): - # Test - strategy = strategies.HandlerStrategy() - # Verify - self.assertRaises(NotImplementedError, strategy._synchronize, None) - - @patch('pulp_node.handlers.validation.Validator.validate', side_effect=ValueError()) - def test_synchronize_validation_exception(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.HandlerStrategy() - strategy.synchronize(request) - # Verify - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.CaughtException.ERROR_ID) - - @patch('pulp_node.handlers.validation.Validator.validate', - side_effect=error.ImporterNotInstalled(REPO_ID, TYPE_ID)) - def test_synchronize_validation_node_error(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.HandlerStrategy() - strategy.synchronize(request) - # Verify - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.ImporterNotInstalled.ERROR_ID) - - @patch('pulp_node.handlers.model.Repository.fetch', side_effect=ValueError()) - def test_synchronize_merge_exception(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.HandlerStrategy() - strategy.synchronize(request) - # Verify - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.CaughtException.ERROR_ID) - - @patch('pulp_node.handlers.model.Repository.fetch', side_effect=ValueError()) - def test_merge_repositories_exception(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.HandlerStrategy() - strategy._merge_repositories(request) - # Verify - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.CaughtException.ERROR_ID) - - @patch('pulp_node.handlers.model.Repository.fetch', - side_effect=error.RepoSyncRestError(REPO_ID, 401)) - def test_merge_repositories_node_error(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.HandlerStrategy() - strategy._merge_repositories(request) - # Verify - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.RepoSyncRestError.ERROR_ID) - self.assertEqual(request.summary.errors[0].details['http_code'], 401) - - @patch('pulp_node.handlers.model.Repository.fetch_all', return_value=[TestRepo('123')]) - @patch('pulp_node.handlers.model.Repository.delete', side_effect=ValueError()) - def test_delete_repositories_exception(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.HandlerStrategy() - strategy._delete_repositories(request) - # Verify - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.CaughtException.ERROR_ID) - - @patch('pulp_node.handlers.model.Repository.fetch_all', return_value=[TestRepo(123)]) - @patch('pulp_node.handlers.model.Repository.delete', - side_effect=error.CaughtException(ValueError())) - def test_delete_repositories_node_error(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.HandlerStrategy() - strategy._delete_repositories(request) - # Verify - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.CaughtException.ERROR_ID) - - @patch('pulp_node.handlers.model.Repository.fetch_all', return_value=[REPO_ID]) - def test_merge_repositories_cancelled(self, *unused): - # Setup - request = self.request(1) - # Test - strategy = strategies.HandlerStrategy() - strategy._merge_repositories(request) - # Verify - self.assertEqual(len(request.summary.errors), 0) - self.assertEqual(len(request.summary.repository), 1) - repository = request.summary.repository[REPO_ID] - self.assertEqual(repository.repo_id, REPO_ID) - self.assertEqual(repository.action, RepositoryReport.CANCELLED) - units = repository.units - self.assertEqual(units.added, 0) - self.assertEqual(units.updated, 0) - self.assertEqual(units.removed, 0) - - @patch('pulp_node.handlers.model.Repository.fetch_all', return_value=[TestRepo(REPO_ID)]) - def test_delete_repositories_cancelled(self, *unused): - # Setup - request = self.request(1) - # Test - strategy = strategies.HandlerStrategy() - strategy._delete_repositories(request) - # Verify - self.assertEqual(len(request.summary.errors), 0) - self.assertEqual(len(request.summary.repository), 1) - repository = request.summary.repository[REPO_ID] - self.assertEqual(repository.repo_id, REPO_ID) - self.assertEqual(repository.action, RepositoryReport.CANCELLED) - units = repository.units - self.assertEqual(units.added, 0) - self.assertEqual(units.updated, 0) - self.assertEqual(units.removed, 0) - - @patch('pulp.bindings.repository.RepositoryActionsAPI.sync', - return_value=TestResponse(202, TestReport(TASK_ID))) - @patch('pulp.bindings.tasks.TasksAPI.cancel_task', - return_value=TestResponse(200)) - def test_model_repo_sync_cancelled(self, mock_cancel, *unused): - # Setup - conduit = TestConduit(1) - # Test - repository = Repository(REPO_ID) - options = {constants.PARENT_SETTINGS: PARENT_SETTINGS} - repository.run_synchronization(None, conduit.cancelled, options) - # Verify - mock_cancel.assert_called_with(TASK_ID) - - def test_strategy_factory(self): - for name, strategy in strategies.STRATEGIES.items(): - self.assertEqual(strategies.find_strategy(name), strategy) - self.assertRaises(strategies.StrategyUnsupported, strategies.find_strategy, '---') diff --git a/nodes/test/nodes_tests/test_importer_strategies.py b/nodes/test/nodes_tests/test_importer_strategies.py deleted file mode 100644 index d679e83467..0000000000 --- a/nodes/test/nodes_tests/test_importer_strategies.py +++ /dev/null @@ -1,263 +0,0 @@ -import os -import shutil -from tempfile import mkdtemp -from unittest import TestCase -from uuid import uuid4 - -from mock import Mock, patch - -from pulp.plugins.model import Unit -from pulp.server.config import config as pulp_conf - -from pulp_node import constants, error -from pulp_node.importers import strategies -from pulp_node.importers.inventory import UnitInventory -from pulp_node.importers.reports import SummaryReport, ProgressListener -from pulp_node.reports import RepositoryProgress - - -class TestConduit: - - def get_units(self): - return [ - Unit('T', {1: 1}, {2: 2}, 'path_1'), - Unit('T', {1: 2}, {2: 2}, 'path_2'), - Unit('T', {1: 3}, {2: 2}, 'path_3'), - ] - - save_unit = Mock() - remove_unit = Mock() - set_progress = Mock() - - -class CancelEvent(object): - - def __init__(self, on_call): - self.on_call = on_call - self.call_count = 0 - - def isSet(self): - self.call_count += 1 - return self.on_call and self.call_count >= self.on_call - - -class TestRepo(object): - - def __init__(self, repo_id, working_dir): - self.id = repo_id - self.working_dir = working_dir - - -class TestManifest: - - def __init__(self, units): - self.units = [(u, TestUnitRef(u)) for u in units] - self.publishing_details = {constants.BASE_URL: BASE_URL} - - def get_units(self): - return self.units - - def fetch(self): - pass - - def fetch_units(self): - pass - - -class TestUnitRef: - - def __init__(self, unit): - self.unit = unit - - def fetch(self): - return self.unit - - -REPO_ID = 'foo' -BASE_URL = 'file://' -DOWNLOADER_ERROR_REPORT = dict(response_code=401, message='go fish') -MANIFEST_ERROR = error.ManifestDownloadError('http://redhat.com/manifest', DOWNLOADER_ERROR_REPORT) -UNIT_ERROR = error.UnitDownloadError('http://redhat.com/unit', REPO_ID, DOWNLOADER_ERROR_REPORT) - - -class TestBase(TestCase): - - @classmethod - def setUpClass(cls): - path = os.path.join( - os.path.abspath(os.path.dirname(__file__)), - 'data', - 'pulp.conf') - pulp_conf.read(path) - - def setUp(self): - super(TestBase, self).setUp() - self.tmp_dir = mkdtemp() - - def tearDown(self): - super(TestBase, self).tearDown() - shutil.rmtree(self.tmp_dir, ignore_errors=True) - - def request(self, cancel_on=0): - conduit = TestConduit() - progress = RepositoryProgress(REPO_ID, ProgressListener(conduit)) - summary = SummaryReport() - cancel_event = CancelEvent(cancel_on) - request = strategies.Request( - cancel_event, - conduit=conduit, - config={}, - downloader=Mock(), - progress=progress, - summary=summary, - repo=TestRepo(REPO_ID, self.tmp_dir) - ) - return request - - def test_abstract(self): - # Test - strategy = strategies.ImporterStrategy() - # Verify - self.assertRaises(NotImplementedError, strategy._synchronize, None) - - @patch('pulp_node.importers.strategies.ImporterStrategy._unit_inventory') - @patch('pulp_node.importers.strategies.ImporterStrategy._synchronize', side_effect=ValueError()) - def test_synchronize_catch_unknown_exception(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.ImporterStrategy() - strategy.synchronize(request) - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.CaughtException.ERROR_ID) - - @patch('pulp_node.importers.strategies.ImporterStrategy._unit_inventory') - @patch('pulp_node.importers.strategies.ImporterStrategy._synchronize', side_effect=UNIT_ERROR) - def test_synchronize_catch_node_error(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.ImporterStrategy() - strategy.synchronize(request) - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.UnitDownloadError.ERROR_ID) - - @patch('pulp_node.importers.strategies.ImporterStrategy._unit_inventory') - @patch.object(TestConduit, 'save_unit', ValueError()) - def test_add_unit_exception(self, *unused): - # Setup - request = self.request() - # Test - unit = dict(unit_id='abc', type_id='T', unit_key={}, metadata={}) - strategy = strategies.ImporterStrategy() - strategy.add_unit(request, unit) - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.AddUnitError.ERROR_ID) - - @patch('pulp_node.importers.strategies.ImporterStrategy._unit_inventory') - @patch.object(TestConduit, 'remove_unit', ValueError()) - def test_delete_units_exception(self, *unused): - # Setup - request = self.request() - # Test - unit = dict(unit_id='abc', type_id='T', unit_key={}, metadata={}) - inventory = UnitInventory(BASE_URL, [], [unit]) - strategy = strategies.ImporterStrategy() - strategy._delete_units(request, inventory) - self.assertEqual(len(request.summary.errors), 1) - self.assertEqual(request.summary.errors[0].error_id, error.DeleteUnitError.ERROR_ID) - - @patch('pulp_node.conduit.NodesConduit.get_units', side_effect=ValueError()) - def test_get_child_units_exception(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.ImporterStrategy() - self.assertRaises(error.GetChildUnitsError, strategy._unit_inventory, request) - - @patch('pulp_node.conduit.NodesConduit.get_units', return_value=[]) - @patch('pulp_node.manifest.RemoteManifest.fetch', side_effect=ValueError()) - def test_get_parent_units_exception(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.ImporterStrategy() - self.assertRaises(error.GetParentUnitsError, strategy._unit_inventory, request) - - @patch('pulp_node.conduit.NodesConduit.get_units', return_value=[]) - @patch('pulp_node.manifest.RemoteManifest.fetch', side_effect=MANIFEST_ERROR) - def test_get_parent_units_manifest_error(self, *unused): - # Setup - request = self.request() - # Test - strategy = strategies.ImporterStrategy() - self.assertRaises(error.ManifestDownloadError, strategy._unit_inventory, request) - - @patch('pulp_node.importers.strategies.ImporterStrategy.add_unit') - def test_cancel_at_add_units(self, mock_add_unit): - # Setup - request = self.request(1) - request.downloader.download = Mock() - unit = dict(unit_id='abc', type_id='T', unit_key={}, metadata={}) - units = [unit] - manifest = TestManifest(units) - inventory = UnitInventory(BASE_URL, manifest.get_units(), []) - # Test - strategy = strategies.ImporterStrategy() - strategy._add_units(request, inventory) - self.assertEqual(request.cancel_event.call_count, 1) - self.assertEqual(mock_add_unit.call_count, 0) - - def test_cancel_at_delete_units(self): - # Setup - request = self.request(1) - unit = dict(unit_id='abc', type_id='T', unit_key={}, metadata={}) - inventory = UnitInventory(BASE_URL, [], [unit]) - request.conduit.remove_unit = Mock() - # Test - strategy = strategies.ImporterStrategy() - strategy._delete_units(request, inventory) - self.assertEqual(request.cancel_event.call_count, 1) - request.conduit.remove_unit.assert_not_called() - - @patch('pulp.server.content.sources.container.ContentContainer.download') - def test_cancel_just_before_downloading(self, mock_download): - # Setup - unit_id = str(uuid4()) - request = self.request(2) - request.downloader.download = Mock() - unit = dict( - unit_id=unit_id, - type_id='T', - unit_key={}, - metadata={}, - storage_path=os.path.join(self.tmp_dir, unit_id), - relative_path=os.path.join(self.tmp_dir, 'testing', unit_id)) - units = [unit] - manifest = TestManifest(units) - inventory = UnitInventory(BASE_URL, manifest.get_units(), []) - # Test - strategy = strategies.ImporterStrategy() - strategy._add_units(request, inventory) - self.assertEqual(request.cancel_event.call_count, 2) - self.assertFalse(mock_download.called) - - def test_needs_update(self): - # Setup - path = os.path.join(self.tmp_dir, 'unit_1') - with open(path, 'w+') as fp: - fp.write('123') - size = os.path.getsize(path) - strategy = strategies.ImporterStrategy() - # Test - unit = {constants.STORAGE_PATH: path, constants.FILE_SIZE: size} - self.assertFalse(strategy._needs_download(unit)) - unit = {constants.STORAGE_PATH: '&&&&&&&', constants.FILE_SIZE: size} - self.assertTrue(strategy._needs_download(unit)) - unit = {constants.STORAGE_PATH: path, constants.FILE_SIZE: size + 1} - self.assertTrue(strategy._needs_download(unit)) - - def test_strategy_factory(self): - for name, strategy in strategies.STRATEGIES.items(): - self.assertEqual(strategies.find_strategy(name), strategy) - self.assertRaises(strategies.StrategyUnsupported, strategies.find_strategy, '---') diff --git a/nodes/test/nodes_tests/test_manifest.py b/nodes/test/nodes_tests/test_manifest.py deleted file mode 100644 index 5a9e2db692..0000000000 --- a/nodes/test/nodes_tests/test_manifest.py +++ /dev/null @@ -1,122 +0,0 @@ -import gzip -import json -import os -import shutil -import tempfile -from unittest import TestCase - -from nectar.config import DownloaderConfig -from nectar.downloaders.local import LocalFileDownloader - -from pulp_node import manifest - - -class TestManifest(TestCase): - - NUM_UNITS = 10 - MANIFEST_ID = '123' - - def setUp(self): - self.tmp_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tmp_dir) - - def verify(self, units_out, units_in): - self.assertEqual(len(units_out), self.NUM_UNITS) - self.assertEqual(len(units_in), self.NUM_UNITS) - for i in range(0, len(units_out)): - for k, v in units_in[i].items(): - self.assertEqual(units_out[i][k], v) - - def test_validation(self): - # Setup - manifest_path = os.path.join(self.tmp_dir, manifest.MANIFEST_FILE_NAME) - m = manifest.Manifest(manifest_path, self.MANIFEST_ID) - # Test valid - self.assertTrue(m.is_valid()) - # Test version mismatch - m.version += 1 - self.assertFalse(m.is_valid()) - - def test_publishing(self): - # Setup - units = [] - manifest_path = os.path.join(self.tmp_dir, manifest.MANIFEST_FILE_NAME) - for i in range(0, self.NUM_UNITS): - unit = dict(unit_id=i, type_id='T', unit_key={}) - units.append(unit) - # Test - units_path = os.path.join(self.tmp_dir, manifest.UNITS_FILE_NAME) - writer = manifest.UnitWriter(units_path) - for u in units: - writer.add(u) - writer.close() - m = manifest.Manifest(manifest_path, self.MANIFEST_ID) - m.units_published(writer) - m.write() - # Verify - self.assertTrue(os.path.exists(manifest_path)) - with open(manifest_path) as fp: - manifest_in = json.load(fp) - self.assertEqual(m.id, manifest_in['id']) - self.assertEqual(m.units[manifest.UNITS_TOTAL], manifest_in['units'][manifest.UNITS_TOTAL]) - self.assertEqual(m.units[manifest.UNITS_TOTAL], writer.total_units) - self.assertEqual(m.units[manifest.UNITS_TOTAL], len(units)) - self.assertTrue(os.path.exists(manifest_path)) - self.assertTrue(os.path.exists(units_path)) - units_in = [] - fp = gzip.open(units_path) - while True: - json_unit = fp.readline() - if json_unit: - units_in.append(json.loads(json_unit)) - else: - break - fp.close() - self.verify(units, units_in) - - def test_round_trip(self): - # Setup - units = [] - manifest_path = os.path.join(self.tmp_dir, manifest.MANIFEST_FILE_NAME) - for i in range(0, self.NUM_UNITS): - unit = dict(unit_id=i, type_id='T', unit_key={}) - units.append(unit) - units_path = os.path.join(self.tmp_dir, manifest.UNITS_FILE_NAME) - writer = manifest.UnitWriter(units_path) - for u in units: - writer.add(u) - writer.close() - m = manifest.Manifest(manifest_path, self.MANIFEST_ID) - m.units_published(writer) - m.write() - # Test - cfg = DownloaderConfig() - downloader = LocalFileDownloader(cfg) - working_dir = os.path.join(self.tmp_dir, 'working_dir') - os.makedirs(working_dir) - path = os.path.join(self.tmp_dir, manifest.MANIFEST_FILE_NAME) - url = 'file://%s' % path - m = manifest.RemoteManifest(url, downloader, working_dir) - m.fetch() - m.fetch_units() - # Verify - self.assertTrue(m.is_valid()) - self.assertTrue(m.has_valid_units()) - units_in = [] - for unit, ref in m.get_units(): - units_in.append(unit) - _unit = ref.fetch() - self.assertEqual(unit, _unit) - self.verify(units, units_in) - # should already be unzipped - self.assertTrue(m.is_valid()) - self.assertTrue(m.has_valid_units()) - self.assertFalse(m.units_path().endswith('.gz')) - units_in = [] - for unit, ref in m.get_units(): - units_in.append(unit) - _unit = ref.fetch() - self.assertEqual(unit, _unit) - self.verify(units, units_in) diff --git a/nodes/test/nodes_tests/test_model.py b/nodes/test/nodes_tests/test_model.py deleted file mode 100644 index 2aff34c7af..0000000000 --- a/nodes/test/nodes_tests/test_model.py +++ /dev/null @@ -1,64 +0,0 @@ -import httplib - -from mock import patch, Mock - -from pulp.common.bundle import Bundle -from pulp.common.plugins import importer_constants - -from base import Response, ServerTests, TaskResult -from pulp_node import constants -from pulp_node.handlers.model import Repository - - -PULP_ID = 'pulp_1' -REPO_ID = 'repo_1' -MAX_BANDWIDTH = 12345 -MAX_CONCURRENCY = 54321 - -NODE_CERTIFICATE = """ - -----BEGIN RSA PRIVATE KEY----- - PULPROCKSPULPROCKSPULPROCKS - -----END RSA PRIVATE KEY----- - -----BEGIN CERTIFICATE----- - PULPROCKSPULPROCKSPULPROCKS - -----END CERTIFICATE----- -""" - -PARENT_SETTINGS = {constants.NODE_CERTIFICATE: NODE_CERTIFICATE} - - -class TestModel(ServerTests): - - def setUp(self): - super(self.__class__, self).setUp() - - def tearDown(self): - super(self.__class__, self).tearDown() - - @patch('pulp_node.poller.TaskPoller.join') - @patch('pulp.bindings.repository.RepositoryActionsAPI.sync', - return_value=Response(httplib.ACCEPTED, TaskResult(0))) - @patch('pulp.agent.lib.conduit.Conduit.consumer_id') - def test_repository(self, *mocks): - # Setup - repository = Repository(REPO_ID) - progress = Mock() - cancelled = Mock(return_value=False) - # Test - options = { - constants.MAX_DOWNLOAD_CONCURRENCY_KEYWORD: MAX_CONCURRENCY, - constants.MAX_DOWNLOAD_BANDWIDTH_KEYWORD: MAX_BANDWIDTH, - constants.PARENT_SETTINGS: PARENT_SETTINGS, - } - repository.run_synchronization(progress, cancelled, options) - binding = mocks[1] - key, certificate = Bundle.split(NODE_CERTIFICATE) - expected_conf = { - importer_constants.KEY_SSL_VALIDATION: False, - importer_constants.KEY_MAX_DOWNLOADS: MAX_CONCURRENCY, - importer_constants.KEY_MAX_SPEED: MAX_BANDWIDTH, - importer_constants.KEY_SSL_CLIENT_KEY: key, - importer_constants.KEY_SSL_CLIENT_CERT: certificate, - } - # Verify - binding.assert_called_with(REPO_ID, expected_conf) diff --git a/nodes/test/nodes_tests/test_plugins.py b/nodes/test/nodes_tests/test_plugins.py deleted file mode 100644 index e6b3ca46ea..0000000000 --- a/nodes/test/nodes_tests/test_plugins.py +++ /dev/null @@ -1,949 +0,0 @@ -import os -import tempfile -import shutil -import random -import tarfile - -from copy import deepcopy -from unittest import TestCase - -from mock import Mock, patch -from base import ServerTests - -from nectar.downloaders.local import LocalFileDownloader -from nectar.config import DownloaderConfig - -from pulp_node.distributors.http.distributor import (NodesHttpDistributor, - entry_point as dist_entry_point) -from pulp_node.importers.http.importer import NodesHttpImporter, entry_point as imp_entry_point -from pulp_node.profilers.nodes import NodeProfiler, entry_point as profiler_entry_point -from pulp_node.handlers.handler import NodeHandler, RepositoryHandler - -from pulp.devel import mock_config -from pulp.plugins.loader import api as plugin_api -from pulp.plugins.types import database as unit_db -from pulp.server import config as pulp_conf -from pulp.server.db import connection -from pulp.server.db import model -from pulp.server.db.model.repository import RepoContentUnit -from pulp.server.db.model.consumer import Consumer, Bind -from pulp.server.db.model.content import ContentType -from pulp.plugins import model as plugin_model -from pulp.plugins.config import PluginCallConfiguration -from pulp.plugins.conduits.repo_publish import RepoPublishConduit -from pulp.plugins.conduits.repo_sync import RepoSyncConduit -from pulp.plugins.util.nectar_config import importer_config_to_nectar_config -from pulp.common.plugins import importer_constants -from pulp.common.config import Config -from pulp.server.managers import factory as managers -from pulp.server.content.sources.model import Request as DownloadRequest -from pulp.agent.lib.conduit import Conduit -from pulp_node.manifest import Manifest, RemoteManifest, MANIFEST_FILE_NAME, UNITS_FILE_NAME -from pulp_node.handlers.strategies import Mirror, Additive -from pulp_node import error -from pulp_node import constants -from pulp_node import pathlib - - -FAKE_DISTRIBUTOR = 'test_distributor' -FAKE_ID = 'fake_plugin_id' -FAKE_DISTRIBUTOR_CONFIG = {'A': 0} - -NODE_CERTIFICATE = """ - -----BEGIN RSA PRIVATE KEY----- - PULPROCKSPULPROCKSPULPROCKS - -----END RSA PRIVATE KEY----- - -----BEGIN CERTIFICATE----- - PULPROCKSPULPROCKSPULPROCKS - -----END CERTIFICATE----- -""" - -REPO_NAME = 'pulp-nodes' -REPO_DESCRIPTION = 'full of goodness' -REPO_NOTES = {'the answer to everything': 42} -REPO_SCRATCHPAD = {'a': 1, 'b': 2} - - -class Repository(object): - - def __init__(self, repo_id, working_dir=None): - self.id = repo_id - self.working_dir = working_dir - - -class FakeDistributor(object): - - @classmethod - def metadata(cls): - return { - 'id': FAKE_DISTRIBUTOR, - 'display_name': 'Fake Distributor', - 'types': ['node'] - } - - def validate_config(self, *unused): - return True, None - - def publish_repo(self, repo, conduit, config): - return conduit.build_success_report('succeeded', {}) - - def distributor_added(self, *unused): - pass - - -class TestStrategy: - - def __init__(self, tester, **options): - self.tester = tester - self.options = options - - def __call__(self): - self.tester.clean(**self.options) - return self._impl()() - - def _impl(self): - raise NotImplementedError() - - -class MirrorTestStrategy(TestStrategy): - - def _impl(self): - return Mirror - - -class AdditiveTestStrategy(TestStrategy): - - def _impl(self): - return Additive - - -class BadDownloadRequest(DownloadRequest): - - def __init__(self, *args, **kwargs): - DownloadRequest.__init__(self, *args, **kwargs) - self.url = 'http:/NOWHERE/FAIL_ME_%f' % random.random() - - -class AgentConduit(Conduit): - - def __init__(self, node_id=None): - self.node_id = node_id - - @property - def consumer_id(self): - return self.node_id - - -class PluginTestBase(ServerTests): - - REPO_ID = 'test-repo' - UNIT_TYPE_ID = 'notarealtype' - UNIT_ID = 'test_unit_%d' - UNIT_KEY = {'A': 'a', 'B': 'b', 'N': 0} - UNIT_METADATA = {'name': 'Elvis', 'age': 42} - TYPEDEF_ID = UNIT_TYPE_ID - NUM_UNITS = 10 - NUM_EXTRA_UNITS = 5 - EXTRA_REPO_IDS = ('extra_1', 'extra_2') - - PARENT_SETTINGS = { - constants.HOST: 'pulp.redhat.com', - constants.PORT: 443, - constants.NODE_CERTIFICATE: NODE_CERTIFICATE, - } - - @classmethod - def tmpdir(cls, role): - tmp_dir = tempfile.mkdtemp(dir=cls.TMP_ROOT, prefix=role) - return tmp_dir - - def setUp(self): - ServerTests.setUp(self) - self.parentfs = self.tmpdir('parent-') - self.childfs = self.tmpdir('child-') - self.alias = (self.parentfs, self.parentfs) - self.temp_dir = tempfile.mkdtemp() - Consumer.get_collection().remove() - Bind.get_collection().remove() - model.Repository.objects.delete() - model.Distributor.objects.delete() - model.Importer.objects.delete() - RepoContentUnit.get_collection().remove() - unit_db.clean() - self.define_plugins() - plugin_api._create_manager() - imp_conf = dict(strategy=constants.MIRROR_STRATEGY) - plugin_api._MANAGER.importers.add_plugin( - constants.HTTP_IMPORTER, NodesHttpImporter, imp_conf) - plugin_api._MANAGER.distributors.add_plugin( - constants.HTTP_DISTRIBUTOR, NodesHttpDistributor, {}) - plugin_api._MANAGER.distributors.add_plugin( - FAKE_DISTRIBUTOR, FakeDistributor, FAKE_DISTRIBUTOR_CONFIG) - plugin_api._MANAGER.profilers.add_plugin(constants.PROFILER_ID, NodeProfiler, {}) - - def tearDown(self): - ServerTests.tearDown(self) - shutil.rmtree(self.parentfs) - shutil.rmtree(self.childfs) - shutil.rmtree(self.temp_dir) - Consumer.get_collection().remove() - Bind.get_collection().remove() - model.Repository.objects.delete() - model.Distributor.objects.delete() - model.Importer.objects.delete() - RepoContentUnit.get_collection().remove() - unit_db.clean() - - def define_plugins(self): - collection = ContentType.get_collection() - collection.save(dict(id=self.TYPEDEF_ID, unit_key=self.UNIT_KEY.keys())) - - def populate(self): - # make content/ dir. - os.makedirs(os.path.join(self.parentfs, 'content')) - repository = model.Repository() - repository.repo_id = self.REPO_ID - repository.save() - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - units = self.add_units(0, self.NUM_UNITS) - self.units = units - - def node_configuration(self): - path = os.path.join(self.parentfs, 'node.crt') - with open(path, 'w+') as fp: - fp.write(NODE_CERTIFICATE) - node_conf = Config({'main': {constants.NODE_CERTIFICATE: path}}) - return node_conf.graph() - - def add_units(self, begin, end): - units = [] - storage_dir = os.path.join(pulp_conf.config.get('server', 'storage_dir'), 'content') - if not os.path.exists(storage_dir): - os.makedirs(storage_dir) - for n in range(begin, end): - unit_id = self.UNIT_ID % n - unit = dict(self.UNIT_KEY) - unit.update(self.UNIT_METADATA) - unit['N'] = n - # add unit file - storage_path = os.path.join(storage_dir, '.'.join((unit_id, self.UNIT_TYPE_ID))) - if n % 2 == 0: # even numbered has file associated - unit['_storage_path'] = storage_path - if n == 0: # 1st one is a directory of files - os.makedirs(storage_path) - dist_path = os.path.join(os.path.dirname(__file__), 'data/distribution.tar') - tb = tarfile.open(dist_path) - tb.extractall(path=storage_path) - tb.close() - else: - with open(storage_path, 'w+') as fp: - fp.write(unit_id) - # add unit - manager = managers.content_manager() - manager.add_content_unit( - self.UNIT_TYPE_ID, - unit_id, - unit) - manager = managers.repo_unit_association_manager() - # associate unit - manager.associate_unit_by_id( - self.REPO_ID, - self.UNIT_TYPE_ID, - unit_id) - units.append(unit) - return units - - def dist_conf(self): - return { - 'protocol': 'file', - 'http': {'alias': self.alias}, - 'https': {'alias': self.alias}, - 'file': {'alias': self.alias}, - } - - -class AgentHandlerTest(PluginTestBase): - - @patch('pulp_node.handlers.model.RepositoryBinding.fetch_all') - def test_node_handler_get_bindings_failed(self, mock_fetch): - # Setup - handler = NodeHandler({}) - mock_fetch.side_effect = error.GetBindingsError(500) - # Test - options = { - constants.PARENT_SETTINGS: self.PARENT_SETTINGS, - constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, - } - conduit = AgentConduit() - report = handler.update(conduit, [], options) - # Verify - details = report.details - errors = details['errors'] - self.assertFalse(report.succeeded) - self.assertEqual(len(errors), 1) - self.assertEqual(errors[0]['error_id'], error.GetBindingsError.ERROR_ID) - self.assertTrue(errors[0]['details']['http_code'], 500) - - @patch('pulp_node.handlers.model.RepositoryBinding.fetch', - side_effect=error.GetBindingsError(500)) - def test_repository_handler_get_bindings_failed(self, *unused): - # Setup - handler = RepositoryHandler({}) - # Test & Verify - options = { - constants.PARENT_SETTINGS: self.PARENT_SETTINGS, - constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, - } - self.assertRaises(error.GetBindingsError, handler.update, AgentConduit(), [], options) - - -# --- pulp plugin tests -------------------------------------------- - - -class TestProfiler(TestCase): - - def test_entry_point(self): - _class, conf = profiler_entry_point() - plugin = _class() - self.assertEqual(conf, {}) - self.assertTrue(isinstance(plugin, NodeProfiler)) - - @patch('pulp_node.profilers.nodes._') - def test_metadata(self, gettext): - self.assertEqual( - NodeProfiler.metadata(), - { - 'id': constants.PROFILER_ID, - 'display_name': gettext.return_value, - 'types': [constants.NODE_SCOPE, constants.REPOSITORY_SCOPE] - }) - - @patch('__builtin__.open') - @patch('pulp_node.profilers.nodes.read_config') - @patch('pulp_node.profilers.nodes.pulp_conf') - def test_inject_parent_settings(self, _pulp_conf, _read_config, _open): - host = 'test-host' - cert_path = '/path/cert' - - _pulp_conf.get.return_value = host - _read_config.return_value.main = Mock(node_certificate=cert_path) - - fp = Mock() - fp.__enter__ = Mock(return_value=fp) - fp.__exit__ = Mock() - _open.return_value = fp - - # test - options = {} - NodeProfiler._inject_parent_settings(options) - - # validation - _pulp_conf.get.assert_called_once_with('server', 'server_name') - _read_config.assert_called_once_with() - _open.assert_called_once_with(cert_path) - settings = options[constants.PARENT_SETTINGS] - self.assertEqual(settings[constants.HOST], host) - self.assertEqual(settings[constants.PORT], 443) - self.assertEqual( - settings[constants.NODE_CERTIFICATE], - _open.return_value.read.return_value) - - @patch('pulp_node.profilers.nodes.managers') - def test_inject_strategy(self, managers): - consumer_id = 'test-consumer' - strategy = 'test-strategy' - notes = {constants.STRATEGY_NOTE_KEY: strategy} - manager = Mock() - manager.get_consumer.return_value = {'notes': notes} - managers.consumer_manager.return_value = manager - - # test - options = {} - NodeProfiler._inject_strategy(consumer_id, options) - - # validation - manager.get_consumer.assert_called_once_with(consumer_id) - self.assertEqual(options[constants.STRATEGY_KEYWORD], strategy) - - def test_update_units(self): - consumer = Mock(id='123') - units = [1, 2, 3] - options = {} - config = Mock() - conduit = Mock() - - # test - profiler = NodeProfiler() - profiler._inject_parent_settings = Mock() - profiler._inject_strategy = Mock() - - # validation - _units = profiler.update_units(consumer, units, options, config, conduit) - profiler._inject_parent_settings.assert_called_once_with(options) - profiler._inject_strategy.assert_called_once_with(consumer.id, options) - self.assertEqual(_units, units) - - -class TestDistributor(PluginTestBase): - - VALID_CONFIGURATION = { - constants.PROTOCOL_KEYWORD: 'https', - 'http': { - 'alias': [ - '/pulp/nodes/http/repos', - '/var/www/pulp/nodes/http/repos' - ] - }, - 'https': { - 'alias': [ - '/pulp/nodes/https/repos', - '/var/www/pulp/nodes/https/repos' - ] - } - } - - PAYLOAD = { - 'repository': None, - 'distributors': [], - 'importers': [ - {'id': 'nodes_http_importer', - 'importer_type_id': 'nodes_http_importer', - 'config': { - 'manifest_url': 'file://localhost/%(tmp_dir)s/%(repo_id)s/manifest.json', - 'strategy': constants.ADDITIVE_STRATEGY - }, } - ] - } - - def test_entry_point(self): - repo = plugin_model.Repository(self.REPO_ID) - _class, conf = dist_entry_point() - plugin = _class() - plugin.validate_config(repo, conf, []) - - def test_metadata(self): - # Test - md = NodesHttpDistributor.metadata() - self.assertTrue(isinstance(md, dict)) - # Verify - self.assertTrue('node' in md['types']) - - def test_valid_config(self): - # Test - dist = NodesHttpDistributor() - repo = plugin_model.Repository(self.REPO_ID) - report = dist.validate_config(repo, self.VALID_CONFIGURATION, []) - # Verify - self.assertTrue(isinstance(report, tuple)) - self.assertTrue(len(report), 2) - self.assertTrue(isinstance(report[0], bool)) - self.assertTrue(report[0]) - self.assertEqual(report[1], None) - - def test_config_missing_protocol(self): - # Test - conf = deepcopy(self.VALID_CONFIGURATION) - del conf[constants.PROTOCOL_KEYWORD] - dist = NodesHttpDistributor() - repo = plugin_model.Repository(self.REPO_ID) - report = dist.validate_config(repo, {}, []) - # Verify - self.assertTrue(isinstance(report, tuple)) - self.assertTrue(len(report), 2) - self.assertTrue(isinstance(report[0], bool)) - self.assertFalse(report[0]) - self.assertFalse(report[1] is None) - - def test_config_missing_http_protocol(self): - # Test - conf = deepcopy(self.VALID_CONFIGURATION) - for protocol in ('http', 'https'): - del conf[protocol] - dist = NodesHttpDistributor() - repo = plugin_model.Repository(self.REPO_ID) - report = dist.validate_config(repo, {}, []) - # Verify - self.assertTrue(isinstance(report, tuple)) - self.assertTrue(len(report), 2) - self.assertTrue(isinstance(report[0], bool)) - self.assertFalse(report[0]) - self.assertFalse(report[1] is None) - - def test_config_missing_alias(self): - # Test - conf = deepcopy(self.VALID_CONFIGURATION) - del conf['https']['alias'] - dist = NodesHttpDistributor() - repo = plugin_model.Repository(self.REPO_ID) - report = dist.validate_config(repo, {}, []) - # Verify - self.assertTrue(isinstance(report, tuple)) - self.assertTrue(len(report), 2) - self.assertTrue(isinstance(report[0], bool)) - self.assertFalse(report[0]) - self.assertFalse(report[1] is None) - - def test_config_missing_invalid_alias(self): - # Test - conf = deepcopy(self.VALID_CONFIGURATION) - conf['https']['alias'] = None - dist = NodesHttpDistributor() - repo = plugin_model.Repository(self.REPO_ID) - report = dist.validate_config(repo, {}, []) - # Verify - self.assertTrue(isinstance(report, tuple)) - self.assertTrue(len(report), 2) - self.assertTrue(isinstance(report[0], bool)) - self.assertFalse(report[0]) - self.assertFalse(report[1] is None) - - @patch('pulp_node.distributors.http.distributor.model.Repository') - def test_payload(self, mock_repo_model): - mock_repo = mock_repo_model.objects.get_repo_or_missing_resource.return_value - self.populate() - - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - dist = NodesHttpDistributor() - repo = Repository(self.REPO_ID) - payload = dist.create_consumer_payload(repo, self.dist_conf(), {}) - f = open('/tmp/payload', 'w+') - f.write(repr(payload['importers'])) - f.close() - - # Verify - distributors = payload['distributors'] - importers = payload['importers'] - repository = payload['repository'] - self.assertTrue(isinstance(distributors, list)) - self.assertTrue(isinstance(importers, list)) - self.assertEqual(repository['id'], mock_repo.repo_id) - self.assertEqual(repository['display_name'], mock_repo.display_name) - self.assertEqual(repository['description'], mock_repo.description) - self.assertEqual(repository['notes'], mock_repo.notes) - self.assertEqual(repository['scratchpad'], mock_repo.scratchpad) - self.assertTrue(len(importers), 1) - for key in ('id', 'importer_type_id', 'config'): - self.assertTrue(key in importers[0]) - for key in (constants.MANIFEST_URL_KEYWORD, constants.STRATEGY_KEYWORD): - self.assertTrue(key in importers[0]['config']) - - @patch('pulp.server.managers.repo.unit_association.repo_controller') - def test_publish(self, mock_repo_ctrl): - # Setup - self.populate() - - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - # Test - dist = NodesHttpDistributor() - repo = Repository(self.REPO_ID) - conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) - dist.publish_repo(repo, conduit, self.dist_conf()) - # Verify - conf = DownloaderConfig() - downloader = LocalFileDownloader(conf) - pub = dist.publisher(repo, self.dist_conf()) - url = pathlib.url_join(pub.base_url, pub.manifest_path()) - working_dir = self.childfs - manifest = RemoteManifest(url, downloader, working_dir) - manifest.fetch() - manifest.fetch_units() - units = [u for u, r in manifest.get_units()] - self.assertEqual(len(units), self.NUM_UNITS) - for n in range(0, self.NUM_UNITS): - unit = units[n] - created = self.units[n] - for p, v in unit['unit_key'].items(): - self.assertEqual(created[p], v) - for p, v in unit['metadata'].items(): - if p in ('_ns', '_content_type_id'): - continue - self.assertEqual(created[p], v) - self.assertEqual(created.get('_storage_path'), unit['storage_path']) - self.assertEqual(unit['type_id'], self.UNIT_TYPE_ID) - - def test_get_publish_dir(self): - dist = NodesHttpDistributor() - repo = plugin_model.Repository(self.REPO_ID) - config = PluginCallConfiguration(self.VALID_CONFIGURATION, {}) - - ret = dist._get_publish_dir(repo.id, config) - - self.assertEqual(ret, '/var/www/pulp/nodes/https/repos/%s' % repo.id) - - @patch('os.system', spec_set=True) - def test_distributor_removed(self, mock_system): - dist = NodesHttpDistributor() - repo = plugin_model.Repository(self.REPO_ID) - config = PluginCallConfiguration(self.VALID_CONFIGURATION, {}) - - dist.distributor_removed(repo, config) - - mock_system.assert_called_once_with('rm -rf /var/www/pulp/nodes/https/repos/%s' % repo.id) - - -class ImporterTest(PluginTestBase): - - VALID_CONFIGURATION = { - constants.STRATEGY_KEYWORD: constants.DEFAULT_STRATEGY, - constants.MANIFEST_URL_KEYWORD: 'http://redhat.com', - } - - def test_entry_point(self): - repo = plugin_model.Repository(self.REPO_ID) - _class, conf = imp_entry_point() - plugin = _class() - plugin.validate_config(repo, conf) - - def test_metadata(self): - # Test - md = NodesHttpImporter.metadata() - # Verify - self.assertTrue(isinstance(md, dict)) - self.assertTrue('node' in md['types']) - self.assertTrue('repository' in md['types']) - - def test_valid_config(self): - # Test - importer = NodesHttpImporter() - repo = plugin_model.Repository(self.REPO_ID) - report = importer.validate_config(repo, self.VALID_CONFIGURATION) - # Verify - self.assertTrue(isinstance(report, tuple)) - self.assertTrue(len(report), 2) - self.assertTrue(isinstance(report[0], bool)) - self.assertTrue(report[0]) - self.assertEqual(len(report[1]), 0) - - def test_config_missing_properties(self): - # Test - importer = NodesHttpImporter() - repo = plugin_model.Repository(self.REPO_ID) - report = importer.validate_config(repo, {}) - # Verify - self.assertTrue(isinstance(report, tuple)) - self.assertTrue(len(report), 2) - self.assertTrue(isinstance(report[0], bool)) - self.assertFalse(report[0]) - self.assertTrue(len(report[1]), 3) - - def test_invalid_strategy(self): - # Test - conf = deepcopy(self.VALID_CONFIGURATION) - conf[constants.STRATEGY_KEYWORD] = '---', - importer = NodesHttpImporter() - repo = plugin_model.Repository(self.REPO_ID) - report = importer.validate_config(repo, conf) - # Verify - self.assertTrue(isinstance(report, tuple)) - self.assertTrue(len(report), 2) - self.assertTrue(isinstance(report[0], bool)) - self.assertFalse(report[0]) - self.assertTrue(len(report[1]), 1) - - @patch('pulp_node.importers.http.importer.Downloader', LocalFileDownloader) - @patch('pulp_node.importers.http.importer.importer_config_to_nectar_config', - wraps=importer_config_to_nectar_config) - @patch('pulp.server.managers.repo._common.get_working_directory', spec_set=True) - def test_import(self, mock_get_working, *mocks): - # Setup - self.populate() - mock_get_working.return_value = self.temp_dir - max_concurrency = 5 - max_bandwidth = 12345 - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - dist = NodesHttpDistributor() - working_dir = os.path.join(self.childfs, 'working_dir') - os.makedirs(working_dir) - repo = Repository(self.REPO_ID, working_dir) - cfg = self.dist_conf() - conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) - dist.publish_repo(repo, conduit, cfg) - model.Distributor.objects.delete() - RepoContentUnit.get_collection().remove() - unit_db.clean() - self.define_plugins() - # Test - importer = NodesHttpImporter() - publisher = dist.publisher(repo, cfg) - manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path()) - configuration = { - constants.MANIFEST_URL_KEYWORD: manifest_url, - constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, - importer_constants.KEY_MAX_DOWNLOADS: max_concurrency, - importer_constants.KEY_MAX_SPEED: max_bandwidth, - } - configuration = PluginCallConfiguration(configuration, {}) - conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock()) - with mock_config.patch({'server': {'storage_dir': self.childfs}}): - with patch('pulp_node.constants.CONTENT_PATH', self.parentfs): - importer.sync_repo(repo, conduit, configuration) - # Verify - units = conduit.get_units() - self.assertEquals(len(units), self.NUM_UNITS) - mock_importer_config_to_nectar_config = mocks[0] - mock_importer_config_to_nectar_config.assert_called_with(configuration.flatten()) - - @patch('pulp_node.importers.http.importer.Downloader', LocalFileDownloader) - @patch('pulp_node.manifest.RemoteManifest.fetch_units') - @patch('pulp.server.managers.repo._common.get_working_directory', spec_set=True) - def test_import_cached_manifest_matched(self, mock_get_working, mock_fetch, *unused): - # Setup - self.populate() - mock_get_working.return_value = self.temp_dir - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - dist = NodesHttpDistributor() - working_dir = os.path.join(self.childfs, 'working_dir') - os.makedirs(working_dir) - repo = Repository(self.REPO_ID, working_dir) - configuration = self.dist_conf() - conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) - dist.publish_repo(repo, conduit, configuration) - model.Distributor.objects.delete() - RepoContentUnit.get_collection().remove() - unit_db.clean() - self.define_plugins() - publisher = dist.publisher(repo, configuration) - manifest_path = publisher.manifest_path() - units_path = os.path.join(os.path.dirname(manifest_path), UNITS_FILE_NAME) - manifest = Manifest(manifest_path) - manifest.read() - shutil.copy(manifest_path, os.path.join(working_dir, MANIFEST_FILE_NAME)) - shutil.copy(units_path, os.path.join(working_dir, UNITS_FILE_NAME)) - # Test - importer = NodesHttpImporter() - manifest_url = pathlib.url_join(publisher.base_url, manifest_path) - configuration = { - constants.MANIFEST_URL_KEYWORD: manifest_url, - constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, - } - configuration = PluginCallConfiguration(configuration, {}) - conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock()) - with mock_config.patch({'server': {'storage_dir': self.childfs}}): - with patch('pulp_node.constants.CONTENT_PATH', self.parentfs): - importer.sync_repo(repo, conduit, configuration) - # Verify - units = conduit.get_units() - self.assertEquals(len(units), self.NUM_UNITS) - self.assertFalse(mock_fetch.called) - - @patch('pulp_node.importers.http.importer.Downloader', LocalFileDownloader) - @patch('pulp.server.managers.repo._common.get_working_directory', spec_set=True) - def test_import_cached_manifest_missing_units(self, mock_get_working, *unused): - # Setup - self.populate() - mock_get_working.return_value = self.temp_dir - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - dist = NodesHttpDistributor() - working_dir = os.path.join(self.childfs, 'working_dir') - os.makedirs(working_dir) - repo = Repository(self.REPO_ID, working_dir) - configuration = self.dist_conf() - conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) - dist.publish_repo(repo, conduit, configuration) - model.Distributor.objects.delete() - RepoContentUnit.get_collection().remove() - unit_db.clean() - self.define_plugins() - publisher = dist.publisher(repo, configuration) - manifest_path = publisher.manifest_path() - manifest = Manifest(manifest_path) - manifest.read() - shutil.copy(manifest_path, os.path.join(working_dir, MANIFEST_FILE_NAME)) - # Test - importer = NodesHttpImporter() - manifest_url = pathlib.url_join(publisher.base_url, manifest_path) - configuration = { - constants.MANIFEST_URL_KEYWORD: manifest_url, - constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, - } - configuration = PluginCallConfiguration(configuration, {}) - conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock()) - with mock_config.patch({'server': {'storage_dir': self.childfs}}): - with patch('pulp_node.constants.CONTENT_PATH', self.parentfs): - importer.sync_repo(repo, conduit, configuration) - # Verify - units = conduit.get_units() - self.assertEquals(len(units), self.NUM_UNITS) - - @patch('pulp_node.importers.http.importer.Downloader', LocalFileDownloader) - @patch('pulp.server.managers.repo._common.get_working_directory', spec_set=True) - def test_import_cached_manifest_units_invalid(self, mock_get_working, *unused): - # Setup - self.populate() - mock_get_working.return_value = self.temp_dir - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - dist = NodesHttpDistributor() - working_dir = os.path.join(self.childfs, 'working_dir') - os.makedirs(working_dir) - repo = Repository(self.REPO_ID, working_dir) - configuration = self.dist_conf() - conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) - dist.publish_repo(repo, conduit, configuration) - model.Distributor.objects.delete() - RepoContentUnit.get_collection().remove() - unit_db.clean() - self.define_plugins() - publisher = dist.publisher(repo, configuration) - manifest_path = publisher.manifest_path() - manifest = Manifest(manifest_path) - manifest.read() - shutil.copy(manifest_path, os.path.join(working_dir, MANIFEST_FILE_NAME)) - with open(os.path.join(working_dir, UNITS_FILE_NAME), 'w+') as fp: - fp.write('invalid-units') - # Test - importer = NodesHttpImporter() - manifest_url = pathlib.url_join(publisher.base_url, manifest_path) - configuration = { - constants.MANIFEST_URL_KEYWORD: manifest_url, - constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, - } - configuration = PluginCallConfiguration(configuration, {}) - conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock()) - with mock_config.patch({'server': {'storage_dir': self.childfs}}): - with patch('pulp_node.constants.CONTENT_PATH', self.parentfs): - importer.sync_repo(repo, conduit, configuration) - # Verify - units = conduit.get_units() - self.assertEquals(len(units), self.NUM_UNITS) - - @patch('pulp_node.importers.http.importer.Downloader', LocalFileDownloader) - @patch('pulp_node.importers.http.importer.importer_config_to_nectar_config', - wraps=importer_config_to_nectar_config) - @patch('pulp.server.managers.repo._common.get_working_directory', spec_set=True) - def test_import_unit_files_already_exist(self, mock_get_working, *mocks): - # Setup - self.populate() - mock_get_working.return_value = self.temp_dir - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - dist = NodesHttpDistributor() - working_dir = os.path.join(self.childfs, 'working_dir') - os.makedirs(working_dir) - repo = Repository(self.REPO_ID, working_dir) - cfg = self.dist_conf() - conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) - dist.publish_repo(repo, conduit, cfg) - model.Distributor.objects.delete() - RepoContentUnit.get_collection().remove() - unit_db.clean() - self.define_plugins() - parent_content = os.path.join(self.parentfs, 'content') - child_content = os.path.join(self.childfs, 'content') - shutil.copytree(parent_content, child_content) - # Test - importer = NodesHttpImporter() - publisher = dist.publisher(repo, cfg) - manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path()) - configuration = { - constants.MANIFEST_URL_KEYWORD: manifest_url, - constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, - } - configuration = PluginCallConfiguration(configuration, {}) - conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock()) - with mock_config.patch({'server': {'storage_dir': self.childfs}}): - with patch('pulp_node.constants.CONTENT_PATH', self.parentfs): - importer.sync_repo(repo, conduit, configuration) - # Verify - units = conduit.get_units() - self.assertEquals(len(units), self.NUM_UNITS) - mock_importer_config_to_nectar_config = mocks[0] - mock_importer_config_to_nectar_config.assert_called_with(configuration.flatten()) - - @patch('pulp_node.importers.http.importer.Downloader', LocalFileDownloader) - @patch('pulp_node.importers.http.importer.importer_config_to_nectar_config', - wraps=importer_config_to_nectar_config) - @patch('pulp.server.managers.repo._common.get_working_directory', spec_set=True) - def test_import_unit_files_already_exist_size_mismatch(self, mock_get_working, *mocks): - # Setup - self.populate() - mock_get_working.return_value = self.temp_dir - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - dist = NodesHttpDistributor() - working_dir = os.path.join(self.childfs, 'working_dir') - os.makedirs(working_dir) - repo = Repository(self.REPO_ID, working_dir) - cfg = self.dist_conf() - conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) - dist.publish_repo(repo, conduit, cfg) - model.Distributor.objects.delete() - RepoContentUnit.get_collection().remove() - unit_db.clean() - self.define_plugins() - parent_content = os.path.join(self.parentfs, 'content') - child_content = os.path.join(self.childfs, 'content') - shutil.copytree(parent_content, child_content) - for fn in os.listdir(child_content): - path = os.path.join(child_content, fn) - if os.path.isdir(path): - continue - with open(path, 'w') as fp: - fp.truncate() - # Test - importer = NodesHttpImporter() - publisher = dist.publisher(repo, cfg) - manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path()) - configuration = { - constants.MANIFEST_URL_KEYWORD: manifest_url, - constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, - } - configuration = PluginCallConfiguration(configuration, {}) - conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock()) - with mock_config.patch({'server': {'storage_dir': self.childfs}}): - with patch('pulp_node.constants.CONTENT_PATH', self.parentfs): - importer.sync_repo(repo, conduit, configuration) - # Verify - units = conduit.get_units() - self.assertEquals(len(units), self.NUM_UNITS) - mock_importer_config_to_nectar_config = mocks[0] - mock_importer_config_to_nectar_config.assert_called_with(configuration.flatten()) - - @patch('pulp_node.importers.http.importer.Downloader', LocalFileDownloader) - @patch('pulp_node.importers.http.importer.importer_config_to_nectar_config', - wraps=importer_config_to_nectar_config) - @patch('pulp.server.managers.repo._common.get_working_directory', spec_set=True) - def test_import_modified_units(self, mock_get_working, *mocks): - # Setup - self.populate() - mock_get_working.return_value = self.temp_dir - max_concurrency = 5 - max_bandwidth = 12345 - with mock_config.patch({'server': {'storage_dir': self.parentfs}}): - dist = NodesHttpDistributor() - working_dir = os.path.join(self.childfs, 'working_dir') - os.makedirs(working_dir) - repo = Repository(self.REPO_ID, working_dir) - cfg = self.dist_conf() - conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR) - dist.publish_repo(repo, conduit, cfg) - # make the published unit have a newer _last_updated. - collection = connection.get_collection(unit_db.unit_collection_name(self.UNIT_TYPE_ID)) - # N=0 (no file) - unit = collection.find_one({'N': 0}) - unit['age'] = 84 # this will be updated back to 42. - unit['_last_updated'] -= 1 - unit['_storage_path'] = None - collection.update({'N': 0}, unit) - # N=1 - unit = collection.find_one({'N': 1}) - unit['age'] = 85 # this will be updated back to 42. - unit['_last_updated'] -= 1 - collection.update({'N': 1}, unit) - # Test - importer = NodesHttpImporter() - publisher = dist.publisher(repo, cfg) - manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path()) - configuration = { - constants.MANIFEST_URL_KEYWORD: manifest_url, - constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY, - importer_constants.KEY_MAX_DOWNLOADS: max_concurrency, - importer_constants.KEY_MAX_SPEED: max_bandwidth, - } - configuration = PluginCallConfiguration(configuration, {}) - conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock()) - with mock_config.patch({'server': {'storage_dir': self.childfs}}): - with patch('pulp_node.constants.CONTENT_PATH', self.parentfs): - importer.sync_repo(repo, conduit, configuration) - # Verify - unit = collection.find_one({'N': 0}) - self.assertEqual(unit['age'], 42) - unit = collection.find_one({'N': 1}) - self.assertEqual(unit['age'], 42) diff --git a/nodes/test/nodes_tests/test_publishers.py b/nodes/test/nodes_tests/test_publishers.py deleted file mode 100644 index 1d576d052b..0000000000 --- a/nodes/test/nodes_tests/test_publishers.py +++ /dev/null @@ -1,138 +0,0 @@ -import os -import shutil -import tarfile -import tempfile -from unittest import TestCase - -from nectar.config import DownloaderConfig -from nectar.downloaders.local import LocalFileDownloader - -from pulp_node import constants, pathlib -from pulp_node.distributors.http.publisher import HttpPublisher -from pulp_node.manifest import RemoteManifest - - -class TestHttp(TestCase): - - TMP_ROOT = '/tmp/pulp/nodes/publishing' - - RELATIVE_PATH = 'redhat/packages' - - UNITS = [ - 'test_1.unit', - 'test_2.unit', - 'test_3.unit', - ] - - NUM_TARED_FILES = 3 - TARED_FILE = '%d.rpm' - - def setUp(self): - if not os.path.exists(self.TMP_ROOT): - os.makedirs(self.TMP_ROOT) - self.tmpdir = tempfile.mkdtemp(dir=self.TMP_ROOT) - self.unit_dir = os.path.join(self.tmpdir, 'content') - shutil.rmtree(self.tmpdir) - os.makedirs(os.path.join(self.unit_dir, self.RELATIVE_PATH)) - - def shutDown(self): - shutil.rmtree(self.TMP_ROOT) - - def populate(self): - units = [] - for n in range(0, 3): - fn = 'test_%d' % n - relative_path = os.path.join(self.RELATIVE_PATH, fn) - path = os.path.join(self.unit_dir, relative_path) - if n == 0: # making the 1st one a directory of files - os.mkdir(path) - for x in range(0, self.NUM_TARED_FILES): - _path = os.path.join(path, self.TARED_FILE % x) - with open(_path, 'w') as fp: - fp.write(str(x)) - else: - with open(path, 'w') as fp: - fp.write(fn) - unit = { - 'type_id': 'unit', - 'unit_key': {'n': n}, - 'storage_path': path, - 'relative_path': relative_path - } - units.append(unit) - return units - - def test_publisher(self): - # setup - units = self.populate() - # test - # publish - repo_id = 'test_repo' - base_url = 'file://' - publish_dir = os.path.join(self.tmpdir, 'nodes/repos') - repo_publish_dir = os.path.join(publish_dir, repo_id) - virtual_host = (publish_dir, publish_dir) - with HttpPublisher(base_url, virtual_host, repo_id, repo_publish_dir) as p: - p.publish(units) - p.commit() - # verify - conf = DownloaderConfig() - downloader = LocalFileDownloader(conf) - manifest_path = p.manifest_path() - working_dir = os.path.join(self.tmpdir, 'working_dir') - os.makedirs(working_dir) - url = pathlib.url_join(base_url, manifest_path) - manifest = RemoteManifest(url, downloader, working_dir) - manifest.fetch() - manifest.fetch_units() - self.assertTrue(manifest.has_valid_units()) - units = manifest.get_units() - n = 0 - for unit, ref in units: - self.assertEqual( - manifest.publishing_details[constants.BASE_URL], - pathlib.url_join(base_url, publish_dir, repo_id)) - if n == 0: # TARBALL - path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH]) - self.assertTrue(os.path.isfile(path)) - if n == 0: # TARBALL - path = pathlib.join(publish_dir, repo_id, unit[constants.TARBALL_PATH]) - tb = tarfile.open(path) - try: - files = sorted(tb.getnames()) - finally: - tb.close() - self.assertEqual(len(files), self.NUM_TARED_FILES) - self.assertEqual(unit['unit_key']['n'], n) - n += 1 - - def test_unstage(self): - # setup - units = self.populate() - # test - # publish - repo_id = 'test_repo' - base_url = 'file://' - publish_dir = os.path.join(self.tmpdir, 'nodes/repos') - repo_publish_dir = os.path.join(publish_dir, repo_id) - virtual_host = (publish_dir, publish_dir) - p = HttpPublisher(base_url, virtual_host, repo_id, repo_publish_dir) - p.publish(units) - p.unstage() - # verify - self.assertFalse(os.path.exists(p.tmp_dir)) - - def test_exit(self): - # setup - units = self.populate() - # test - # publish - repo_id = 'test_repo' - base_url = 'file://' - publish_dir = os.path.join(self.tmpdir, 'nodes/repos') - repo_publish_dir = os.path.join(publish_dir, repo_id) - virtual_host = (publish_dir, publish_dir) - with HttpPublisher(base_url, virtual_host, repo_id, repo_publish_dir) as p: - p.publish(units) - # verify - self.assertFalse(os.path.exists(p.tmp_dir)) diff --git a/nodes/test/nodes_tests/test_resources.py b/nodes/test/nodes_tests/test_resources.py deleted file mode 100644 index d7f33428c7..0000000000 --- a/nodes/test/nodes_tests/test_resources.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -This module contains tests for the pulp_node.resources module. -""" -import unittest - -from pulp.common import config -import mock - -from pulp_node import resources - - -class TestParentBindings(unittest.TestCase): - """ - This class contains tests for the parent_bindings() function. - """ - @mock.patch('pulp_node.resources.read_config') - def test_verify_ssl_false(self, read_config): - """ - Make sure that verify_ssl is passed correctly when it is false. - """ - ca_path = '/some/path.crt' - node_config = {'parent_oauth': {'key': 'some_key', 'secret': 'ssssh!', 'user_id': 'bgates'}, - 'main': {'verify_ssl': 'fAlsE', 'ca_path': ca_path}} - node_config = config.Config(node_config).graph() - read_config.return_value = node_config - - bindings = resources.parent_bindings('host') - - self.assertEqual(bindings.bindings.server.ca_path, ca_path) - self.assertEqual(bindings.bindings.server.verify_ssl, False) - - @mock.patch('pulp_node.resources.read_config') - def test_verify_ssl_true(self, read_config): - """ - Make sure that verify_ssl is passed correctly when it is true. - """ - ca_path = '/some/path' - node_config = {'parent_oauth': {'key': 'some_key', 'secret': 'ssssh!', 'user_id': 'bgates'}, - 'main': {'verify_ssl': 'tRue', 'ca_path': ca_path}} - node_config = config.Config(node_config).graph() - read_config.return_value = node_config - - bindings = resources.parent_bindings('host') - - self.assertEqual(bindings.bindings.server.ca_path, ca_path) - self.assertEqual(bindings.bindings.server.verify_ssl, True) - - -class TestPulpBindings(unittest.TestCase): - """ - This class contains tests for the pulp_bindings() function. - """ - @mock.patch('pulp_node.resources.read_config') - def test_verify_ssl_false(self, read_config): - """ - Make sure that verify_ssl is passed correctly when it is false. - """ - ca_path = '/some/path.crt' - node_config = {'parent_oauth': {'key': 'some_key', 'secret': 'ssssh!', 'user_id': 'bgates'}, - 'main': {'verify_ssl': 'fAlsE', 'ca_path': ca_path}} - node_config = config.Config(node_config).graph() - read_config.return_value = node_config - - bindings = resources.pulp_bindings() - - self.assertEqual(bindings.bindings.server.ca_path, ca_path) - self.assertEqual(bindings.bindings.server.verify_ssl, False) - - @mock.patch('pulp_node.resources.read_config') - def test_verify_ssl_true(self, read_config): - """ - Make sure that verify_ssl is passed correctly when it is true. - """ - ca_path = '/some/path' - node_config = {'parent_oauth': {'key': 'some_key', 'secret': 'ssssh!', 'user_id': 'bgates'}, - 'main': {'verify_ssl': 'True', 'ca_path': ca_path}} - node_config = config.Config(node_config).graph() - read_config.return_value = node_config - - bindings = resources.pulp_bindings() - - self.assertEqual(bindings.bindings.server.ca_path, ca_path) - self.assertEqual(bindings.bindings.server.verify_ssl, True) diff --git a/pulp-dev.py b/pulp-dev.py index 8a306f7957..0132d7a55c 100755 --- a/pulp-dev.py +++ b/pulp-dev.py @@ -55,8 +55,6 @@ '/etc/pulp/content/sources/conf.d', '/etc/pulp/server', '/etc/pulp/server/plugins.conf.d', - '/etc/pulp/server/plugins.conf.d/nodes/importer', - '/etc/pulp/server/plugins.conf.d/nodes/distributor', '/etc/pulp/vhosts80', '/usr/share/pulp', '/usr/share/pulp/templates', @@ -66,9 +64,6 @@ '/usr/lib/pulp/plugins', '/usr/lib/pulp/plugins/types', '/var/lib/pulp/celery', - '/var/lib/pulp/nodes/published', - '/var/lib/pulp/nodes/published/http', - '/var/lib/pulp/nodes/published/https', '/var/lib/pulp/published', '/var/lib/pulp/static', '/var/lib/pulp/uploads', @@ -76,7 +71,6 @@ '/var/run/pulp', '/var/www/pulp', '/var/www/streamer', - '/var/www/pulp/nodes', '/var/www/.python-eggs', # needed for older versions of mod_wsgi ]) @@ -119,17 +113,6 @@ ('streamer/etc/httpd/conf.d/pulp_streamer.conf', '/etc/httpd/conf.d/pulp_streamer.conf'), ('streamer/etc/pulp/streamer.conf', '/etc/pulp/streamer.conf'), - # Pulp Nodes - ('/var/lib/pulp/content', '/var/www/pulp/nodes/content'), - ('/var/lib/pulp/nodes/published/http', '/var/www/pulp/nodes/http'), - ('/var/lib/pulp/nodes/published/https', '/var/www/pulp/nodes/https'), - ('nodes/parent/etc/httpd/conf.d/pulp_nodes.conf', '/etc/httpd/conf.d/pulp_nodes.conf'), - ('nodes/child/etc/pulp/server/plugins.conf.d/nodes/importer/http.conf', - '/etc/pulp/server/plugins.conf.d/nodes/importer/http.conf'), - ('nodes/parent/etc/pulp/server/plugins.conf.d/nodes/distributor/http.conf', - '/etc/pulp/server/plugins.conf.d/nodes/distributor/http.conf'), - ('nodes/child/etc/pulp/agent/conf.d/nodes.conf', '/etc/pulp/agent/conf.d/nodes.conf'), - ('nodes/child/pulp_node/importers/types/nodes.json', DIR_PLUGINS + '/types/node.json'), # Static Content ('/etc/pki/pulp/rsa_pub.key', '/var/lib/pulp/static/rsa_pub.key'), @@ -229,10 +212,6 @@ def get_paths_to_copy(): {'source': 'client_admin/etc/pulp/admin/admin.conf', 'destination': '/etc/pulp/admin/admin.conf', 'owner': 'root', 'group': 'root', 'mode': '644', 'overwrite': False}, - # This should really be 640, but the unit tests require the ability to read it. They - # should mock instead, but until they do we need to keep this world readable - {'source': 'nodes/common/etc/pulp/nodes.conf', 'destination': '/etc/pulp/nodes.conf', - 'owner': 'root', 'group': 'apache', 'mode': '644', 'overwrite': False}, # This really should be 640 since that's how the RPM installs it, but the unit tests try # to read the settings rather than mocking them. Once we've fixed that, we should fix # this to be the same as the spec file. @@ -385,8 +364,6 @@ def install(opts): print('generating certificates') if not os.path.exists('/etc/pki/pulp/ca.crt'): os.system(os.path.join(ROOT_DIR, 'server/bin/pulp-gen-ca-certificate')) - if not os.path.exists('/etc/pki/pulp/nodes/node.crt'): - os.system(os.path.join(ROOT_DIR, 'nodes/common/bin/pulp-gen-nodes-certificate')) # Unfortunately, our unit tests fail to mock the CA certificate and key, so we need to make # those world readable. Until we fix this, we cannot close #1048297 diff --git a/rel-eng/packages/pulp-nodes b/rel-eng/packages/pulp-nodes deleted file mode 100644 index f190bee842..0000000000 --- a/rel-eng/packages/pulp-nodes +++ /dev/null @@ -1 +0,0 @@ -2.6.0-0.5.beta nodes/ diff --git a/run-tests.py b/run-tests.py index b5c98d9d99..b75c88d063 100755 --- a/run-tests.py +++ b/run-tests.py @@ -22,7 +22,6 @@ PACKAGES = [ os.path.dirname(__file__), 'pulp', - 'pulp_node', ] @@ -36,7 +35,6 @@ TESTS_NON_RHEL5 = [ 'client_admin/test/unit', - 'nodes/test/nodes_tests', 'streamer/test/unit', 'server/test/unit', 'repoauth/test',