From 7c213fd1735503540a06ee596eb628fc6b3b0162 Mon Sep 17 00:00:00 2001 From: Brian Bouterse Date: Fri, 4 Sep 2020 15:32:17 -0400 Subject: [PATCH 1/8] Removes the ``test_file_decriptors`` test The ``test_file_decriptors`` test which was not actually testing Pulp code and was failing intermittently. closes #7454 (cherry picked from commit 6e09ef4666e64ab1fb3bff38a3ef7d869c07e4ae) --- CHANGES/7454.misc | 2 ++ pulp_file/tests/functional/api/test_sync.py | 40 +-------------------- 2 files changed, 3 insertions(+), 39 deletions(-) create mode 100644 CHANGES/7454.misc diff --git a/CHANGES/7454.misc b/CHANGES/7454.misc new file mode 100644 index 00000000..56df73a1 --- /dev/null +++ b/CHANGES/7454.misc @@ -0,0 +1,2 @@ +Removes the ``test_file_decriptors`` test which was not actually testing Pulp code and was failing +intermittently. diff --git a/pulp_file/tests/functional/api/test_sync.py b/pulp_file/tests/functional/api/test_sync.py index 8fc49d19..85724f05 100644 --- a/pulp_file/tests/functional/api/test_sync.py +++ b/pulp_file/tests/functional/api/test_sync.py @@ -2,8 +2,7 @@ """Tests that sync file plugin repositories.""" import unittest -from pulp_smash import cli, config -from pulp_smash.pulp3.constants import MEDIA_PATH +from pulp_smash import config from pulp_smash.pulp3.utils import gen_repo, get_added_content_summary, get_content_summary from pulp_file.tests.functional.constants import ( @@ -85,43 +84,6 @@ def test_sync(self): self.assertEqual(latest_version_href, repo.latest_version_href) self.assertDictEqual(get_content_summary(repo.to_dict()), FILE_FIXTURE_SUMMARY) - def test_file_decriptors(self): - """Test whether file descriptors are closed properly. - - This test targets the following issue: - - `Pulp #4073 `_ - - Do the following: - - 1. Check if 'lsof' is installed. If it is not, skip this test. - 2. Create and sync a repo. - 3. Run the 'lsof' command to verify that files in the - path ``/var/lib/pulp/`` are closed after the sync. - 4. Assert that issued command returns `0` opened files. - """ - cli_client = cli.Client(self.cfg, cli.echo_handler) - - # check if 'lsof' is available - if cli_client.run(("which", "lsof")).returncode != 0: - raise unittest.SkipTest("lsof package is not present") - - repo_api = RepositoriesFileApi(self.client) - repo = repo_api.create(gen_repo()) - self.addCleanup(repo_api.delete, repo.pulp_href) - - remote_api = RemotesFileApi(self.client) - remote = remote_api.create(gen_file_remote()) - self.addCleanup(remote_api.delete, remote.pulp_href) - - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) - sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) - monitor_task(sync_response.task) - - cmd = "lsof -t +D {}".format(MEDIA_PATH).split() - response = cli_client.run(cmd).stdout - self.assertEqual(len(response), 0, response) - class SyncInvalidTestCase(unittest.TestCase): """Sync a repository with a given url on the remote.""" From 1e3edd909dfe862e6318308577d30d43d5dbfc58 Mon Sep 17 00:00:00 2001 From: David Davis Date: Thu, 17 Sep 2020 12:08:47 -0400 Subject: [PATCH 2/8] Adding docs for file system exporter fixes #7515 (cherry picked from commit 5c942a239c802c4c8235151d4759adef83bee992) --- CHANGES/7515.doc | 1 + docs/_scripts/docs_check_upload_publish.sh | 3 +++ docs/_scripts/export.sh | 9 +------- docs/_scripts/exporter.sh | 7 ++++++ docs/workflows/export.rst | 27 ++++++++++++++++++++++ docs/workflows/index.rst | 1 + 6 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 CHANGES/7515.doc mode change 100644 => 100755 docs/_scripts/export.sh create mode 100755 docs/_scripts/exporter.sh create mode 100644 docs/workflows/export.rst diff --git a/CHANGES/7515.doc b/CHANGES/7515.doc new file mode 100644 index 00000000..5fdb3cd5 --- /dev/null +++ b/CHANGES/7515.doc @@ -0,0 +1 @@ +Added docs for using FileSystemExporter. diff --git a/docs/_scripts/docs_check_upload_publish.sh b/docs/_scripts/docs_check_upload_publish.sh index 40c8af5f..83891213 100755 --- a/docs/_scripts/docs_check_upload_publish.sh +++ b/docs/_scripts/docs_check_upload_publish.sh @@ -17,3 +17,6 @@ source add_remove.sh source publication.sh source distribution.sh source download_after_upload.sh + +source exporter.sh +source export.sh diff --git a/docs/_scripts/export.sh b/docs/_scripts/export.sh old mode 100644 new mode 100755 index 364913f8..e3580c9d --- a/docs/_scripts/export.sh +++ b/docs/_scripts/export.sh @@ -1,12 +1,5 @@ #!/usr/bin/env bash -export EXPORTER_NAME=$(head /dev/urandom | tr -dc a-z | head -c5) -export DEST_DIR="/data" - -echo "Created a new file system exporter $EXPORTER_NAME." -export EXPORTER_HREF=$(http POST $BASE_ADDR/pulp/api/v3/exporters/file/filesystem/ \ - name=$EXPORTER_NAME path=$DEST_DIR | jq -r '.pulp_href') - -export TASK_URL=$(http POST $BASE_ADDR$EXPORTER_HREF'exports/' publication=$PUBLICATION_HREF +export TASK_URL=$(http POST $BASE_ADDR$EXPORTER_HREF'exports/' publication=$PUBLICATION_HREF \ | jq -r '.task') # Poll the task (here we use a function defined in docs/_scripts/base.sh) diff --git a/docs/_scripts/exporter.sh b/docs/_scripts/exporter.sh new file mode 100755 index 00000000..bc5bad1a --- /dev/null +++ b/docs/_scripts/exporter.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +export EXPORTER_NAME=$(head /dev/urandom | tr -dc a-z | head -c5) +export DEST_DIR=$(mktemp -d -t export-XXXXXXXX) + +echo "Created a new file system exporter $EXPORTER_NAME." +export EXPORTER_HREF=$(http POST $BASE_ADDR/pulp/api/v3/exporters/file/filesystem/ \ + name=$EXPORTER_NAME path=$DEST_DIR | jq -r '.pulp_href') diff --git a/docs/workflows/export.rst b/docs/workflows/export.rst new file mode 100644 index 00000000..5e71938e --- /dev/null +++ b/docs/workflows/export.rst @@ -0,0 +1,27 @@ +Export +====== + +Filesystem +---------- + +This section describes how to export file content to the filesystem. Exporting content to the +filesystem is useful if you want to serve content with your own web server and not use Pulp's +Content App, or if you want to transfer your content to another location (via rsync, etc). + +If DEFAULT_FILE_STORAGE is filesystem, the FilesystemExporter will create hard links. Otherwise, it +will fetch and write the content to the filesystem. + +This example assumes you have a Publication and that publication's href is stored as +$PUBLICATION_HREF. + +First, create a FilesystemExporter and give it the desired path. Your ALLOWED_EXPORT_PATHS setting must +include this path. + +.. literalinclude:: ../_scripts/exporter.sh + :language: bash + +Next you can use this exporter to create a filesystem export that will export your Publication's +content to the filesystem. + +.. literalinclude:: ../_scripts/export.sh + :language: bash diff --git a/docs/workflows/index.rst b/docs/workflows/index.rst index 8337227a..af6db50b 100644 --- a/docs/workflows/index.rst +++ b/docs/workflows/index.rst @@ -41,3 +41,4 @@ library with: sync upload publish-host + export From 60ffc997534b6e4ee78ca1f55cae1195560f914d Mon Sep 17 00:00:00 2001 From: David Davis Date: Fri, 18 Sep 2020 06:41:36 -0400 Subject: [PATCH 3/8] Fixed exception when hitting file system exports path fixes #7522 (cherry picked from commit dbfef9ed1cf1c244ffccbb3fe577ab367805d9be) --- CHANGES/7522.bugfix | 1 + pulp_file/app/viewsets.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 CHANGES/7522.bugfix diff --git a/CHANGES/7522.bugfix b/CHANGES/7522.bugfix new file mode 100644 index 00000000..4bccdff9 --- /dev/null +++ b/CHANGES/7522.bugfix @@ -0,0 +1 @@ +Fixed exception when hitting ``/pulp/api/v3/exporters/file/filesystem//exports/``. diff --git a/pulp_file/app/viewsets.py b/pulp_file/app/viewsets.py index 84efda06..85445055 100644 --- a/pulp_file/app/viewsets.py +++ b/pulp_file/app/viewsets.py @@ -201,7 +201,6 @@ class FileFilesystemExportViewSet(ExportViewSet): """ parent_viewset = FileFilesystemExporterViewSet - parent_lookup_kwargs = {"filesystem_exporter_pk": "filesystem_exporter__pk"} @extend_schema( request=PublicationExportSerializer, From 4695bbd54f86e5756560df1c4947971fee92b189 Mon Sep 17 00:00:00 2001 From: David Davis Date: Thu, 27 Aug 2020 09:17:32 -0400 Subject: [PATCH 4/8] Fixing black issues with new version of black [noissue] (cherry picked from commit 4b87978f8ed44c85b0884672b4592723173a19fc) --- .travis/release.py | 6 +++--- pulp_file/tests/unit/test_core/test_repo_version.py | 12 ++++-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.travis/release.py b/.travis/release.py index 197871cc..5f13f714 100644 --- a/.travis/release.py +++ b/.travis/release.py @@ -105,15 +105,15 @@ def validate_redmine_data(redmine_query_url, redmine_issues): parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) parser.add_argument( - "release_type", type=str, help="Whether the release should be major, minor or patch.", + "release_type", type=str, help="Whether the release should be major, minor or patch." ) parser.add_argument( - "--lower", type=str, required=False, help="Lower bound of pulpcore requirement.", + "--lower", type=str, required=False, help="Lower bound of pulpcore requirement." ) parser.add_argument( - "--upper", type=str, required=False, help="Upper bound of pulpcore requirement.", + "--upper", type=str, required=False, help="Upper bound of pulpcore requirement." ) args = parser.parse_args() diff --git a/pulp_file/tests/unit/test_core/test_repo_version.py b/pulp_file/tests/unit/test_core/test_repo_version.py index e5ff2461..e4525bef 100644 --- a/pulp_file/tests/unit/test_core/test_repo_version.py +++ b/pulp_file/tests/unit/test_core/test_repo_version.py @@ -10,12 +10,10 @@ class RepositoryVersionCRUDTestCase(TestCase): - """Test RepositoryVersion CRUD and content association. - """ + """Test RepositoryVersion CRUD and content association.""" def setUp(self): - """Create Artifact, Content, ContentArtifact, and Repository. - """ + """Create Artifact, Content, ContentArtifact, and Repository.""" artifact = Artifact.objects.create( md5="ec0df26316b1deb465d2d18af7b600f5", sha1="cf6121b0425c2f2e3a2fcfe6f402d59730eb5661", @@ -49,15 +47,13 @@ def setUp(self): @patch("pulpcore.app.models.task.get_current_job") def test_create_repository_version(self, mock_task): - """Test creating a RepositoryVersion. - """ + """Test creating a RepositoryVersion.""" mock_task.return_value.id = self.task.pk with self.repository.new_version() as new_version: new_version.add_content(FileContent.objects.filter(pk=self.content.pk)) self.assertTrue(RepositoryVersion.objects.filter().exists()) def test_remove_repository_version(self): - """Test deleting a RepositoryVersion. - """ + """Test deleting a RepositoryVersion.""" RepositoryVersion.objects.filter().delete() self.assertFalse(RepositoryVersion.objects.filter().exists()) From aae7f5906e5128e6a022d44ef121d2797abc5e36 Mon Sep 17 00:00:00 2001 From: Daniel Alley Date: Tue, 8 Sep 2020 10:39:40 -0400 Subject: [PATCH 5/8] Fix unittest after pulpcore change [noissue] (cherry picked from commit 244aeea89e889ddad48426f35152934860a8b9b5) --- pulp_file/tests/unit/test_core/test_repo_version.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulp_file/tests/unit/test_core/test_repo_version.py b/pulp_file/tests/unit/test_core/test_repo_version.py index e4525bef..098fbbee 100644 --- a/pulp_file/tests/unit/test_core/test_repo_version.py +++ b/pulp_file/tests/unit/test_core/test_repo_version.py @@ -1,3 +1,4 @@ +import uuid from unittest.mock import patch from django.core.files.storage import default_storage as storage @@ -42,7 +43,9 @@ def setUp(self): self.content_artifact.save() self.repository = FileRepository.objects.create(name="foo") self.repository.save() - self.task = Task.objects.create(state="Completed", name="test-task") + self.task = Task.objects.create( + state="Completed", name="test-task", _resource_job_id=uuid.uuid4() + ) self.task.save() @patch("pulpcore.app.models.task.get_current_job") From a56a50aadfa0050d6dfda806cda6bb18483d170f Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Thu, 17 Sep 2020 12:27:50 -0400 Subject: [PATCH 6/8] Updates unit tests to not use MD5 checksum. [noissue] (cherry picked from commit 075a6f6b796e7fc659c0391d964ad0961e46839d) --- pulp_file/tests/unit/test_core/test_repo_version.py | 1 - pulp_file/tests/unit/test_serializers.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pulp_file/tests/unit/test_core/test_repo_version.py b/pulp_file/tests/unit/test_core/test_repo_version.py index 098fbbee..dfd8f199 100644 --- a/pulp_file/tests/unit/test_core/test_repo_version.py +++ b/pulp_file/tests/unit/test_core/test_repo_version.py @@ -16,7 +16,6 @@ class RepositoryVersionCRUDTestCase(TestCase): def setUp(self): """Create Artifact, Content, ContentArtifact, and Repository.""" artifact = Artifact.objects.create( - md5="ec0df26316b1deb465d2d18af7b600f5", sha1="cf6121b0425c2f2e3a2fcfe6f402d59730eb5661", sha224="9a6297eb28d91fad5277c0833856031d0e940432ad807658bd2b60f4", sha256="c8ddb3dcf8da48278d57b0b94486832c66a8835316ccf7ca39e143cbfeb9184f", diff --git a/pulp_file/tests/unit/test_serializers.py b/pulp_file/tests/unit/test_serializers.py index 64bd2543..1a9eb362 100644 --- a/pulp_file/tests/unit/test_serializers.py +++ b/pulp_file/tests/unit/test_serializers.py @@ -13,7 +13,6 @@ class TestFileContentSerializer(TestCase): def setUp(self): """Set up the FileContentSerializer tests.""" self.artifact = Artifact.objects.create( - md5="ec0df26316b1deb465d2d18af7b600f5", sha1="cf6121b0425c2f2e3a2fcfe6f402d59730eb5661", sha224="9a6297eb28d91fad5277c0833856031d0e940432ad807658bd2b60f4", sha256="c8ddb3dcf8da48278d57b0b94486832c66a8835316ccf7ca39e143cbfeb9184f", From 63a34f2c5afb7b40d475cd7ac909408d5599791b Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Wed, 16 Sep 2020 14:57:05 -0400 Subject: [PATCH 7/8] Disables md5 in pulpcore's settings so the pulpcore tests can pass [noissue] (cherry picked from commit a6cbbef7d8646817ce6ee44c827069e226461a32) --- .travis/install.sh | 2 +- template_config.yml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis/install.sh b/.travis/install.sh index 9e77456a..9277fc52 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -85,7 +85,7 @@ VARSYAML fi cat >> vars/main.yaml << VARSYAML -pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]} +pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]} VARSYAML if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "s3" ]]; then diff --git a/template_config.yml b/template_config.yml index c9e5db00..e938f2d0 100644 --- a/template_config.yml +++ b/template_config.yml @@ -31,6 +31,12 @@ pulp_settings: - /tmp allowed_import_paths: - /tmp + allowed_content_checksums: + - sha1 + - sha224 + - sha256 + - sha384 + - sha512 pulpcore_branch: master pulpcore_pip_version_specifier: null pydocstyle: true From 76665339e0cb7863029246834f03a73430fb4da9 Mon Sep 17 00:00:00 2001 From: Brian Bouterse Date: Tue, 22 Sep 2020 16:15:10 -0400 Subject: [PATCH 8/8] Update to be pulpcore 3.7 and 3.8 compatible [noissue] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fbf0fcdb..6d07f4be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -pulpcore>=3.6,<3.7 +pulpcore>=3.7,<3.9