diff --git a/.ci/ansible/filter/repr.py b/.ci/ansible/filter/repr.py index 8455c3442..c8c1678dd 100644 --- a/.ci/ansible/filter/repr.py +++ b/.ci/ansible/filter/repr.py @@ -1,4 +1,5 @@ from __future__ import absolute_import, division, print_function + from packaging.version import parse as parse_version __metaclass__ = type diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 66c494e97..83e197aa7 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -7,11 +7,12 @@ import argparse import fileinput -import urllib.request import sys +import urllib.request + +import yaml from packaging.requirements import Requirement from packaging.version import Version -import yaml try: import tomllib diff --git a/.ci/scripts/check_gettext.sh b/.ci/scripts/check_gettext.sh deleted file mode 100755 index 39bdeb04f..000000000 --- a/.ci/scripts/check_gettext.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../.. - -set -uv - -MATCHES=$(grep -n -r --include \*.py "_(f") - -if [ $? -ne 1 ]; then - printf "\nERROR: Detected mix of f-strings and gettext:\n" - echo "$MATCHES" - exit 1 -fi diff --git a/.ci/scripts/check_pulpcore_imports.sh b/.ci/scripts/check_pulpcore_imports.sh index 5d9c6f481..e1867c625 100755 --- a/.ci/scripts/check_pulpcore_imports.sh +++ b/.ci/scripts/check_pulpcore_imports.sh @@ -10,10 +10,10 @@ # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../.. -set -uv +set -u # check for imports not from pulpcore.plugin. exclude tests -MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin") +MATCHES="$(grep -n -r --include \*.py "from pulpcore.*import" pulp_python | grep -v "tests\|plugin")" if [ $? -ne 1 ]; then printf "\nERROR: Detected bad imports from pulpcore:\n" diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 6e0799952..32e2a2ffd 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -9,16 +9,16 @@ # /// import argparse -import re import os +import re import sys -import tomllib import typing as t from pathlib import Path +import tomllib import yaml -from packaging.version import Version from git import Repo +from packaging.version import Version RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" Y_CHANGELOG_EXTS = [".feature"] @@ -157,9 +157,9 @@ def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int: if reasons: curr_version = Version(last_tag) - assert curr_version.base_version.startswith( - branch - ), "Current-version has to belong to the current branch!" + assert curr_version.base_version.startswith(branch), ( + "Current-version has to belong to the current branch!" + ) next_version = Version(f"{branch}.{curr_version.micro + 1}") print( f"A Z-release is needed for {branch}, " diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index cf9efbe97..71253cb3e 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,15 +5,14 @@ # # For more info visit https://github.com/pulp/plugin_template -import tomllib import warnings -from packaging.requirements import Requirement +import tomllib +from packaging.requirements import Requirement CHECK_MATRIX = [ ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), - ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), ("doc_requirements.txt", False, True, False), ("lint_requirements.txt", False, True, True), diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index fbb5d59d0..5d88b5c1f 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -18,14 +18,13 @@ import json import os import re -import tomllib import urllib.request from pathlib import Path +import tomllib from git import GitCommandError, Repo from packaging.version import parse as parse_version - PYPI_PROJECT = "pulp_python" # Read Towncrier settings diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py index 0c478a212..4f801c39e 100755 --- a/.ci/scripts/pr_labels.py +++ b/.ci/scripts/pr_labels.py @@ -4,9 +4,9 @@ import re import sys -import tomllib from pathlib import Path +import tomllib from git import Repo diff --git a/.ci/scripts/schema.py b/.ci/scripts/schema.py index 9f56caa66..9c8e11b2e 100644 --- a/.ci/scripts/schema.py +++ b/.ci/scripts/schema.py @@ -7,7 +7,9 @@ But some pulp paths start with curly brackets e.g. {artifact_href} This script modifies drf-spectacular schema validation to accept slashes and curly brackets. """ + import json + from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH with open(JSON_SCHEMA_SPEC_PATH) as fh: diff --git a/.ci/scripts/skip_tests.py b/.ci/scripts/skip_tests.py index a68d000d6..380a3da9f 100755 --- a/.ci/scripts/skip_tests.py +++ b/.ci/scripts/skip_tests.py @@ -15,12 +15,13 @@ *: Error """ -import sys +import argparse import os import re -import git +import sys import textwrap -import argparse + +import git DOC_PATTERNS = [ r"^docs/", diff --git a/.ci/scripts/update_github.py b/.ci/scripts/update_github.py index b298ad836..f5e81a5fb 100755 --- a/.ci/scripts/update_github.py +++ b/.ci/scripts/update_github.py @@ -6,6 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template import os + from github import Github g = Github(os.environ.get("GITHUB_TOKEN")) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index a4dc9004a..96b84172f 100644 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -5,10 +5,10 @@ import re import subprocess import sys -import tomllib -import yaml from pathlib import Path +import tomllib +import yaml from github import Github diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 9e00b3faa..000000000 --- a/.flake8 +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -[flake8] -exclude = ./docs/*,*/migrations/* -per-file-ignores = */__init__.py: F401 - -ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 -max-line-length = 100 - -# Flake8 builtin codes -# -------------------- -# E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed -# W503: This enforces operators before line breaks which is not pep8 or black compatible. -# F824: 'nonlocal' is unused: name is never assigned in scope - -# Flake8-quotes extension codes -# ----------------------------- -# Q000: double or single quotes only, default is double (don't want to enforce this) -# Q003: Change outer quotes to avoid escaping inner quotes - -# Flake8-docstring extension codes -# -------------------------------- -# D100: missing docstring in public module -# D104: missing docstring in public package -# D106: missing docstring in public nested class (complains about "class Meta:" and documenting those is silly) -# D200: one-line docstring should fit on one line with quotes -# D205: 1 blank line required between summary line and description -# D400: First line should end with a period -# D401: first line should be imperative (nitpicky) -# D402: first line should not be the function’s “signature” (false positives) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d07a39a08..8da63002e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -36,14 +36,13 @@ jobs: run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows - # Lint code. - - name: "Run flake8" + - name: "Check formating" run: | - flake8 + ruff format --check --diff - - name: "Check for common gettext problems" + - name: "Lint code" run: | - sh .ci/scripts/check_gettext.sh + ruff check - name: "Run extra lint checks" run: | diff --git a/.github/workflows/scripts/stage-changelog-for-default-branch.py b/.github/workflows/scripts/stage-changelog-for-default-branch.py index 3950d7f9c..255c58424 100755 --- a/.github/workflows/scripts/stage-changelog-for-default-branch.py +++ b/.github/workflows/scripts/stage-changelog-for-default-branch.py @@ -12,7 +12,6 @@ from git import Repo from git.exc import GitCommandError - helper = textwrap.dedent( """\ Stage the changelog for a release on main branch. diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py index 967984a42..063314b6c 100755 --- a/.github/workflows/scripts/update_backport_labels.py +++ b/.github/workflows/scripts/update_backport_labels.py @@ -5,10 +5,11 @@ # # For more info visit https://github.com/pulp/plugin_template +import os +import random + import requests import yaml -import random -import os def random_color(): diff --git a/dev_requirements.txt b/dev_requirements.txt deleted file mode 100644 index 0d7d2e745..000000000 --- a/dev_requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -black -check-manifest -flake8 -flake8-docstrings -flake8-tuple -flake8-quotes -# pin pydocstyle until https://gitlab.com/pycqa/flake8-docstrings/issues/36 is resolved -pydocstyle<4 -requests diff --git a/lint_requirements.txt b/lint_requirements.txt index a29362097..fd592d7d8 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -7,6 +7,6 @@ bump-my-version check-manifest -flake8 packaging +ruff yamllint diff --git a/pulp_python/__init__.py b/pulp_python/__init__.py index 5d57d16b3..50f87cf34 100644 --- a/pulp_python/__init__.py +++ b/pulp_python/__init__.py @@ -1 +1 @@ -default_app_config = 'pulp_python.app.PulpPythonPluginAppConfig' +default_app_config = "pulp_python.app.PulpPythonPluginAppConfig" diff --git a/pulp_python/app/fields.py b/pulp_python/app/fields.py index 110d2c55c..b124ea31a 100644 --- a/pulp_python/app/fields.py +++ b/pulp_python/app/fields.py @@ -1,5 +1,5 @@ -from drf_spectacular.utils import extend_schema_field from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers diff --git a/pulp_python/app/modelresource.py b/pulp_python/app/modelresource.py index 7fd75bd0e..8ccb5ef51 100644 --- a/pulp_python/app/modelresource.py +++ b/pulp_python/app/modelresource.py @@ -1,5 +1,6 @@ from pulpcore.plugin.importexport import BaseContentResource from pulpcore.plugin.modelresources import RepositoryResource + from pulp_python.app.models import ( PythonPackageContent, PythonRepository, diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 1adcfc74c..c8de77747 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -1,29 +1,30 @@ from logging import getLogger +from pathlib import PurePath from aiohttp.web import json_response from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ObjectDoesNotExist from django.db import models + from pulpcore.plugin.models import ( Content, - Publication, Distribution, + Publication, Remote, Repository, ) +from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version from pulpcore.plugin.responses import ArtifactResponse from pulpcore.plugin.util import get_domain -from pathlib import PurePath from .utils import ( + PYPI_LAST_SERIAL, + PYPI_SERIAL_CONSTANT, canonicalize_name, get_project_metadata_from_artifact, parse_project_metadata, python_content_to_json, - PYPI_LAST_SERIAL, - PYPI_SERIAL_CONSTANT, ) -from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version log = getLogger(__name__) diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index 906293afb..56503caf4 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -1,11 +1,13 @@ import logging from gettext import gettext as _ +from django.db.utils import IntegrityError from rest_framework import serializers -from pulp_python.app.utils import DIST_EXTENSIONS -from pulp_python.app import fields + from pulpcore.plugin.models import Artifact -from django.db.utils import IntegrityError + +from pulp_python.app import fields +from pulp_python.app.utils import DIST_EXTENSIONS log = logging.getLogger(__name__) @@ -46,7 +48,7 @@ class PackageUploadSerializer(serializers.Serializer): action = serializers.CharField( help_text=_("Defaults to `file_upload`, don't change it or request will fail!"), default="file_upload", - source=":action" + source=":action", ) sha256_digest = serializers.CharField( help_text=_("SHA256 of package to validate upload integrity."), @@ -59,17 +61,17 @@ def validate(self, data): """Validates the request.""" action = data.get(":action") if action != "file_upload": - raise serializers.ValidationError( - _("We do not support the :action {}").format(action) - ) + raise serializers.ValidationError(_("We do not support the :action {}").format(action)) file = data.get("content") for ext, packagetype in DIST_EXTENSIONS.items(): if file.name.endswith(ext): break else: - raise serializers.ValidationError(_( - "Extension on {} is not a valid python extension " - "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(file.name) + raise serializers.ValidationError( + _( + "Extension on {} is not a valid python extension " + "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)" + ).format(file.name) ) sha256 = data.get("sha256_digest") digests = {"sha256": sha256} if sha256 else None diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index d2e34f6bf..d59f61a96 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -1,53 +1,52 @@ import logging -import requests - -from rest_framework.viewsets import ViewSet -from rest_framework.response import Response -from rest_framework.permissions import IsAuthenticatedOrReadOnly -from django.core.exceptions import ObjectDoesNotExist -from django.shortcuts import redirect -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone +from itertools import chain +from pathlib import PurePath +from urllib.parse import urljoin, urlparse, urlunsplit -from rest_framework.reverse import reverse +import requests from django.contrib.sessions.models import Session +from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from django.db.utils import DatabaseError from django.http.response import ( Http404, - HttpResponseForbidden, HttpResponseBadRequest, - StreamingHttpResponse + HttpResponseForbidden, + StreamingHttpResponse, ) +from django.shortcuts import redirect from drf_spectacular.utils import extend_schema from dynaconf import settings -from itertools import chain from packaging.utils import canonicalize_name -from urllib.parse import urljoin, urlparse, urlunsplit -from pathlib import PurePath from pypi_simple.parse_stream import parse_links_stream_response +from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.response import Response +from rest_framework.reverse import reverse +from rest_framework.viewsets import ViewSet from pulpcore.plugin.tasking import dispatch + +from pulp_python.app import tasks from pulp_python.app.models import ( PythonDistribution, PythonPackageContent, PythonPublication, ) from pulp_python.app.pypi.serializers import ( - SummarySerializer, PackageMetadataSerializer, PackageUploadSerializer, - PackageUploadTaskSerializer + PackageUploadTaskSerializer, + SummarySerializer, ) from pulp_python.app.utils import ( - write_simple_index, - write_simple_detail, - python_content_to_json, PYPI_LAST_SERIAL, PYPI_SERIAL_CONSTANT, + python_content_to_json, + write_simple_detail, + write_simple_index, ) -from pulp_python.app import tasks - log = logging.getLogger(__name__) BASE_CONTENT_URL = urljoin(settings.CONTENT_ORIGIN, settings.CONTENT_PATH_PREFIX) @@ -133,11 +132,16 @@ def upload(self, request, path): if settings.PYTHON_GROUP_UPLOADS: return self.upload_package_group(repo, artifact, filename, request.session) - result = dispatch(tasks.upload, exclusive_resources=[artifact, repo], - kwargs={"artifact_sha256": artifact.sha256, - "filename": filename, - "repository_pk": str(repo.pk)}) - return Response(data={"task": reverse('tasks-detail', args=[result.pk], request=None)}) + result = dispatch( + tasks.upload, + exclusive_resources=[artifact, repo], + kwargs={ + "artifact_sha256": artifact.sha256, + "filename": filename, + "repository_pk": str(repo.pk), + }, + ) + return Response(data={"task": reverse("tasks-detail", args=[result.pk], request=None)}) def upload_package_group(self, repo, artifact, filename, session): """Steps 4 & 5, spawns tasks to add packages to index.""" @@ -150,10 +154,10 @@ def upload_package_group(self, repo, artifact, filename, session): try: with transaction.atomic(): sq.first() - current_start = datetime.fromisoformat(session['start']) + current_start = datetime.fromisoformat(session["start"]) if current_start >= datetime.now(tz=timezone.utc): - session['artifacts'].append((str(artifact.sha256), filename)) - session['start'] = str(start_time) + session["artifacts"].append((str(artifact.sha256), filename)) + session["start"] = str(start_time) session.modified = False session.save() else: @@ -166,14 +170,19 @@ def upload_package_group(self, repo, artifact, filename, session): def create_group_upload_task(self, cur_session, repository, artifact, filename, start_time): """Creates the actual task that adds the packages to the index.""" - cur_session['start'] = str(start_time) - cur_session['artifacts'] = [(str(artifact.sha256), filename)] + cur_session["start"] = str(start_time) + cur_session["artifacts"] = [(str(artifact.sha256), filename)] cur_session.modified = False cur_session.save() - result = dispatch(tasks.upload_group, exclusive_resources=[artifact, repository], - kwargs={"session_pk": str(cur_session.session_key), - "repository_pk": str(repository.pk)}) - return reverse('tasks-detail', args=[result.pk], request=None) + result = dispatch( + tasks.upload_group, + exclusive_resources=[artifact, repository], + kwargs={ + "session_pk": str(cur_session.session_key), + "repository_pk": str(repository.pk), + }, + ) + return reverse("tasks-detail", args=[result.pk], request=None) class SimpleView(ViewSet, PackageUploadMixin): @@ -186,21 +195,22 @@ def list(self, request, path): """Gets the simple api html page for the index.""" distro, repo_version, content = self.get_drvc(path) if self.should_redirect(distro, repo_version=repo_version): - return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/')) - names = content.order_by('name').values_list('name', flat=True).distinct().iterator() + return redirect(urljoin(BASE_CONTENT_URL, f"{path}/simple/")) + names = content.order_by("name").values_list("name", flat=True).distinct().iterator() return StreamingHttpResponse(write_simple_index(names, streamed=True)) def pull_through_package_simple(self, package, path, remote): """Gets the package's simple page from remote.""" + def parse_url(link): parsed = urlparse(link.url) - digest, _, value = parsed.fragment.partition('=') + digest, _, value = parsed.fragment.partition("=") stripped_url = urlunsplit(chain(parsed[:3], ("", ""))) - redirect = f'{path}/{link.text}?redirect={stripped_url}' + redirect = f"{path}/{link.text}?redirect={stripped_url}" d_url = urljoin(BASE_CONTENT_URL, redirect) - return link.text, d_url, value if digest == 'sha256' else '' + return link.text, d_url, value if digest == "sha256" else "" - url = remote.get_remote_artifact_url(f'simple/{package}/') + url = remote.get_remote_artifact_url(f"simple/{package}/") kwargs = {} if proxy_url := remote.proxy_url: if remote.proxy_username or remote.proxy_password: @@ -224,10 +234,10 @@ def retrieve(self, request, path, package): if not repo_ver or not content.filter(name__normalize=normalized).exists(): return self.pull_through_package_simple(normalized, path, distro.remote) if self.should_redirect(distro, repo_version=repo_ver): - return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/{normalized}/')) + return redirect(urljoin(BASE_CONTENT_URL, f"{path}/simple/{normalized}/")) packages = ( content.filter(name__normalize=normalized) - .values_list('filename', 'sha256', 'name') + .values_list("filename", "sha256", "name") .iterator() ) try: @@ -237,12 +247,14 @@ def retrieve(self, request, path, package): else: packages = chain([present], packages) name = present[2] - releases = ((f, urljoin(BASE_CONTENT_URL, f'{path}/{f}'), d) for f, d, _ in packages) + releases = ((f, urljoin(BASE_CONTENT_URL, f"{path}/{f}"), d) for f, d, _ in packages) return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True)) - @extend_schema(request=PackageUploadSerializer, - responses={200: PackageUploadTaskSerializer}, - summary="Upload a package") + @extend_schema( + request=PackageUploadSerializer, + responses={200: PackageUploadTaskSerializer}, + summary="Upload a package", + ) def create(self, request, path): """ Upload package to the index. @@ -259,9 +271,11 @@ class MetadataView(ViewSet, PyPIMixin): authentication_classes = [] permission_classes = [] - @extend_schema(tags=["Pypi: Metadata"], - responses={200: PackageMetadataSerializer}, - summary="Get package metadata") + @extend_schema( + tags=["Pypi: Metadata"], + responses={200: PackageMetadataSerializer}, + summary="Get package metadata", + ) def retrieve(self, request, path, meta): """ Retrieves the package's core-metadata specified by @@ -294,8 +308,7 @@ class PyPIView(ViewSet, PyPIMixin): authentication_classes = [] permission_classes = [] - @extend_schema(responses={200: SummarySerializer}, - summary="Get index summary") + @extend_schema(responses={200: SummarySerializer}, summary="Get index summary") def retrieve(self, request, path): """Gets package summary stats of index.""" distro, repo_ver, content = self.get_drvc(path) @@ -309,9 +322,11 @@ def retrieve(self, request, path): class UploadView(ViewSet, PackageUploadMixin): """View for the `/legacy` upload endpoint.""" - @extend_schema(request=PackageUploadSerializer, - responses={200: PackageUploadTaskSerializer}, - summary="Upload a package") + @extend_schema( + request=PackageUploadSerializer, + responses={200: PackageUploadTaskSerializer}, + summary="Upload a package", + ) def create(self, request, path): """ Upload package to the index. diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 36fb6395e..62f1ef9fc 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -1,4 +1,5 @@ from gettext import gettext as _ + from django.conf import settings from packaging.requirements import Requirement from rest_framework import serializers @@ -6,8 +7,8 @@ from pulpcore.plugin import models as core_models from pulpcore.plugin import serializers as core_serializers -from pulp_python.app import models as python_models from pulp_python.app import fields +from pulp_python.app import models as python_models from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata @@ -44,15 +45,14 @@ class PythonDistributionSerializer(core_serializers.DistributionSerializer): ) base_url = serializers.SerializerMethodField(read_only=True) allow_uploads = serializers.BooleanField( - default=True, - help_text=_("Allow packages to be uploaded to this index.") + default=True, help_text=_("Allow packages to be uploaded to this index.") ) remote = core_serializers.DetailRelatedField( required=False, - help_text=_('Remote that can be used to fetch content when using pull-through caching.'), + help_text=_("Remote that can be used to fetch content when using pull-through caching."), view_name_pattern=r"remotes(-.*/.*)?-detail", queryset=core_models.Remote.objects.all(), - allow_null=True + allow_null=True, ) def get_base_url(self, obj): @@ -61,7 +61,9 @@ def get_base_url(self, obj): class Meta: fields = core_serializers.DistributionSerializer.Meta.fields + ( - 'publication', "allow_uploads", "remote" + "publication", + "allow_uploads", + "remote", ) model = python_models.PythonDistribution @@ -72,126 +74,162 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa """ filename = serializers.CharField( - help_text=_('The name of the distribution package, usually of the format:' - ' {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}' - '-{platform tag}.{packagetype}'), + help_text=_( + "The name of the distribution package, usually of the format:" + " {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}" + "-{platform tag}.{packagetype}" + ), read_only=True, ) packagetype = serializers.CharField( - help_text=_('The type of the distribution package ' - '(e.g. sdist, bdist_wheel, bdist_egg, etc)'), + help_text=_( + "The type of the distribution package (e.g. sdist, bdist_wheel, bdist_egg, etc)" + ), read_only=True, ) name = serializers.CharField( - help_text=_('The name of the python project.'), + help_text=_("The name of the python project."), read_only=True, ) version = serializers.CharField( - help_text=_('The packages version number.'), + help_text=_("The packages version number."), read_only=True, ) sha256 = serializers.CharField( - default='', - help_text=_('The SHA256 digest of this package.'), + default="", + help_text=_("The SHA256 digest of this package."), ) metadata_version = serializers.CharField( - help_text=_('Version of the file format'), + help_text=_("Version of the file format"), read_only=True, ) summary = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A one-line summary of what the package does.') + required=False, + allow_blank=True, + help_text=_("A one-line summary of what the package does."), ) description = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A longer description of the package that can run to several paragraphs.') + required=False, + allow_blank=True, + help_text=_("A longer description of the package that can run to several paragraphs."), ) description_content_type = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A string stating the markup syntax (if any) used in the distribution’s' - ' description, so that tools can intelligently render the description.') + required=False, + allow_blank=True, + help_text=_( + "A string stating the markup syntax (if any) used in the distribution’s" + " description, so that tools can intelligently render the description." + ), ) keywords = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Additional keywords to be used to assist searching for the ' - 'package in a larger catalog.') + required=False, + allow_blank=True, + help_text=_( + "Additional keywords to be used to assist searching for the " + "package in a larger catalog." + ), ) home_page = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The URL for the package\'s home page.') + required=False, allow_blank=True, help_text=_("The URL for the package's home page.") ) download_url = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Legacy field denoting the URL from which this package can be downloaded.') + required=False, + allow_blank=True, + help_text=_("Legacy field denoting the URL from which this package can be downloaded."), ) author = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Text containing the author\'s name. Contact information can also be added,' - ' separated with newlines.') + required=False, + allow_blank=True, + help_text=_( + "Text containing the author's name. Contact information can also be added," + " separated with newlines." + ), ) author_email = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The author\'s e-mail address. ') + required=False, allow_blank=True, help_text=_("The author's e-mail address. ") ) maintainer = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The maintainer\'s name at a minimum; ' - 'additional contact information may be provided.') + required=False, + allow_blank=True, + help_text=_( + "The maintainer's name at a minimum; additional contact information may be provided." + ), ) maintainer_email = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The maintainer\'s e-mail address.') + required=False, allow_blank=True, help_text=_("The maintainer's e-mail address.") ) license = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Text indicating the license covering the distribution') + required=False, + allow_blank=True, + help_text=_("Text indicating the license covering the distribution"), ) requires_python = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The Python version(s) that the distribution is guaranteed to be ' - 'compatible with.') + required=False, + allow_blank=True, + help_text=_( + "The Python version(s) that the distribution is guaranteed to be compatible with." + ), ) project_url = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A browsable URL for the project and a label for it, separated by a comma.') + required=False, + allow_blank=True, + help_text=_("A browsable URL for the project and a label for it, separated by a comma."), ) project_urls = fields.JSONObjectField( - required=False, default=dict, - help_text=_('A dictionary of labels and URLs for the project.') + required=False, + default=dict, + help_text=_("A dictionary of labels and URLs for the project."), ) platform = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A comma-separated list of platform specifications, ' - 'summarizing the operating systems supported by the package.') + required=False, + allow_blank=True, + help_text=_( + "A comma-separated list of platform specifications, " + "summarizing the operating systems supported by the package." + ), ) supported_platform = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ') + required=False, + allow_blank=True, + help_text=_("Field to specify the OS and CPU for which the binary package was compiled. "), ) requires_dist = fields.JSONObjectField( - required=False, default=list, - help_text=_('A JSON list containing names of some other distutils project ' - 'required by this distribution.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of some other distutils project " + "required by this distribution." + ), ) provides_dist = fields.JSONObjectField( - required=False, default=list, - help_text=_('A JSON list containing names of a Distutils project which is contained' - ' within this distribution.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of a Distutils project which is contained" + " within this distribution." + ), ) obsoletes_dist = fields.JSONObjectField( - required=False, default=list, - help_text=_('A JSON list containing names of a distutils project\'s distribution which ' - 'this distribution renders obsolete, meaning that the two projects should not ' - 'be installed at the same time.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of a distutils project's distribution which " + "this distribution renders obsolete, meaning that the two projects should not " + "be installed at the same time." + ), ) requires_external = fields.JSONObjectField( - required=False, default=list, - help_text=_('A JSON list containing some dependency in the system that the distribution ' - 'is to be used.') + required=False, + default=list, + help_text=_( + "A JSON list containing some dependency in the system that the distribution " + "is to be used." + ), ) classifiers = fields.JSONObjectField( - required=False, default=list, - help_text=_('A JSON list containing classification values for a Python package.') + required=False, + default=list, + help_text=_("A JSON list containing classification values for a Python package."), ) def deferred_validate(self, data): @@ -210,29 +248,33 @@ def deferred_validate(self, data): try: filename = data["relative_path"] except KeyError: - raise serializers.ValidationError(detail={"relative_path": _('This field is required')}) + raise serializers.ValidationError(detail={"relative_path": _("This field is required")}) artifact = data["artifact"] try: metadata = get_project_metadata_from_artifact(filename, artifact) except ValueError: - raise serializers.ValidationError(_( - "Extension on {} is not a valid python extension " - "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(filename) + raise serializers.ValidationError( + _( + "Extension on {} is not a valid python extension " + "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)" + ).format(filename) ) if data.get("sha256") and data["sha256"] != artifact.sha256: raise serializers.ValidationError( - detail={"sha256": _( - "The uploaded artifact's sha256 checksum does not match the one provided" - )} + detail={ + "sha256": _( + "The uploaded artifact's sha256 checksum does not match the one provided" + ) + } ) _data = parse_project_metadata(vars(metadata)) - _data['packagetype'] = metadata.packagetype - _data['version'] = metadata.version - _data['filename'] = filename - _data['sha256'] = artifact.sha256 + _data["packagetype"] = metadata.packagetype + _data["version"] = metadata.version + _data["filename"] = filename + _data["sha256"] = artifact.sha256 data.update(_data) @@ -244,11 +286,33 @@ def retrieve(self, validated_data): class Meta: fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + ( - 'filename', 'packagetype', 'name', 'version', 'sha256', 'metadata_version', 'summary', - 'description', 'description_content_type', 'keywords', 'home_page', 'download_url', - 'author', 'author_email', 'maintainer', 'maintainer_email', 'license', - 'requires_python', 'project_url', 'project_urls', 'platform', 'supported_platform', - 'requires_dist', 'provides_dist', 'obsoletes_dist', 'requires_external', 'classifiers' + "filename", + "packagetype", + "name", + "version", + "sha256", + "metadata_version", + "summary", + "description", + "description_content_type", + "keywords", + "home_page", + "download_url", + "author", + "author_email", + "maintainer", + "maintainer_email", + "license", + "requires_python", + "project_url", + "project_urls", + "platform", + "supported_platform", + "requires_dist", + "provides_dist", + "obsoletes_dist", + "requires_external", + "classifiers", ) model = python_models.PythonPackageContent @@ -260,7 +324,11 @@ class MinimalPythonPackageContentSerializer(PythonPackageContentSerializer): class Meta: fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + ( - 'filename', 'packagetype', 'name', 'version', 'sha256', + "filename", + "packagetype", + "name", + "version", + "sha256", ) model = python_models.PythonPackageContent @@ -284,47 +352,49 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer): child=serializers.CharField(allow_blank=False), required=False, allow_empty=True, - help_text=_( - "A list containing project specifiers for Python packages to include." - ), + help_text=_("A list containing project specifiers for Python packages to include."), ) excludes = serializers.ListField( child=serializers.CharField(allow_blank=False), required=False, allow_empty=True, - help_text=_( - "A list containing project specifiers for Python packages to exclude." - ), + help_text=_("A list containing project specifiers for Python packages to exclude."), ) prereleases = serializers.BooleanField( - required=False, - help_text=_('Whether or not to include pre-release packages in the sync.') + required=False, help_text=_("Whether or not to include pre-release packages in the sync.") ) policy = serializers.ChoiceField( - help_text=_("The policy to use when downloading content. The possible values include: " - "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default."), + help_text=_( + "The policy to use when downloading content. The possible values include: " + "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default." + ), choices=core_models.Remote.POLICY_CHOICES, - default=core_models.Remote.ON_DEMAND + default=core_models.Remote.ON_DEMAND, ) package_types = MultipleChoiceArrayField( required=False, - help_text=_("The package types to sync for Python content. Leave blank to get every" - "package type."), + help_text=_( + "The package types to sync for Python content. Leave blank to get everypackage type." + ), choices=python_models.PACKAGE_TYPES, - default=list + default=list, ) keep_latest_packages = serializers.IntegerField( required=False, - help_text=_("The amount of latest versions of a package to keep on sync, includes" - "pre-releases if synced. Default 0 keeps all versions."), - default=0 + help_text=_( + "The amount of latest versions of a package to keep on sync, includes" + "pre-releases if synced. Default 0 keeps all versions." + ), + default=0, ) exclude_platforms = MultipleChoiceArrayField( required=False, - help_text=_("List of platforms to exclude syncing Python packages for. Possible values" - "include: windows, macos, freebsd, and linux."), + help_text=_( + "List of platforms to exclude syncing Python packages for. Possible values" + "include: windows, macos, freebsd, and linux." + ), choices=python_models.PLATFORMS, - default=list + default=list, ) def validate_includes(self, value): @@ -334,7 +404,7 @@ def validate_includes(self, value): Requirement(pkg) except ValueError as ve: raise serializers.ValidationError( - _("includes specifier {} is invalid. {}".format(pkg, ve)) + _("includes specifier {} is invalid. {}").format(pkg, ve) ) return value @@ -345,13 +415,17 @@ def validate_excludes(self, value): Requirement(pkg) except ValueError as ve: raise serializers.ValidationError( - _("excludes specifier {} is invalid. {}".format(pkg, ve)) + _("excludes specifier {} is invalid. {}").format(pkg, ve) ) return value class Meta: fields = core_serializers.RemoteSerializer.Meta.fields + ( - "includes", "excludes", "prereleases", "package_types", "keep_latest_packages", + "includes", + "excludes", + "prereleases", + "package_types", + "keep_latest_packages", "exclude_platforms", ) model = python_models.PythonRemote @@ -373,10 +447,12 @@ class PythonBanderRemoteSerializer(serializers.Serializer): ) policy = serializers.ChoiceField( - help_text=_("The policy to use when downloading content. The possible values include: " - "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default."), + help_text=_( + "The policy to use when downloading content. The possible values include: " + "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default." + ), choices=core_models.Remote.POLICY_CHOICES, - default=core_models.Remote.ON_DEMAND + default=core_models.Remote.ON_DEMAND, ) @@ -386,8 +462,9 @@ class PythonPublicationSerializer(core_serializers.PublicationSerializer): """ distributions = core_serializers.DetailRelatedField( - help_text=_('This publication is currently being hosted as configured by these ' - 'distributions.'), + help_text=_( + "This publication is currently being hosted as configured by these distributions." + ), source="distribution_set", view_name="pythondistributions-detail", many=True, @@ -395,5 +472,5 @@ class PythonPublicationSerializer(core_serializers.PublicationSerializer): ) class Meta: - fields = core_serializers.PublicationSerializer.Meta.fields + ('distributions',) + fields = core_serializers.PublicationSerializer.Meta.fields + ("distributions",) model = python_models.PythonPublication diff --git a/pulp_python/app/settings.py b/pulp_python/app/settings.py index 3aeb3a74d..e69938892 100644 --- a/pulp_python/app/settings.py +++ b/pulp_python/app/settings.py @@ -1,4 +1,4 @@ import socket PYTHON_GROUP_UPLOADS = False -PYPI_API_HOSTNAME = 'https://' + socket.getfqdn() +PYPI_API_HOSTNAME = "https://" + socket.getfqdn() diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index c4fb0478b..9609912a9 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -1,6 +1,6 @@ -from gettext import gettext as _ import logging import os +from gettext import gettext as _ from django.core.files import File from packaging.utils import canonicalize_name @@ -8,8 +8,7 @@ from pulpcore.plugin import models from pulp_python.app import models as python_models -from pulp_python.app.utils import write_simple_index, write_simple_detail - +from pulp_python.app.utils import write_simple_detail, write_simple_index log = logging.getLogger(__name__) @@ -24,15 +23,17 @@ def publish(repository_version_pk): """ repository_version = models.RepositoryVersion.objects.get(pk=repository_version_pk) - log.info(_('Publishing: repository={repo}, version={version}').format( - repo=repository_version.repository.name, - version=repository_version.number, - )) + log.info( + _("Publishing: repository={repo}, version={version}").format( + repo=repository_version.repository.name, + version=repository_version.number, + ) + ) with python_models.PythonPublication.create(repository_version, pass_through=True) as pub: write_simple_api(pub) - log.info(_('Publication: {pk} created').format(pk=pub.pk)) + log.info(_("Publication: {pk} created").format(pk=pub.pk)) return pub @@ -49,26 +50,24 @@ def write_simple_api(publication): publication (pulpcore.plugin.models.Publication): A publication to generate metadata for """ - simple_dir = 'simple/' + simple_dir = "simple/" os.mkdir(simple_dir) project_names = ( python_models.PythonPackageContent.objects.filter( pk__in=publication.repository_version.content ) - .order_by('name') - .values_list('name', flat=True) + .order_by("name") + .values_list("name", flat=True) .distinct() ) # write the root index, which lists all of the projects for which there is a package available - index_path = '{simple_dir}index.html'.format(simple_dir=simple_dir) - with open(index_path, 'w') as index: + index_path = "{simple_dir}index.html".format(simple_dir=simple_dir) + with open(index_path, "w") as index: index.write(write_simple_index(project_names)) index_metadata = models.PublishedMetadata.create_from_file( - relative_path=index_path, - publication=publication, - file=File(open(index_path, 'rb')) + relative_path=index_path, publication=publication, file=File(open(index_path, "rb")) ) index_metadata.save() @@ -84,41 +83,41 @@ def write_simple_api(publication): current_name = project_names[ind] package_releases = [] for release in releases.iterator(): - if release['name'] != current_name: + if release["name"] != current_name: write_project_page( name=canonicalize_name(current_name), simple_dir=simple_dir, package_releases=package_releases, - publication=publication + publication=publication, ) package_releases = [] ind += 1 current_name = project_names[ind] - relative_path = release['filename'] + relative_path = release["filename"] path = f"../../{relative_path}" - checksum = release['sha256'] + checksum = release["sha256"] package_releases.append((relative_path, path, checksum)) # Write the final project's page write_project_page( name=canonicalize_name(current_name), simple_dir=simple_dir, package_releases=package_releases, - publication=publication + publication=publication, ) def write_project_page(name, simple_dir, package_releases, publication): """Writes a project's simple page.""" - project_dir = f'{simple_dir}{name}/' + project_dir = f"{simple_dir}{name}/" os.mkdir(project_dir) - metadata_relative_path = f'{project_dir}index.html' + metadata_relative_path = f"{project_dir}index.html" - with open(metadata_relative_path, 'w') as simple_metadata: + with open(metadata_relative_path, "w") as simple_metadata: simple_metadata.write(write_simple_detail(name, package_releases)) project_metadata = models.PublishedMetadata.create_from_file( relative_path=metadata_relative_path, publication=publication, - file=File(open(metadata_relative_path, 'rb')) + file=File(open(metadata_relative_path, "rb")), ) project_metadata.save() # change to bulk create when multi-table supported diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 1293c6130..f3892cc03 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -1,13 +1,18 @@ import logging import tempfile -from typing import Optional, Any, AsyncGenerator - -import aiohttp -from aiohttp import ClientResponseError, ClientError -from lxml.etree import LxmlError from gettext import gettext as _ from os import environ +from typing import Any, AsyncGenerator, Optional +from urllib.parse import urljoin, urlsplit, urlunsplit +import aiohttp +from aiohttp import ClientError, ClientResponseError +from bandersnatch.configuration import BandersnatchConfig +from bandersnatch.master import Master +from bandersnatch.mirror import Mirror +from lxml.etree import LxmlError +from packaging.requirements import Requirement +from pypi_simple import parse_repo_index_page from rest_framework import serializers from pulpcore.plugin.models import Artifact, ProgressReport, Remote, Repository @@ -22,14 +27,7 @@ PythonPackageContent, PythonRemote, ) -from pulp_python.app.utils import parse_metadata, PYPI_LAST_SERIAL -from pypi_simple import parse_repo_index_page - -from bandersnatch.mirror import Mirror -from bandersnatch.master import Master -from bandersnatch.configuration import BandersnatchConfig -from packaging.requirements import Requirement -from urllib.parse import urljoin, urlsplit, urlunsplit +from pulp_python.app.utils import PYPI_LAST_SERIAL, parse_metadata logger = logging.getLogger(__name__) @@ -53,9 +51,7 @@ def sync(remote_pk, repository_pk, mirror): repository = Repository.objects.get(pk=repository_pk) if not remote.url: - raise serializers.ValidationError( - detail=_("A remote must have a url attribute to sync.") - ) + raise serializers.ValidationError(detail=_("A remote must have a url attribute to sync.")) first_stage = PythonBanderStage(remote) DeclarativeVersion(first_stage, repository, mirror).create() @@ -125,8 +121,8 @@ async def run(self): creds = f"{self.remote.proxy_username}:{self.remote.proxy_password}" netloc = f"{creds}@{parsed_proxy.netloc}" proxy_url = urlunsplit((parsed_proxy.scheme, netloc, "", "", "")) - environ['http_proxy'] = proxy_url - environ['https_proxy'] = proxy_url + environ["http_proxy"] = proxy_url + environ["https_proxy"] = proxy_url # Bandersnatch includes leading slash when forming API urls url = self.remote.url.rstrip("/") # local & global timeouts defaults to 10secs and 5 hours @@ -146,9 +142,7 @@ async def run(self): ) packages_to_sync = None if self.remote.includes: - packages_to_sync = [ - Requirement(pkg).name for pkg in self.remote.includes - ] + packages_to_sync = [Requirement(pkg).name for pkg in self.remote.includes] await pmirror.synchronize(packages_to_sync) @@ -176,9 +170,7 @@ class PulpMirror(Mirror): Pulp Mirror Class to perform syncing using Bandersnatch """ - def __init__( - self, serial, master, workers, deferred_download, python_stage, progress_report - ): + def __init__(self, serial, master, workers, deferred_download, python_stage, progress_report): """Initialize Bandersnatch Mirror""" super().__init__(master=master, workers=workers) self.synced_serial = serial @@ -193,11 +185,7 @@ async def determine_packages_to_sync(self): """ number_xmlrpc_attempts = 3 for attempt in range(number_xmlrpc_attempts): - logger.info( - "Attempt {} to get package list from {}".format( - attempt, self.master.url - ) - ) + logger.info("Attempt {} to get package list from {}".format(attempt, self.master.url)) try: if not self.synced_serial: logger.info("Syncing all packages.") @@ -209,9 +197,7 @@ async def determine_packages_to_sync(self): ) else: logger.info("Syncing based on changelog.") - changed_packages = await self.master.changed_packages( - self.synced_serial - ) + changed_packages = await self.master.changed_packages(self.synced_serial) self.packages_to_sync.update(changed_packages) self.target_serial = max( [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()] diff --git a/pulp_python/app/tasks/upload.py b/pulp_python/app/tasks/upload.py index de9e89bad..78ca67eeb 100644 --- a/pulp_python/app/tasks/upload.py +++ b/pulp_python/app/tasks/upload.py @@ -1,9 +1,10 @@ import time - from datetime import datetime, timezone -from django.db import transaction + from django.contrib.sessions.models import Session -from pulpcore.plugin.models import Artifact, CreatedResource, ContentArtifact +from django.db import transaction + +from pulpcore.plugin.models import Artifact, ContentArtifact, CreatedResource from pulp_python.app.models import PythonPackageContent, PythonRepository from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata @@ -40,10 +41,10 @@ def upload_group(session_pk, repository_pk=None): with transaction.atomic(): session_data = s_query.first().get_decoded() now = datetime.now(tz=timezone.utc) - start_time = datetime.fromisoformat(session_data['start']) + start_time = datetime.fromisoformat(session_data["start"]) if now >= start_time: content_to_add = PythonPackageContent.objects.none() - for artifact_sha256, filename in session_data['artifacts']: + for artifact_sha256, filename in session_data["artifacts"]: pre_check = PythonPackageContent.objects.filter(sha256=artifact_sha256) content = pre_check or create_content(artifact_sha256, filename) content.get().touch() @@ -73,17 +74,15 @@ def create_content(artifact_sha256, filename): metadata = get_project_metadata_from_artifact(filename, artifact) data = parse_project_metadata(vars(metadata)) - data['packagetype'] = metadata.packagetype - data['version'] = metadata.version - data['filename'] = filename - data['sha256'] = artifact.sha256 + data["packagetype"] = metadata.packagetype + data["version"] = metadata.version + data["filename"] = filename + data["sha256"] = artifact.sha256 @transaction.atomic() def create(): content = PythonPackageContent.objects.create(**data) - ContentArtifact.objects.create( - artifact=artifact, content=content, relative_path=filename - ) + ContentArtifact.objects.create(artifact=artifact, content=content, relative_path=filename) return content new_content = create() diff --git a/pulp_python/app/urls.py b/pulp_python/app/urls.py index d6a513782..64b07b5cb 100644 --- a/pulp_python/app/urls.py +++ b/pulp_python/app/urls.py @@ -1,8 +1,8 @@ from django.urls import path -from pulp_python.app.pypi.views import SimpleView, MetadataView, PyPIView, UploadView +from pulp_python.app.pypi.views import MetadataView, PyPIView, SimpleView, UploadView -PYPI_API_URL = 'pypi//' +PYPI_API_URL = "pypi//" # TODO: Implement remaining PyPI endpoints # path("project/", PackageProject.as_view()), # Endpoints to nicely see contents of index # path("search/", PackageSearch.as_view()), @@ -12,17 +12,17 @@ path( PYPI_API_URL + "pypi//", MetadataView.as_view({"get": "retrieve"}), - name="pypi-metadata" + name="pypi-metadata", ), path( PYPI_API_URL + "simple//", SimpleView.as_view({"get": "retrieve"}), - name="simple-package-detail" + name="simple-package-detail", ), path( - PYPI_API_URL + 'simple/', + PYPI_API_URL + "simple/", SimpleView.as_view({"get": "list", "post": "create"}), - name="simple-detail" + name="simple-detail", ), path(PYPI_API_URL, PyPIView.as_view({"get": "retrieve"}), name="pypi-detail"), ] diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index db6acdd09..687632504 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -1,14 +1,14 @@ -import pkginfo +import json import shutil import tempfile -import json from collections import defaultdict + +import pkginfo from django.conf import settings from jinja2 import Template from packaging.utils import canonicalize_name from packaging.version import parse - PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" """TODO This serial constant is temporary until Python repositories implements serials""" PYPI_SERIAL_CONSTANT = 1000000000 @@ -71,28 +71,28 @@ def parse_project_metadata(project): """ package = {} - package['name'] = project.get('name') or "" - package['metadata_version'] = project.get('metadata_version') or "" - package['summary'] = project.get('summary') or "" - package['description'] = project.get('description') or "" - package['keywords'] = project.get('keywords') or "" - package['home_page'] = project.get('home_page') or "" - package['download_url'] = project.get('download_url') or "" - package['author'] = project.get('author') or "" - package['author_email'] = project.get('author_email') or "" - package['maintainer'] = project.get('maintainer') or "" - package['maintainer_email'] = project.get('maintainer_email') or "" - package['license'] = project.get('license') or "" - package['project_url'] = project.get('project_url') or "" - package['platform'] = project.get('platform') or "" - package['supported_platform'] = project.get('supported_platform') or "" - package['requires_dist'] = json.dumps(project.get('requires_dist', [])) - package['provides_dist'] = json.dumps(project.get('provides_dist', [])) - package['obsoletes_dist'] = json.dumps(project.get('obsoletes_dist', [])) - package['requires_external'] = json.dumps(project.get('requires_external', [])) - package['classifiers'] = json.dumps(project.get('classifiers', [])) - package['project_urls'] = json.dumps(project.get('project_urls', {})) - package['description_content_type'] = project.get('description_content_type') or "" + package["name"] = project.get("name") or "" + package["metadata_version"] = project.get("metadata_version") or "" + package["summary"] = project.get("summary") or "" + package["description"] = project.get("description") or "" + package["keywords"] = project.get("keywords") or "" + package["home_page"] = project.get("home_page") or "" + package["download_url"] = project.get("download_url") or "" + package["author"] = project.get("author") or "" + package["author_email"] = project.get("author_email") or "" + package["maintainer"] = project.get("maintainer") or "" + package["maintainer_email"] = project.get("maintainer_email") or "" + package["license"] = project.get("license") or "" + package["project_url"] = project.get("project_url") or "" + package["platform"] = project.get("platform") or "" + package["supported_platform"] = project.get("supported_platform") or "" + package["requires_dist"] = json.dumps(project.get("requires_dist", [])) + package["provides_dist"] = json.dumps(project.get("provides_dist", [])) + package["obsoletes_dist"] = json.dumps(project.get("obsoletes_dist", [])) + package["requires_external"] = json.dumps(project.get("requires_external", [])) + package["classifiers"] = json.dumps(project.get("classifiers", [])) + package["project_urls"] = json.dumps(project.get("project_urls", {})) + package["description_content_type"] = project.get("description_content_type") or "" return package @@ -115,13 +115,13 @@ def parse_metadata(project, version, distribution): """ package = {} - package['filename'] = distribution.get('filename') or "" - package['packagetype'] = distribution.get('packagetype') or "" - package['version'] = version - package['url'] = distribution.get('url') or "" - package['sha256'] = distribution.get('digests', {}).get('sha256') or "" - package['python_version'] = distribution.get('python_version') or "" - package['requires_python'] = distribution.get('requires_python') or "" + package["filename"] = distribution.get("filename") or "" + package["packagetype"] = distribution.get("packagetype") or "" + package["version"] = version + package["url"] = distribution.get("url") or "" + package["sha256"] = distribution.get("digests", {}).get("sha256") or "" + package["python_version"] = distribution.get("python_version") or "" + package["requires_python"] = distribution.get("requires_python") or "" package.update(parse_project_metadata(project)) @@ -142,7 +142,7 @@ def get_project_metadata_from_artifact(filename, artifact): # Copy file to a temp directory under the user provided filename, we do this # because pkginfo validates that the filename has a valid extension before # reading it - with tempfile.NamedTemporaryFile('wb', dir=".", suffix=filename) as temp_file: + with tempfile.NamedTemporaryFile("wb", dir=".", suffix=filename) as temp_file: artifact.file.seek(0) shutil.copyfileobj(artifact.file, temp_file) temp_file.flush() @@ -267,10 +267,12 @@ def python_content_to_download_info(content, base_path): Takes in a PythonPackageContent and base path of the distribution to create a dictionary of download information for that content. This dictionary is used by Releases and Urls. """ + def find_artifact(): _art = content_artifact.artifact if not _art: from pulpcore.plugin import models + _art = models.RemoteArtifact.objects.filter(content_artifact=content_artifact).first() return _art @@ -295,7 +297,7 @@ def find_artifact(): "upload_time_iso_8601": str(artifact.pulp_created.isoformat()), "url": url, "yanked": False, - "yanked_reason": None + "yanked_reason": None, } diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 12e3bfecc..8249d1626 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -24,15 +24,12 @@ class PythonRepositoryViewSet(core_viewsets.RepositoryViewSet, ModifyRepositoryA synced, added, or removed. """ - endpoint_name = 'python' + endpoint_name = "python" queryset = python_models.PythonRepository.objects.all() serializer_class = python_serializers.PythonRepositorySerializer - @extend_schema( - summary="Sync from remote", - responses={202: AsyncOperationResponseSerializer} - ) - @action(detail=True, methods=['post'], serializer_class=RepositorySyncURLSerializer) + @extend_schema(summary="Sync from remote", responses={202: AsyncOperationResponseSerializer}) + @action(detail=True, methods=["post"], serializer_class=RepositorySyncURLSerializer) def sync(self, request, pk): """ @@ -42,22 +39,21 @@ def sync(self, request, pk): """ repository = self.get_object() serializer = RepositorySyncURLSerializer( - data=request.data, - context={'request': request, "repository_pk": pk} + data=request.data, context={"request": request, "repository_pk": pk} ) serializer.is_valid(raise_exception=True) - remote = serializer.validated_data.get('remote', repository.remote) - mirror = serializer.validated_data.get('mirror') + remote = serializer.validated_data.get("remote", repository.remote) + mirror = serializer.validated_data.get("mirror") result = dispatch( tasks.sync, exclusive_resources=[repository], shared_resources=[remote], kwargs={ - 'remote_pk': str(remote.pk), - 'repository_pk': str(repository.pk), - 'mirror': mirror - } + "remote_pk": str(remote.pk), + "repository_pk": str(repository.pk), + "mirror": mirror, + }, ) return core_viewsets.OperationPostponedResponse(result, request) @@ -81,7 +77,7 @@ class PythonDistributionViewSet(core_viewsets.DistributionViewSet): href="./#tag/Content:-Packages">Python Package Content. """ - endpoint_name = 'pypi' + endpoint_name = "pypi" queryset = python_models.PythonDistribution.objects.all() serializer_class = python_serializers.PythonDistributionSerializer @@ -94,19 +90,20 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter): class Meta: model = python_models.PythonPackageContent fields = { - 'name': ['exact', 'in'], - 'author': ['exact', 'in'], - 'packagetype': ['exact', 'in'], - 'requires_python': ['exact', 'in', "contains"], - 'filename': ['exact', 'in', 'contains'], - 'keywords': ['in', 'contains'], - 'sha256': ['exact', 'in'], - 'version': ['exact', 'gt', 'lt', 'gte', 'lte'] + "name": ["exact", "in"], + "author": ["exact", "in"], + "packagetype": ["exact", "in"], + "requires_python": ["exact", "in", "contains"], + "filename": ["exact", "in", "contains"], + "keywords": ["in", "contains"], + "sha256": ["exact", "in"], + "version": ["exact", "gt", "lt", "gte", "lte"], } class PythonPackageSingleArtifactContentUploadViewSet( - core_viewsets.SingleArtifactContentUploadViewSet): + core_viewsets.SingleArtifactContentUploadViewSet +): """ PythonPackageContent represents each individually installable Python package. In the Python @@ -117,7 +114,7 @@ class PythonPackageSingleArtifactContentUploadViewSet( """ - endpoint_name = 'packages' + endpoint_name = "packages" queryset = python_models.PythonPackageContent.objects.all() serializer_class = python_serializers.PythonPackageContentSerializer minimal_serializer_class = python_serializers.MinimalPythonPackageContentSerializer @@ -133,7 +130,7 @@ class PythonRemoteViewSet(core_viewsets.RemoteViewSet): """ - endpoint_name = 'python' + endpoint_name = "python" queryset = python_models.PythonRemote.objects.all() serializer_class = python_serializers.PythonRemoteSerializer @@ -141,8 +138,11 @@ class PythonRemoteViewSet(core_viewsets.RemoteViewSet): summary="Create from Bandersnatch", responses={201: python_serializers.PythonRemoteSerializer}, ) - @action(detail=False, methods=["post"], - serializer_class=python_serializers.PythonBanderRemoteSerializer) + @action( + detail=False, + methods=["post"], + serializer_class=python_serializers.PythonBanderRemoteSerializer, + ) def from_bandersnatch(self, request): """ @@ -154,11 +154,12 @@ def from_bandersnatch(self, request): name = serializer.validated_data.get("name") policy = serializer.validated_data.get("policy") bander_config = BandersnatchConfig(bander_config_file.file.name).config - data = {"name": name, - "policy": policy, - "url": bander_config.get("mirror", "master"), - "download_concurrency": bander_config.get("mirror", "workers"), - } + data = { + "name": name, + "policy": policy, + "url": bander_config.get("mirror", "master"), + "download_concurrency": bander_config.get("mirror", "workers"), + } enabled = bander_config.get("plugins", "enabled") enabled_all = "all" in enabled data["prereleases"] = not (enabled_all or "prerelease_release" in enabled) @@ -175,8 +176,9 @@ def from_bandersnatch(self, request): "exclude_platform": ("blocklist", "platforms", "exclude_platforms"), } for plugin, options in plugin_filters.items(): - if (enabled_all or plugin in enabled) and \ - bander_config.has_option(options[0], options[1]): + if (enabled_all or plugin in enabled) and bander_config.has_option( + options[0], options[1] + ): data[options[2]] = bander_config.get(options[0], options[1]).split() remote = python_serializers.PythonRemoteSerializer(data=data, context={"request": request}) remote.is_valid(raise_exception=True) @@ -193,13 +195,11 @@ class PythonPublicationViewSet(core_viewsets.PublicationViewSet): """ - endpoint_name = 'pypi' + endpoint_name = "pypi" queryset = python_models.PythonPublication.objects.exclude(complete=False) serializer_class = python_serializers.PythonPublicationSerializer - @extend_schema( - responses={202: AsyncOperationResponseSerializer} - ) + @extend_schema(responses={202: AsyncOperationResponseSerializer}) def create(self, request): """ @@ -207,18 +207,16 @@ def create(self, request): """ serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) - repository_version = serializer.validated_data.get('repository_version') + repository_version = serializer.validated_data.get("repository_version") # Safe because version OR repository is enforced by serializer. if not repository_version: - repository = serializer.validated_data.get('repository') + repository = serializer.validated_data.get("repository") repository_version = RepositoryVersion.latest(repository) result = dispatch( tasks.publish, shared_resources=[repository_version.repository], - kwargs={ - 'repository_version_pk': str(repository_version.pk) - } + kwargs={"repository_version_pk": str(repository_version.pk)}, ) return core_viewsets.OperationPostponedResponse(result, request) diff --git a/pulp_python/tests/functional/api/test_auto_publish.py b/pulp_python/tests/functional/api/test_auto_publish.py index cafd97c99..594c8b597 100644 --- a/pulp_python/tests/functional/api/test_auto_publish.py +++ b/pulp_python/tests/functional/api/test_auto_publish.py @@ -1,16 +1,18 @@ # coding=utf-8 """Tests automatic updating of publications and distributions.""" + from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import download_content_unit +from pulpcore.client.pulp_python import RepositorySyncURL + from pulp_python.tests.functional.utils import ( + TestCaseUsingBindings, + TestHelpersMixin, cfg, gen_python_remote, - TestCaseUsingBindings, - TestHelpersMixin ) from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 -from pulpcore.client.pulp_python import RepositorySyncURL class AutoPublishDistributeTestCase(TestCaseUsingBindings, TestHelpersMixin): diff --git a/pulp_python/tests/functional/api/test_consume_content.py b/pulp_python/tests/functional/api/test_consume_content.py index a46358be3..3257f512b 100644 --- a/pulp_python/tests/functional/api/test_consume_content.py +++ b/pulp_python/tests/functional/api/test_consume_content.py @@ -1,28 +1,28 @@ # coding=utf-8 """Tests that perform actions over content unit.""" + +from urllib.parse import urljoin, urlsplit + from pulp_smash import cli from pulp_smash.pulp3.bindings import delete_orphans, monitor_task from pulp_smash.pulp3.utils import modify_repo +from pulp_smash.utils import http_get from pulp_python.tests.functional.constants import ( + PYPI_URL, PYTHON_FIXTURE_URL, - PYTHON_FIXTURES_PACKAGES, PYTHON_FIXTURES_FILENAMES, + PYTHON_FIXTURES_PACKAGES, PYTHON_LIST_PROJECT_SPECIFIER, - PYPI_URL, ) - from pulp_python.tests.functional.utils import ( + TestCaseUsingBindings, + TestHelpersMixin, cfg, gen_artifact, gen_python_content_attrs, - TestCaseUsingBindings, - TestHelpersMixin, ) from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 -from urllib.parse import urljoin, urlsplit - -from pulp_smash.utils import http_get class PipInstallContentTestCase(TestCaseUsingBindings, TestHelpersMixin): @@ -101,9 +101,7 @@ def test_workflow_02(self): def check_consume(self, distribution): """Tests that pip packages hosted in a distribution can be consumed""" host_base_url = cfg.get_content_host_base_url() - url = "".join( - [host_base_url, "/pulp/content/", distribution["base_path"], "/simple/"] - ) + url = "".join([host_base_url, "/pulp/content/", distribution["base_path"], "/simple/"]) for pkg in self.PACKAGES: out = self.install(self.cli_client, pkg, host=url) self.assertTrue(self.check_install(self.cli_client, pkg), out) diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index f3dbe073b..79df4150f 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -1,28 +1,29 @@ # coding=utf-8 """Tests that perform actions over content unit.""" -import pytest -from pulp_smash.pulp3.bindings import delete_orphans, monitor_task, PulpTaskError -from pulp_python.tests.functional.utils import ( - gen_artifact, - gen_python_content_attrs, - skip_if, - TestCaseUsingBindings, - TestHelpersMixin, -) -from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 from tempfile import NamedTemporaryFile from urllib.parse import urljoin -from pypi_simple import PyPISimple +import pytest +from pulp_smash.pulp3.bindings import PulpTaskError, delete_orphans, monitor_task from pulp_smash.utils import http_get +from pypi_simple import PyPISimple + from pulp_python.tests.functional.constants import ( - PYTHON_FIXTURES_URL, - PYTHON_PACKAGE_DATA, PYTHON_EGG_FILENAME, PYTHON_EGG_URL, + PYTHON_FIXTURES_URL, + PYTHON_PACKAGE_DATA, PYTHON_SM_FIXTURE_CHECKSUMS, ) +from pulp_python.tests.functional.utils import ( + TestCaseUsingBindings, + TestHelpersMixin, + gen_artifact, + gen_python_content_attrs, + skip_if, +) +from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 class ContentUnitTestCase(TestCaseUsingBindings, TestHelpersMixin): @@ -82,9 +83,7 @@ def test_03_partially_update(self): """ attrs = gen_python_content_attrs(self.artifact) with self.assertRaises(AttributeError) as exc: - self.content_api.partial_update( - self.content_unit["pulp_href"], attrs - ) + self.content_api.partial_update(self.content_unit["pulp_href"], attrs) msg = "object has no attribute 'partial_update'" self.assertIn(msg, exc.exception.args[0]) @@ -195,9 +194,7 @@ def test_09_upload_same_repo_filename_different_artifact(self): response = self.do_upload(repository=repo.pulp_href, remote_path=url) created_resources = monitor_task(response.task).created_resources content_unit2 = self.content_api.read(created_resources[1]) - content_list_search = self.content_api.list( - repository_version=created_resources[0] - ).results + content_list_search = self.content_api.list(repository_version=created_resources[0]).results self.assertEqual(len(content_list_search), 1) self.assertEqual(content_unit2.pulp_href, content_list_search[0].pulp_href) @@ -214,9 +211,7 @@ def test_10_upload_with_mismatched_sha256(self): msg = "The uploaded artifact's sha256 checksum does not match the one provided" self.assertTrue(msg in task_report["error"]["description"]) - def do_upload( - self, filename=PYTHON_EGG_FILENAME, remote_path=PYTHON_EGG_URL, **kwargs - ): + def do_upload(self, filename=PYTHON_EGG_FILENAME, remote_path=PYTHON_EGG_URL, **kwargs): """Takes in attributes dict for a file and creates content from it""" with NamedTemporaryFile() as file_to_upload: file_to_upload.write(http_get(remote_path)) diff --git a/pulp_python/tests/functional/api/test_crud_publications.py b/pulp_python/tests/functional/api/test_crud_publications.py index fb863da74..9a92cfc73 100644 --- a/pulp_python/tests/functional/api/test_crud_publications.py +++ b/pulp_python/tests/functional/api/test_crud_publications.py @@ -1,35 +1,32 @@ # coding=utf-8 """Tests that publish python plugin repositories.""" + import random from random import choice +from urllib.parse import urljoin from pulp_smash.pulp3.bindings import monitor_task -from pulp_smash.pulp3.utils import ( - get_content, - get_versions, - modify_repo -) -from urllib.parse import urljoin +from pulp_smash.pulp3.utils import get_content, get_versions, modify_repo + +from pulpcore.client.pulp_python.exceptions import ApiException from pulp_python.tests.functional.constants import ( - PYTHON_CONTENT_NAME, PULP_CONTENT_BASE_URL, - PYTHON_SM_PROJECT_SPECIFIER, - PYTHON_SM_FIXTURE_RELEASES, - PYTHON_SM_FIXTURE_CHECKSUMS, + PYTHON_CONTENT_NAME, PYTHON_EGG_FILENAME, + PYTHON_SM_FIXTURE_CHECKSUMS, + PYTHON_SM_FIXTURE_RELEASES, + PYTHON_SM_PROJECT_SPECIFIER, PYTHON_WHEEL_FILENAME, ) from pulp_python.tests.functional.utils import ( - cfg, TestCaseUsingBindings, TestHelpersMixin, + cfg, ensure_simple, ) from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 -from pulpcore.client.pulp_python.exceptions import ApiException - class PublishAnyRepoVersionTestCase(TestCaseUsingBindings, TestHelpersMixin): """Test whether a particular repository version can be published. @@ -231,8 +228,9 @@ def test_all_content_published(self): distro = self._create_distribution_from_publication(pub) url = urljoin(PULP_CONTENT_BASE_URL, f"{distro.base_path}/simple/") - proper, msgs = ensure_simple(url, PYTHON_SM_FIXTURE_RELEASES, - sha_digests=PYTHON_SM_FIXTURE_CHECKSUMS) + proper, msgs = ensure_simple( + url, PYTHON_SM_FIXTURE_RELEASES, sha_digests=PYTHON_SM_FIXTURE_CHECKSUMS + ) self.assertTrue(proper, msg=msgs) def test_removed_content_not_published(self): diff --git a/pulp_python/tests/functional/api/test_crud_remotes.py b/pulp_python/tests/functional/api/test_crud_remotes.py index 64bf0197c..16341a08d 100644 --- a/pulp_python/tests/functional/api/test_crud_remotes.py +++ b/pulp_python/tests/functional/api/test_crud_remotes.py @@ -1,31 +1,30 @@ # coding=utf-8 """Tests that CRUD python remotes.""" -from random import choice + import unittest +from random import choice +from tempfile import NamedTemporaryFile from pulp_smash import utils - from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.constants import ON_DEMAND_DOWNLOAD_POLICIES -from pulp_python.tests.functional.utils import ( - gen_python_client, - gen_python_remote, - skip_if, -) -from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 - from pulpcore.client.pulp_python import RemotesPythonApi from pulpcore.client.pulp_python.exceptions import ApiException from pulp_python.tests.functional.constants import ( BANDERSNATCH_CONF, DEFAULT_BANDER_REMOTE_BODY, - PYTHON_INVALID_SPECIFIER_NO_NAME, PYTHON_INVALID_SPECIFIER_BAD_VERSION, + PYTHON_INVALID_SPECIFIER_NO_NAME, PYTHON_VALID_SPECIFIER_NO_VERSION, ) -from tempfile import NamedTemporaryFile +from pulp_python.tests.functional.utils import ( + gen_python_client, + gen_python_remote, + skip_if, +) +from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 class CRUDRemotesTestCase(unittest.TestCase): @@ -243,9 +242,7 @@ def test_03_invalid_policy(self): """ remote = self.remote_api.read(self.remote["pulp_href"]).to_dict() with self.assertRaises(ApiException): - self.remote_api.partial_update( - self.remote["pulp_href"], {"policy": utils.uuid4()} - ) + self.remote_api.partial_update(self.remote["pulp_href"], {"policy": utils.uuid4()}) self.remote.update(self.remote_api.read(self.remote["pulp_href"]).to_dict()) self.assertEqual(remote["policy"], self.remote["policy"], self.remote) diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 236f21070..dd902a39a 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -1,27 +1,29 @@ # coding=utf-8 """Tests that verify download of content served by Pulp.""" -import pytest + import hashlib from random import choice from urllib.parse import urljoin +import pytest from pulp_smash import utils from pulp_smash.pulp3.utils import ( download_content_unit, get_content_summary, ) + from pulp_python.tests.functional.constants import ( PYTHON_FIXTURE_URL, - PYTHON_MD_PROJECT_SPECIFIER, - PYTHON_MD_FIXTURE_SUMMARY, PYTHON_LG_FIXTURE_SUMMARY, PYTHON_LG_PROJECT_SPECIFIER, + PYTHON_MD_FIXTURE_SUMMARY, + PYTHON_MD_PROJECT_SPECIFIER, ) from pulp_python.tests.functional.utils import ( - cfg, - get_python_content_paths, TestCaseUsingBindings, TestHelpersMixin, + cfg, + get_python_content_paths, ) from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 @@ -60,14 +62,10 @@ def test_all(self): pub = self._create_publication(repo) distro = self._create_distribution_from_publication(pub) # Pick a content unit (of each type), and download it from both Pulp Fixtures… - unit_paths = [ - choice(paths) for paths in get_python_content_paths(repo.to_dict()).values() - ] + unit_paths = [choice(paths) for paths in get_python_content_paths(repo.to_dict()).values()] fixtures_hashes = [ hashlib.sha256( - utils.http_get( - urljoin(urljoin(PYTHON_FIXTURE_URL, "packages/"), unit_path[0]) - ) + utils.http_get(urljoin(urljoin(PYTHON_FIXTURE_URL, "packages/"), unit_path[0])) ).hexdigest() for unit_path in unit_paths ] @@ -100,12 +98,7 @@ def test_basic_pulp_to_pulp_sync(self): repo = self._create_repo_and_sync_with_remote(remote) pub = self._create_publication(repo) distro = self._create_distribution_from_publication(pub) - url_fragments = [ - cfg.get_content_host_base_url(), - "pulp/content", - distro.base_path, - "" - ] + url_fragments = [cfg.get_content_host_base_url(), "pulp/content", distro.base_path, ""] unit_url = "/".join(url_fragments) # Sync using old Pulp content api diff --git a/pulp_python/tests/functional/api/test_export_import.py b/pulp_python/tests/functional/api/test_export_import.py index 33f40bd4c..4dd0e341f 100644 --- a/pulp_python/tests/functional/api/test_export_import.py +++ b/pulp_python/tests/functional/api/test_export_import.py @@ -4,11 +4,14 @@ NOTE: assumes ALLOWED_EXPORT_PATHS setting contains "/tmp" - all tests will fail if this is not the case. """ -import pytest + import uuid +import pytest + from pulp_python.tests.functional.constants import ( - PYTHON_XS_PROJECT_SPECIFIER, PYTHON_SM_PROJECT_SPECIFIER + PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_XS_PROJECT_SPECIFIER, ) @@ -32,12 +35,8 @@ def test_export_then_import( remote_b = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER, policy="immediate") repo_a = python_repo_factory() repo_b = python_repo_factory() - sync_response_a = python_repo_api_client.sync( - repo_a.pulp_href, {"remote": remote_a.pulp_href} - ) - sync_response_b = python_repo_api_client.sync( - repo_b.pulp_href, {"remote": remote_b.pulp_href} - ) + sync_response_a = python_repo_api_client.sync(repo_a.pulp_href, {"remote": remote_a.pulp_href}) + sync_response_b = python_repo_api_client.sync(repo_b.pulp_href, {"remote": remote_b.pulp_href}) monitor_task(sync_response_a.task) monitor_task(sync_response_b.task) diff --git a/pulp_python/tests/functional/api/test_full_mirror.py b/pulp_python/tests/functional/api/test_full_mirror.py index 3abb1f274..dc44073b4 100644 --- a/pulp_python/tests/functional/api/test_full_mirror.py +++ b/pulp_python/tests/functional/api/test_full_mirror.py @@ -1,38 +1,40 @@ # coding=utf-8 """Tests that python plugin can fully mirror PyPi and other Pulp repositories""" + +import socket import unittest +from urllib.parse import urljoin, urlsplit -from pulp_smash import config, cli +import requests +from pulp_smash import cli, config from pulp_smash.pulp3.bindings import delete_orphans, monitor_task from pulp_smash.pulp3.utils import gen_repo, get_content_summary +from pypi_simple import parse_repo_project_response + +from pulpcore.client.pulp_python import ( + RemotesPythonApi, + RepositoriesPythonApi, + RepositorySyncURL, +) +from pulpcore.client.pulpcore import ApiClient as CoreApiClient +from pulpcore.client.pulpcore import Configuration, TasksApi from pulp_python.tests.functional.constants import ( PULP_CONTENT_BASE_URL, PULP_PYPI_BASE_URL, - PYTHON_CONTENT_NAME, PYPI_URL, + PYTHON_CONTENT_NAME, PYTHON_XS_FIXTURE_CHECKSUMS, ) from pulp_python.tests.functional.utils import ( + TestCaseUsingBindings, + TestHelpersMixin, cfg, gen_python_client, gen_python_remote, - TestCaseUsingBindings, - TestHelpersMixin, ) from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 -from pulpcore.client.pulpcore import TasksApi, ApiClient as CoreApiClient, Configuration -from pulpcore.client.pulp_python import ( - RepositoriesPythonApi, - RepositorySyncURL, - RemotesPythonApi, -) -from pypi_simple import parse_repo_project_response -import requests -import socket -from urllib.parse import urljoin, urlsplit - @unittest.skip class PyPiMirrorTestCase(unittest.TestCase): @@ -50,10 +52,10 @@ def setUpClass(cls): cls.cfg = config.get_config() cls.client = gen_python_client() configuration = Configuration() - configuration.username = 'admin' - configuration.password = 'password' - configuration.host = 'http://{}:24817'.format(socket.gethostname()) - configuration.safe_chars_for_path_param = '/' + configuration.username = "admin" + configuration.password = "password" + configuration.host = "http://{}:24817".format(socket.gethostname()) + configuration.safe_chars_for_path_param = "/" cls.core_client = CoreApiClient(configuration) def test_on_demand_pypi_full_sync(self): diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 7e6fa6261..ae9ce9ed7 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -1,39 +1,42 @@ """Tests all the PyPI apis available at `pypi/`.""" + import os -import requests import subprocess import tempfile -import pytest - from urllib.parse import urljoin +import pytest +import requests from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import get_added_content_summary, get_content_summary + +from pulpcore.client.pulp_python import PypiApi + from pulp_python.tests.functional.constants import ( - PYTHON_CONTENT_NAME, - PYTHON_SM_PROJECT_SPECIFIER, - PYTHON_SM_FIXTURE_RELEASES, - PYTHON_SM_FIXTURE_CHECKSUMS, - PYTHON_MD_PROJECT_SPECIFIER, - PYTHON_MD_PYPI_SUMMARY, PULP_CONTENT_BASE_URL, PULP_PYPI_BASE_URL, + PYTHON_CONTENT_NAME, PYTHON_EGG_FILENAME, - PYTHON_EGG_URL, PYTHON_EGG_SHA256, + PYTHON_EGG_URL, + PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_MD_PYPI_SUMMARY, + PYTHON_SM_FIXTURE_CHECKSUMS, + PYTHON_SM_FIXTURE_RELEASES, + PYTHON_SM_PROJECT_SPECIFIER, PYTHON_WHEEL_FILENAME, - PYTHON_WHEEL_URL, PYTHON_WHEEL_SHA256, + PYTHON_WHEEL_URL, SHELF_PYTHON_JSON, ) - from pulp_python.tests.functional.utils import ( - py_client as client, - ensure_simple, TestCaseUsingBindings, TestHelpersMixin, + ensure_simple, +) +from pulp_python.tests.functional.utils import ( + py_client as client, ) -from pulpcore.client.pulp_python import PypiApi PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" PYPI_SERIAL_CONSTANT = 1000000000 @@ -44,6 +47,7 @@ @pytest.fixture def python_empty_repo_distro(python_repo_factory, python_distribution_factory): """Returns an empty repo with and distribution serving it.""" + def _generate_empty_repo_distro(repo_body=None, distro_body=None): repo_body = repo_body or {} distro_body = distro_body or {} @@ -145,9 +149,7 @@ def test_package_upload(self): files={"content": open(self.egg, "rb")}, ) self.assertEqual(response.status_code, 400) - self.assertEqual( - response.reason, f"Package {PYTHON_EGG_FILENAME} already exists in index" - ) + self.assertEqual(response.reason, f"Package {PYTHON_EGG_FILENAME} already exists in index") def test_package_upload_session(self): """Tests that multiple uploads will be broken up into multiple tasks.""" @@ -303,16 +305,12 @@ def test_pypi_last_serial(self): repo = self._create_repo_and_sync_with_remote(remote) pub = self._create_publication(repo) distro = self._create_distribution_from_publication(pub) - content_url = urljoin( - PULP_CONTENT_BASE_URL, f"{distro.base_path}/pypi/shelf-reader/json" - ) + content_url = urljoin(PULP_CONTENT_BASE_URL, f"{distro.base_path}/pypi/shelf-reader/json") pypi_url = urljoin(PYPI_HOST, f"{distro.base_path}/pypi/shelf-reader/json/") for url in [content_url, pypi_url]: response = requests.get(url) self.assertIn(PYPI_LAST_SERIAL, response.headers, msg=url) - self.assertEqual( - response.headers[PYPI_LAST_SERIAL], str(PYPI_SERIAL_CONSTANT), msg=url - ) + self.assertEqual(response.headers[PYPI_LAST_SERIAL], str(PYPI_SERIAL_CONSTANT), msg=url) def assert_pypi_json(self, package): """Asserts that shelf-reader package json is correct.""" @@ -322,9 +320,7 @@ def assert_pypi_json(self, package): self.assert_download_info( SHELF_PYTHON_JSON["urls"], package["urls"], "Failed to match URLS" ) - self.assertTrue( - SHELF_PYTHON_JSON["releases"].keys() <= package["releases"].keys() - ) + self.assertTrue(SHELF_PYTHON_JSON["releases"].keys() <= package["releases"].keys()) for version in SHELF_PYTHON_JSON["releases"].keys(): self.assert_download_info( SHELF_PYTHON_JSON["releases"][version], diff --git a/pulp_python/tests/functional/api/test_sync.py b/pulp_python/tests/functional/api/test_sync.py index 79a07b0c9..9de2eb32b 100644 --- a/pulp_python/tests/functional/api/test_sync.py +++ b/pulp_python/tests/functional/api/test_sync.py @@ -1,50 +1,51 @@ # coding=utf-8 """Tests that sync python plugin repositories.""" -import pytest + import unittest +import pytest from pulp_smash import config -from pulp_smash.pulp3.bindings import monitor_task, PulpTaskError +from pulp_smash.pulp3.bindings import PulpTaskError, monitor_task from pulp_smash.pulp3.utils import ( gen_repo, get_added_content_summary, - get_removed_content_summary, get_content_summary, + get_removed_content_summary, +) + +from pulpcore.client.pulp_python import ( + RemotesPythonApi, + RepositoriesPythonApi, + RepositorySyncURL, ) from pulp_python.tests.functional.constants import ( - PYTHON_XS_FIXTURE_SUMMARY, - PYTHON_XS_PACKAGE_COUNT, - PYTHON_INVALID_FIXTURE_URL, - PYTHON_WITHOUT_PRERELEASE_FIXTURE_SUMMARY, - PYTHON_PRERELEASE_TEST_SPECIFIER, - PYTHON_WITH_PRERELEASE_FIXTURE_SUMMARY, - PYTHON_WITH_PRERELEASE_COUNT, - PYTHON_WITHOUT_PRERELEASE_COUNT, + DJANGO_LATEST_3, PYTHON_CONTENT_NAME, - PYTHON_XS_PROJECT_SPECIFIER, - PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_INVALID_FIXTURE_URL, + PYTHON_LG_FIXTURE_COUNTS, + PYTHON_LG_FIXTURE_SUMMARY, + PYTHON_LG_PROJECT_SPECIFIER, PYTHON_MD_FIXTURE_SUMMARY, PYTHON_MD_PACKAGE_COUNT, - PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_PRERELEASE_TEST_SPECIFIER, PYTHON_SM_PACKAGE_COUNT, + PYTHON_SM_PROJECT_SPECIFIER, PYTHON_UNAVAILABLE_PACKAGE_COUNT, PYTHON_UNAVAILABLE_PROJECT_SPECIFIER, - PYTHON_LG_PROJECT_SPECIFIER, - PYTHON_LG_FIXTURE_SUMMARY, - PYTHON_LG_FIXTURE_COUNTS, - DJANGO_LATEST_3, + PYTHON_WITH_PRERELEASE_COUNT, + PYTHON_WITH_PRERELEASE_FIXTURE_SUMMARY, + PYTHON_WITHOUT_PRERELEASE_COUNT, + PYTHON_WITHOUT_PRERELEASE_FIXTURE_SUMMARY, + PYTHON_XS_FIXTURE_SUMMARY, + PYTHON_XS_PACKAGE_COUNT, + PYTHON_XS_PROJECT_SPECIFIER, SCIPY_COUNTS, ) from pulp_python.tests.functional.utils import gen_python_client, gen_python_remote from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 -from pulpcore.client.pulp_python import ( - RepositoriesPythonApi, - RepositorySyncURL, - RemotesPythonApi, -) - class BasicSyncTestCase(unittest.TestCase): """Sync a repository with the python plugin.""" @@ -93,12 +94,8 @@ def test_sync(self): repo = repo_api.read(repo.pulp_href) self.assertIsNotNone(repo.latest_version_href) - self.assertDictEqual( - get_content_summary(repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY - ) - self.assertDictEqual( - get_added_content_summary(repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY - ) + self.assertDictEqual(get_content_summary(repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY) + self.assertDictEqual(get_added_content_summary(repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY) # Sync the repository again. latest_version_href = repo.latest_version_href @@ -108,9 +105,7 @@ def test_sync(self): repo = repo_api.read(repo.pulp_href) self.assertEqual(latest_version_href, repo.latest_version_href) - self.assertDictEqual( - get_content_summary(repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY - ) + self.assertDictEqual(get_content_summary(repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY) class SyncInvalidTestCase(unittest.TestCase): @@ -203,9 +198,7 @@ def test_01_excluding_prereleases(self): by the specifiers. """ - body = gen_python_remote( - includes=PYTHON_PRERELEASE_TEST_SPECIFIER, prereleases=False - ) + body = gen_python_remote(includes=PYTHON_PRERELEASE_TEST_SPECIFIER, prereleases=False) sync_to_remote(self, body, create=True) self.assertDictEqual( @@ -296,9 +289,7 @@ def test_01_basic_include(self): body = gen_python_remote(includes=PYTHON_XS_PROJECT_SPECIFIER) sync_to_remote(self, body, create=True) - self.assertDictEqual( - get_content_summary(self.repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY - ) + self.assertDictEqual(get_content_summary(self.repo.to_dict()), PYTHON_XS_FIXTURE_SUMMARY) def test_02_add_superset_include(self): """ @@ -311,9 +302,7 @@ def test_02_add_superset_include(self): body = {"includes": PYTHON_MD_PROJECT_SPECIFIER} sync_to_remote(self, body) - self.assertDictEqual( - get_content_summary(self.repo.to_dict()), PYTHON_MD_FIXTURE_SUMMARY - ) + self.assertDictEqual(get_content_summary(self.repo.to_dict()), PYTHON_MD_FIXTURE_SUMMARY) def test_03_add_subset_exclude(self): """ @@ -479,7 +468,7 @@ def test_latest_kept_sync_all(self): self.assertEqual( get_content_summary(self.repo.to_dict())[PYTHON_CONTENT_NAME], - PYTHON_LG_FIXTURE_COUNTS["latest_3"] + PYTHON_LG_FIXTURE_COUNTS["latest_3"], ) @@ -521,7 +510,7 @@ def test_sdist_sync_only(self): self.assertEqual( get_content_summary(self.repo.to_dict())[PYTHON_CONTENT_NAME], - PYTHON_LG_FIXTURE_COUNTS["sdist"] + PYTHON_LG_FIXTURE_COUNTS["sdist"], ) def test_bdist_wheel_sync_only(self): @@ -535,7 +524,7 @@ def test_bdist_wheel_sync_only(self): self.assertEqual( get_content_summary(self.repo.to_dict())[PYTHON_CONTENT_NAME], - PYTHON_LG_FIXTURE_COUNTS["bdist_wheel"] + PYTHON_LG_FIXTURE_COUNTS["bdist_wheel"], ) def test_both_together_sync(self): @@ -547,10 +536,7 @@ def test_both_together_sync(self): ) sync_to_remote(self, body, create=True) - self.assertEqual( - get_content_summary(self.repo.to_dict()), - PYTHON_LG_FIXTURE_SUMMARY - ) + self.assertEqual(get_content_summary(self.repo.to_dict()), PYTHON_LG_FIXTURE_SUMMARY) class PlatformExcludeTestCase(unittest.TestCase): @@ -595,7 +581,7 @@ def test_no_windows_sync(self): self.assertEqual( get_content_summary(self.repo.to_dict())[PYTHON_CONTENT_NAME], - SCIPY_COUNTS["total"] - SCIPY_COUNTS["windows"] + SCIPY_COUNTS["total"] - SCIPY_COUNTS["windows"], ) def test_no_macos_sync(self): @@ -609,7 +595,7 @@ def test_no_macos_sync(self): self.assertEqual( get_content_summary(self.repo.to_dict())[PYTHON_CONTENT_NAME], - SCIPY_COUNTS["total"] - SCIPY_COUNTS["macos"] + SCIPY_COUNTS["total"] - SCIPY_COUNTS["macos"], ) def test_no_linux_sync(self): @@ -623,7 +609,7 @@ def test_no_linux_sync(self): self.assertEqual( get_content_summary(self.repo.to_dict())[PYTHON_CONTENT_NAME], - SCIPY_COUNTS["total"] - SCIPY_COUNTS["linux"] + SCIPY_COUNTS["total"] - SCIPY_COUNTS["linux"], ) def test_no_platform_sync(self): @@ -636,8 +622,7 @@ def test_no_platform_sync(self): sync_to_remote(self, body, create=True) self.assertEqual( - get_content_summary(self.repo.to_dict())[PYTHON_CONTENT_NAME], - SCIPY_COUNTS["no_os"] + get_content_summary(self.repo.to_dict())[PYTHON_CONTENT_NAME], SCIPY_COUNTS["no_os"] ) @@ -650,7 +635,7 @@ def test_sync_multiple_filters( includes=PYTHON_LG_PROJECT_SPECIFIER, package_types=["bdist_wheel"], keep_latest_packages=1, - prereleases=False + prereleases=False, ) repo = python_repo_with_sync(remote) @@ -713,9 +698,7 @@ def sync_to_remote(self, body, create=False, mirror=False): monitor_task(remote_task.task) type(self).remote = self.remote_api.read(self.remote.pulp_href) - repository_sync_data = RepositorySyncURL( - remote=self.remote.pulp_href, mirror=mirror - ) + repository_sync_data = RepositorySyncURL(remote=self.remote.pulp_href, mirror=mirror) sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) type(self).repo = self.repo_api.read(self.repo.pulp_href) diff --git a/pulp_python/tests/functional/conftest.py b/pulp_python/tests/functional/conftest.py index 732695e44..1a098b240 100644 --- a/pulp_python/tests/functional/conftest.py +++ b/pulp_python/tests/functional/conftest.py @@ -1,24 +1,25 @@ -import pytest import subprocess import uuid +import pytest from pulp_smash.pulp3.utils import gen_distribution -from pulp_python.tests.functional.utils import gen_python_remote -from pulp_python.tests.functional.constants import PYTHON_URL, PYTHON_EGG_FILENAME from pulpcore.client.pulp_python import ( ApiClient, ContentPackagesApi, DistributionsPypiApi, PublicationsPypiApi, + RemotesPythonApi, RepositoriesPythonApi, RepositoriesPythonVersionsApi, - RemotesPythonApi, ) +from pulp_python.tests.functional.constants import PYTHON_EGG_FILENAME, PYTHON_URL +from pulp_python.tests.functional.utils import gen_python_remote # Bindings API Fixtures + @pytest.fixture def python_bindings_client(cid, bindings_cfg): """Provides the python bindings client object.""" @@ -65,9 +66,11 @@ def python_publication_api_client(python_bindings_client): # Object Generation Fixtures + @pytest.fixture def python_repo_factory(python_repo_api_client, gen_object_with_cleanup): """A factory to generate a Python Repository with auto-cleanup.""" + def _gen_python_repo(**kwargs): kwargs.setdefault("name", str(uuid.uuid4())) return gen_object_with_cleanup(python_repo_api_client, kwargs) @@ -84,6 +87,7 @@ def python_repo(python_repo_factory): @pytest.fixture def python_distribution_factory(python_distro_api_client, gen_object_with_cleanup): """A factory to generate a Python Distribution with auto-cleanup.""" + def _gen_python_distribution(**kwargs): distro_data = gen_distribution(**kwargs) return gen_object_with_cleanup(python_distro_api_client, distro_data) @@ -94,6 +98,7 @@ def _gen_python_distribution(**kwargs): @pytest.fixture def python_publication_factory(python_publication_api_client, gen_object_with_cleanup): """A factory to generate a Python Publication with auto-cleanup.""" + def _gen_python_publication(repository, version=None): if version: body = {"repository_version": f"{repository.versions_href}{version}/"} @@ -107,6 +112,7 @@ def _gen_python_publication(repository, version=None): @pytest.fixture def python_remote_factory(python_remote_api_client, gen_object_with_cleanup): """A factory to generate a Python Remote with auto-cleanup.""" + def _gen_python_remote(**kwargs): body = gen_python_remote(**kwargs) return gen_object_with_cleanup(python_remote_api_client, body) @@ -119,6 +125,7 @@ def python_repo_with_sync( python_repo_api_client, python_repo_factory, python_remote_factory, monitor_task ): """A factory to generate a Python Repository synced with the passed in Remote.""" + def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): kwargs = {} if pulp_domain := body.get("pulp_domain"): @@ -135,6 +142,7 @@ def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): @pytest.fixture def download_python_file(tmp_path, http_get): """Download a Python file and return its path.""" + def _download_python_file(relative_path, url): file_path = tmp_path / relative_path with open(file_path, mode="wb") as f: @@ -153,6 +161,7 @@ def python_file(download_python_file): @pytest.fixture def python_content_factory(python_content_api_client, download_python_file, monitor_task): """A factory to create a Python Package Content.""" + def _gen_python_content(relative_path=PYTHON_EGG_FILENAME, url=None, **body): body["relative_path"] = relative_path if url: @@ -185,6 +194,7 @@ def shelf_reader_cleanup(): @pytest.fixture def python_content_summary(python_repo_api_client, python_repo_version_api_client): """Get a summary of the repository version's content.""" + def _gen_summary(repository_version=None, repository=None, version=None): if repository_version is None: repo_href = get_href(repository) diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index f0e53046f..2692a73a6 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -2,14 +2,13 @@ from pulp_smash import config from pulp_smash.pulp3.constants import ( + BASE_CONTENT_PATH, BASE_DISTRIBUTION_PATH, BASE_PUBLICATION_PATH, BASE_REMOTE_PATH, BASE_REPO_PATH, - BASE_CONTENT_PATH, ) - PULP_FIXTURES_BASE_URL = config.get_config().get_fixtures_url() PYPI_URL = "https://pypi.org/" @@ -45,9 +44,7 @@ "pylint", # matches 0 ] PYTHON_UNAVAILABLE_PACKAGE_COUNT = 5 -PYTHON_UNAVAILABLE_FIXTURE_SUMMARY = { - PYTHON_CONTENT_NAME: PYTHON_UNAVAILABLE_PACKAGE_COUNT -} +PYTHON_UNAVAILABLE_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_UNAVAILABLE_PACKAGE_COUNT} # no "name" field PYTHON_INVALID_SPECIFIER_NO_NAME = [ @@ -71,13 +68,9 @@ "Django", ] PYTHON_WITH_PRERELEASE_COUNT = 46 -PYTHON_WITH_PRERELEASE_FIXTURE_SUMMARY = { - PYTHON_CONTENT_NAME: PYTHON_WITH_PRERELEASE_COUNT -} +PYTHON_WITH_PRERELEASE_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_WITH_PRERELEASE_COUNT} PYTHON_WITHOUT_PRERELEASE_COUNT = 30 -PYTHON_WITHOUT_PRERELEASE_FIXTURE_SUMMARY = { - PYTHON_CONTENT_NAME: PYTHON_WITHOUT_PRERELEASE_COUNT -} +PYTHON_WITHOUT_PRERELEASE_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_WITHOUT_PRERELEASE_COUNT} # Specifier for basic sync / publish tests. PYTHON_XS_PROJECT_SPECIFIER = ["shelf-reader"] # matches 2 @@ -113,21 +106,15 @@ "aiohttp-3.2.1.tar.gz": "1b95d53f8dac13898f0a3e4af76f6f36d540fbfaefc4f4c9f43e436fa0e53d22", "aiohttp-3.2.0.tar.gz": "1be3903fe6a36d20492e74efb326522dd4702bf32b45ffc7acbc0fb34ab240a6", "Django-1.10.4.tar.gz": "fff7f062e510d812badde7cfc57745b7779edb4d209b2bc5ea8d954c22305c2b", - "Django-1.10.4-py2.py3-none-any.whl": - "a8e1a552205cda15023c39ecf17f7e525e96c5b0142e7879e8bd0c445351f2cc", + "Django-1.10.4-py2.py3-none-any.whl": "a8e1a552205cda15023c39ecf17f7e525e96c5b0142e7879e8bd0c445351f2cc", "Django-1.10.3.tar.gz": "6f92f08dee8a1bd7680e098a91bf5acd08b5cdfe74137f695b60fd79f4478c30", - "Django-1.10.3-py2.py3-none-any.whl": - "94426cc28d8721fbf13c333053f08d32427671a4ca7986f7030fc82bdf9c88c1", + "Django-1.10.3-py2.py3-none-any.whl": "94426cc28d8721fbf13c333053f08d32427671a4ca7986f7030fc82bdf9c88c1", "Django-1.10.2.tar.gz": "e127f12a0bfb34843b6e8c82f91e26fff6445a7ca91d222c0794174cf97cbce1", - "Django-1.10.2-py2.py3-none-any.whl": - "4d48ab8e84a7c8b2bc4b2f4f199bc3a8bfcc9cbdbc29e355ac5c44a501d73a1a", + "Django-1.10.2-py2.py3-none-any.whl": "4d48ab8e84a7c8b2bc4b2f4f199bc3a8bfcc9cbdbc29e355ac5c44a501d73a1a", "Django-1.10.1.tar.gz": "d6e6c5b25cb67f46afd7c82f536529b11981183423dad8932e15bce93d1a24f3", - "Django-1.10.1-py2.py3-none-any.whl": - "3d689905cd0635bbb33b87f9a5df7ca70a3db206faae4ec58cda5e7f5f47050d", - "celery-4.2.0-py2.py3-none-any.whl": - "2082cbd82effa8ac8a8a58977d70bb203a9f362817e3b66f4578117b9f93d8a9", - "celery-4.1.1-py2.py3-none-any.whl": - "6fc4678d1692af97e137b2a9f1c04efd8e7e2fb7134c5c5ad60738cdd927762f", + "Django-1.10.1-py2.py3-none-any.whl": "3d689905cd0635bbb33b87f9a5df7ca70a3db206faae4ec58cda5e7f5f47050d", + "celery-4.2.0-py2.py3-none-any.whl": "2082cbd82effa8ac8a8a58977d70bb203a9f362817e3b66f4578117b9f93d8a9", + "celery-4.1.1-py2.py3-none-any.whl": "6fc4678d1692af97e137b2a9f1c04efd8e7e2fb7134c5c5ad60738cdd927762f", } PYTHON_MD_PROJECT_SPECIFIER = [ @@ -179,9 +166,7 @@ # Intended to be used with the XS specifier PYTHON_WHEEL_FILENAME = "shelf_reader-0.1-py2-none-any.whl" -PYTHON_WHEEL_URL = urljoin( - urljoin(PYTHON_FIXTURES_URL, "packages/"), PYTHON_WHEEL_FILENAME -) +PYTHON_WHEEL_URL = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), PYTHON_WHEEL_FILENAME) PYTHON_WHEEL_SHA256 = "2eceb1643c10c5e4a65970baf63bde43b79cbdac7de81dae853ce47ab05197e9" PYTHON_XS_FIXTURE_CHECKSUMS = { @@ -248,11 +233,15 @@ # "provides_dist": [], # "obsoletes_dist": [], # "requires_external": [], - "classifiers": ['Development Status :: 4 - Beta', 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', - 'Natural Language :: English', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7'], + "classifiers": [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", + "Natural Language :: English", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + ], "downloads": {"last_day": -1, "last_month": -1, "last_week": -1}, # maybe add description, license is long for this one } @@ -288,7 +277,7 @@ "download_concurrency": 3, "policy": "on_demand", "prereleases": False, - "excludes": ["example1", "example2"] + "excludes": ["example1", "example2"], } BANDERSNATCH_CONF = b""" @@ -352,7 +341,7 @@ "requires_python": None, "size": 22455, "yanked": False, - "yanked_reason": None + "yanked_reason": None, } SHELF_SDIST_PYTHON_DOWNLOAD = { @@ -370,7 +359,7 @@ "requires_python": None, "size": 19097, "yanked": False, - "yanked_reason": None + "yanked_reason": None, } SHELF_0DOT1_RELEASE = [SHELF_BDIST_PYTHON_DOWNLOAD, SHELF_SDIST_PYTHON_DOWNLOAD] @@ -378,9 +367,6 @@ SHELF_PYTHON_JSON = { "info": PYTHON_INFO_DATA, "last_serial": 0, - "releases": { - "0.1": SHELF_0DOT1_RELEASE - }, - "urls": SHELF_0DOT1_RELEASE - + "releases": {"0.1": SHELF_0DOT1_RELEASE}, + "urls": SHELF_0DOT1_RELEASE, } diff --git a/pulp_python/tests/functional/utils.py b/pulp_python/tests/functional/utils.py index 9fd54e8c9..e5304b6f7 100644 --- a/pulp_python/tests/functional/utils.py +++ b/pulp_python/tests/functional/utils.py @@ -1,13 +1,13 @@ # coding=utf-8 """Utilities for tests for the python plugin.""" + from functools import partial -from unittest import SkipTest, TestCase from tempfile import NamedTemporaryFile +from unittest import SkipTest, TestCase from urllib.parse import urljoin -from lxml import html +from lxml import html from pulp_smash import config, selectors -from pulp_smash.utils import http_get from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import ( gen_distribution, @@ -17,30 +17,33 @@ require_pulp_3, require_pulp_plugins, ) +from pulp_smash.utils import http_get -from pulp_python.tests.functional.constants import ( - PYTHON_CONTENT_NAME, - PYTHON_FIXTURE_URL, - PYTHON_URL, - PYTHON_EGG_FILENAME, - PYTHON_XS_PROJECT_SPECIFIER, -) - -from pulpcore.client.pulpcore import ( - ApiClient as CoreApiClient, - ArtifactsApi, - TasksApi, -) from pulpcore.client.pulp_python import ApiClient as PythonApiClient from pulpcore.client.pulp_python import ( - DistributionsPypiApi, - RepositoriesPythonApi, ContentPackagesApi, + DistributionsPypiApi, PublicationsPypiApi, PythonPythonPublication, RemotesPythonApi, + RepositoriesPythonApi, RepositorySyncURL, ) +from pulpcore.client.pulpcore import ( + ApiClient as CoreApiClient, +) +from pulpcore.client.pulpcore import ( + ArtifactsApi, + TasksApi, +) + +from pulp_python.tests.functional.constants import ( + PYTHON_CONTENT_NAME, + PYTHON_EGG_FILENAME, + PYTHON_FIXTURE_URL, + PYTHON_URL, + PYTHON_XS_PROJECT_SPECIFIER, +) cfg = config.get_config() configuration = cfg.get_bindings_config() @@ -346,6 +349,7 @@ def ensure_simple(simple_url, packages, sha_digests=None): in the simple index and thus be accessible from the distribution, but if one can't see it how would one know that it's there?* """ + def explore_links(page_url, page_name, links_found, msgs): legit_found_links = [] page = html.fromstring(http_get(page_url)) @@ -378,8 +382,16 @@ def explore_links(page_url, page_name, links_found, msgs): package = package_link.split("/")[-1] if sha_digests[package] != sha: msgs += f"\nRelease has bad sha256 attached to it {package}" - msgs += "".join(map(lambda x: f"\nSimple link not found for {x}", - [name for name, val in packages_found.items() if not val])) - msgs += "".join(map(lambda x: f"\nReleases link not found for {x}", - [name for name, val in releases_found.items() if not val])) + msgs += "".join( + map( + lambda x: f"\nSimple link not found for {x}", + [name for name, val in packages_found.items() if not val], + ) + ) + msgs += "".join( + map( + lambda x: f"\nReleases link not found for {x}", + [name for name, val in releases_found.items() if not val], + ) + ) return len(msgs) == 0, msgs diff --git a/pulp_python/tests/upgrade/post/test_publish.py b/pulp_python/tests/upgrade/post/test_publish.py index c6b7d46e5..3a50af36d 100644 --- a/pulp_python/tests/upgrade/post/test_publish.py +++ b/pulp_python/tests/upgrade/post/test_publish.py @@ -1,34 +1,35 @@ """Tests that publish file plugin repositories.""" + import unittest from random import choice from pulp_smash import config from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import ( + gen_distribution, gen_repo, get_content, - gen_distribution, get_versions, ) -from pulp_python.tests.functional.constants import PYTHON_CONTENT_NAME -from pulp_python.tests.functional.utils import ( - gen_python_client, - gen_python_remote, -) -from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 - from pulpcore.client.pulp_python import ( DistributionsPypiApi, PublicationsPypiApi, + PythonPythonPublication, + RemotesPythonApi, RepositoriesPythonApi, RepositoryAddRemoveContent, RepositorySyncURL, - RemotesPythonApi, - PythonPythonPublication, ) from pulpcore.client.pulp_python.exceptions import ApiException +from pulp_python.tests.functional.constants import PYTHON_CONTENT_NAME +from pulp_python.tests.functional.utils import ( + gen_python_client, + gen_python_remote, +) +from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + class PublishAnyRepoVersionTestCase(unittest.TestCase): """Test whether a particular repository version can be published. diff --git a/pulp_python/tests/upgrade/pre/test_publish.py b/pulp_python/tests/upgrade/pre/test_publish.py index 899155a26..044d7eb76 100644 --- a/pulp_python/tests/upgrade/pre/test_publish.py +++ b/pulp_python/tests/upgrade/pre/test_publish.py @@ -1,34 +1,35 @@ """Tests that publish file plugin repositories.""" + import unittest from random import choice from pulp_smash import config from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import ( + gen_distribution, gen_repo, get_content, - gen_distribution, get_versions, ) -from pulp_python.tests.functional.constants import PYTHON_CONTENT_NAME -from pulp_python.tests.functional.utils import ( - gen_python_client, - gen_python_remote, -) -from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 - from pulpcore.client.pulp_python import ( DistributionsPypiApi, PublicationsPypiApi, - RepositoryAddRemoveContent, + PythonPythonPublication, + RemotesPythonApi, RepositoriesPythonApi, + RepositoryAddRemoveContent, RepositorySyncURL, - RemotesPythonApi, - PythonPythonPublication, ) from pulpcore.client.pulp_python.exceptions import ApiException +from pulp_python.tests.functional.constants import PYTHON_CONTENT_NAME +from pulp_python.tests.functional.utils import ( + gen_python_client, + gen_python_remote, +) +from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + class PublishAnyRepoVersionTestCase(unittest.TestCase): """Test whether a particular repository version can be published. diff --git a/pyproject.toml b/pyproject.toml index 8488e2427..729c743e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,3 +74,30 @@ replace = "version = \"{new_version}\"" [[tool.bumpversion.files]] filename = "./setup.py" + + +[tool.ruff] +# This section is managed by the plugin template. Do not edit manually. +line-length = 100 +extend-exclude = [ + "docs/**", + "**/migrations/*.py", +] + +[tool.ruff.lint] +# This section is managed by the plugin template. Do not edit manually. +extend-select = [ + "I", + "INT", + "TID", + "T10", +] + +[tool.ruff.lint.flake8-tidy-imports.banned-api] +# This section is managed by the plugin template. Do not edit manually. +"pulpcore.app".msg = "The 'pulpcore' apis must only be consumed via 'pulpcore.plugin'." + +[tool.ruff.lint.isort] +# This section is managed by the plugin template. Do not edit manually. +sections = { second-party = ["pulpcore"] } +section-order = ["future", "standard-library", "third-party", "second-party", "first-party", "local-folder"] diff --git a/setup.py b/setup.py index e4c5fed49..dea889e97 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from setuptools import setup, find_packages +from setuptools import find_packages, setup with open("requirements.txt") as requirements: requirements = requirements.readlines() @@ -32,5 +32,9 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", ), - entry_points={"pulpcore.plugin": ["pulp_python = pulp_python:default_app_config", ]}, + entry_points={ + "pulpcore.plugin": [ + "pulp_python = pulp_python:default_app_config", + ] + }, ) diff --git a/template_config.yml b/template_config.yml index 2112c3ba7..f168108a0 100644 --- a/template_config.yml +++ b/template_config.yml @@ -6,9 +6,7 @@ # After editing this file please always reapply the plugin template before committing any changes. --- -black: false check_commit_message: true -check_gettext: true check_manifest: true check_stray_pulpcore_imports: true ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" @@ -23,10 +21,9 @@ deploy_to_pypi: true disabled_redis_runners: [] docker_fixtures: false extra_files: [] -flake8: true -flake8_ignore: [] github_org: "pulp" latest_release_branch: null +lint_ignore: [] lint_requirements: true os_required_packages: [] parallel_test_workers: 8 @@ -80,7 +77,6 @@ pulp_settings_s3: staticfiles: BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" api_root: "/rerouted/djnd/" -pydocstyle: true release_email: "pulp-infra@redhat.com" release_user: "pulpbot" stalebot: true