Skip to content

Commit

Permalink
Make noop async
Browse files Browse the repository at this point in the history
closes #9368

(cherry picked from commit d2cf02f)
  • Loading branch information
fao89 committed Oct 5, 2021
1 parent 933bdc9 commit 84640e6
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 39 deletions.
1 change: 1 addition & 0 deletions CHANGES/9368.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Ensure syncing is async safe
76 changes: 37 additions & 39 deletions pulp_ansible/app/tasks/collections.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from galaxy_importer.exceptions import ImporterError
from pkg_resources import Requirement

from pulpcore.plugin.constants import TASK_STATES
from pulpcore.plugin.models import (
Artifact,
ContentArtifact,
Expand Down Expand Up @@ -122,10 +121,14 @@ def sync(remote_pk, repository_pk, mirror, optimize):
remote = CollectionRemote.objects.get(pk=remote_pk)
repository = AnsibleRepository.objects.get(pk=repository_pk)

is_repo_remote = False
if repository.remote:
is_repo_remote = remote.pk == repository.remote.pk

if not remote.url:
raise ValueError(_("A CollectionRemote must have a 'url' specified to synchronize."))

first_stage = CollectionSyncFirstStage(remote, repository, optimize)
first_stage = CollectionSyncFirstStage(remote, repository, is_repo_remote, optimize)
d_version = AnsibleDeclarativeVersion(first_stage, repository, mirror=mirror)
repo_version = d_version.create()

Expand Down Expand Up @@ -349,19 +352,21 @@ class CollectionSyncFirstStage(Stage):
The first stage of a pulp_ansible sync pipeline.
"""

def __init__(self, remote, repository, optimize):
def __init__(self, remote, repository, is_repo_remote, optimize):
"""
The first stage of a pulp_ansible sync pipeline.
Args:
remote (CollectionRemote): The remote data to be used when syncing
repository (AnsibleRepository): The repository being syncedself.
is_repo_remote (bool): True if the remote is the repository's remote.
optimize (boolean): Whether to optimize sync or not.
"""
super().__init__()
self.remote = remote
self.repository = repository
self.is_repo_remote = is_repo_remote
self.optimize = optimize
self.collection_info = parse_collections_requirements_file(remote.requirements_file)
self.add_dependents = self.collection_info and self.remote.sync_dependencies
Expand Down Expand Up @@ -705,46 +710,39 @@ async def _find_all_collections(self):
async def _should_we_sync(self):
"""Check last synced metadata time."""
msg = _("no_change: Checking if remote changed since last sync.")
noop = ProgressReport(message=msg, code="sync.no_change")
noop.state = TASK_STATES.COMPLETED
noop.save()

if not self.repository.remote:
return True

if self.remote != self.repository.remote.cast():
return True

root, api_version = await self._get_root_api(self.remote.url)
if api_version == 3:
downloader = self.remote.get_downloader(
url=root, silence_errors_for_response_status_codes={404}
)
try:
metadata = parse_metadata(await downloader.run())
except FileNotFoundError:
async with ProgressReport(message=msg, code="sync.no_change") as noop:
if not self.is_repo_remote:
return True

try:
self.last_synced_metadata_time = parse_datetime(metadata["published"])
except KeyError:
return True

sources = set()
if self.collection_info:
sources = {r.source for r in self.collection_info if r.source}
sources.add(self.remote.url)
if len(sources) > 1:
return True
root, api_version = await self._get_root_api(self.remote.url)
if api_version == 3:
downloader = self.remote.get_downloader(
url=root, silence_errors_for_response_status_codes={404}
)
try:
metadata = parse_metadata(await downloader.run())
except FileNotFoundError:
return True

if self.last_synced_metadata_time == self.repository.last_synced_metadata_time:
noop.message = _(
"no-op: {remote} did not change since last sync - {published}".format(
remote=self.remote.url, published=self.last_synced_metadata_time
try:
self.last_synced_metadata_time = parse_datetime(metadata["published"])
except KeyError:
return True

sources = set()
if self.collection_info:
sources = {r.source for r in self.collection_info if r.source}
sources.add(self.remote.url)
if len(sources) > 1:
return True

if self.last_synced_metadata_time == self.repository.last_synced_metadata_time:
noop.message = _(
"no-op: {remote} did not change since last sync - {published}".format(
remote=self.remote.url, published=self.last_synced_metadata_time
)
)
)
noop.save()
return False
return False

return True

Expand Down

0 comments on commit 84640e6

Please sign in to comment.