-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introduce concept of phase1 commit [RHELDST-20490]
Internally, items have always been categorized into either phase1 or phase2 during commit, with phase2 being committed last. This is needed to ensure, for example, that a repomd.xml is not committed until after all other files referenced by it have been committed. This change now documents that concept and exposes it via the API. By default, clients get the same behavior as always, but they can now also explicitly request a phase1 commit. A phase1 commit writes phase1 items to the DB, same as always, but then stops without proceeding to write phase2 items. It also leaves the publish open for later modifications and a later phase2 commit. === Why? === This will be used to solve the following Pub/rhsm-pulp/exodus-gw integration problem: Imagine that you need to publish multiple Pulp yum repos. You want this to be atomic, so you want them all to use the same exodus-gw publish. You ask the repos to publish, and they succeed, but then something goes wrong during the later exodus-gw commit which is performed outside of Pulp. So, you restart the whole process and republish the repos again using a new exodus-gw publish, and commit that successfully. Problem: as far as Pulp is concerned, the publish tasks from the first attempt were completely successful, as it does not "see" the later failure to commit. Therefore, Pulp incorrectly thinks that the RPMs processed by those tasks are fully published to the CDN, and so skips publishing them during later tasks. This leads to missing content on the CDN. Solution: exodus-rsync, as invoked by Pulp during publish, will request a phase1 commit during publish of each repo. This ensures the processed RPMs (or other non-entrypoint files) are fully published on the CDN by the time the Pulp publish task succeeds, matching Pulp's expectations. The publish of repomd.xml is still held back until a later phase2 commit, retaining the atomic semantics across multiple repos.
- Loading branch information
Showing
8 changed files
with
585 additions
and
95 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
"""Adds columns supporting phase1 commit | ||
Revision ID: 1d51b80e64ba | ||
Revises: 0d88322fe0b3 | ||
Create Date: 2023-10-02 11:44:04.604593 | ||
""" | ||
import sqlalchemy as sa | ||
from alembic import op | ||
|
||
from exodus_gw.migrations.test import tested_by | ||
|
||
# revision identifiers, used by Alembic. | ||
revision = "1d51b80e64ba" | ||
down_revision = "0d88322fe0b3" | ||
branch_labels = None | ||
depends_on = None | ||
|
||
|
||
def upgrade_testdata(): | ||
# Make a commit_task exist so we can verify it's transformed | ||
# into phase2 commit | ||
task_id = "41400ff1-9198-4b35-b24e-a71a29957ae1" | ||
publish_id = "f7a38eb1-0d75-4245-a4ef-3dfd02d8129f" | ||
op.bulk_insert( | ||
sa.table( | ||
"tasks", | ||
sa.column("id", sa.Uuid(as_uuid=False)), | ||
sa.column("state", sa.String()), | ||
sa.column("type", sa.String()), | ||
), | ||
[ | ||
{ | ||
"id": task_id, | ||
"state": "NOT_STARTED", | ||
"type": "commit", | ||
}, | ||
], | ||
) | ||
op.bulk_insert( | ||
sa.table( | ||
"commit_tasks", | ||
sa.column("id", sa.Uuid(as_uuid=False)), | ||
sa.column("publish_id", sa.Uuid(as_uuid=False)), | ||
), | ||
[ | ||
{ | ||
"id": task_id, | ||
"publish_id": publish_id, | ||
}, | ||
], | ||
) | ||
|
||
# and make some items exist too, which will be marked dirty | ||
op.bulk_insert( | ||
sa.table( | ||
"items", | ||
sa.column("id", sa.Uuid(as_uuid=False)), | ||
sa.column("web_uri", sa.String()), | ||
sa.column("object_key", sa.String()), | ||
sa.column("publish_id", sa.Uuid(as_uuid=False)), | ||
), | ||
[ | ||
{ | ||
"id": "f021da4d-5c3b-483f-af8d-85117fb64b2c", | ||
"publish_id": publish_id, | ||
"web_uri": "/foo", | ||
"object_key": "a1b2c3", | ||
}, | ||
{ | ||
"id": "9dafa529-03e6-4412-85db-f681ea98d75d", | ||
"publish_id": publish_id, | ||
"web_uri": "/bar", | ||
"object_key": "a1b2c3", | ||
}, | ||
], | ||
) | ||
|
||
|
||
@tested_by(upgrade_testdata) | ||
def upgrade(): | ||
op.add_column( | ||
"commit_tasks", | ||
sa.Column( | ||
"commit_mode", sa.String(), nullable=False, server_default="phase2" | ||
), | ||
) | ||
op.add_column( | ||
"items", | ||
sa.Column( | ||
"dirty", sa.Boolean(), nullable=False, server_default="TRUE" | ||
), | ||
) | ||
|
||
|
||
def downgrade(): | ||
op.drop_column("items", "dirty") | ||
op.drop_column("commit_tasks", "commit_mode") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.