Migration workflow

Migration workflow steps

Restore test

Index:

sonar

poetry run invenio rero es snapshot create backup -n migration-20250924 -w

sonar dev

uv run invenio rero es snapshot repository create backup /invenio/storage/prod/es-backup

# clean indices
uv run invenio rero es snapshot list backup
uv run invenio rero es snapshot restore backup migration-20250924 --wait

Files [root@nagma.rero.ch]

rm -rf /invenio/storage/dev/files/*
cp -rv /invenio/storage/prod/files/* /invenio/storage/dev/files/
# rsync -avr --delete /data/sonar/prod/files/ /data/sonar/dev/files/  --exclude=lost+found/
chown -R debian9:debian9 /data/sonar/dev/files/*

DB kerosen

export kerosen

ssh root@kerosen
su - postgres
pg_dump -Fc sonar > /database/dump/sonar_prod_20250924.sql
exit

#### import sonar dev
pg_restore  --username=sonar --jobs=4  --dbname=sonar --clean /invenio/storage/backup/dump/sonar_prod_20250924.sql
exit

Correct problems

poetry run /invenio/storage/scripts/correct_sonar.py utils correct-wrong-pids -t doc -v -c

# v1.11.0 --> v1.12.0

DB migration 9 -> 17 ???

Alembic

uv run invenio alembic stamp 428b919be0ea
uv run invenio alembic upgrade
# TODO: uv run invenio rero es index update-mapping

Update mapping

from invenio_search import current_search_client
current_search_client.indices.create('records-record-v1.0.0')

uv run invenio rero es index update-mapping

Documents

Remove _oai in documents.

# estimation for 300'000 records: 5h
from invenio_pidstore.models import PersistentIdentifier
from rero_invenio_base.modules.tasks import run_on_worker
from rero_invenio_base.modules.utils import chunk


code = '''
def pop_oai(_ids):
    from sonar.modules.documents.api import DocumentRecord
    from invenio_db import db
    success = 0
    failed = 0
    for uuid in _ids:
        try:
            record = DocumentRecord.get_record(uuid)
            if record.get('_oai'):
                record['_oai'].pop('updated', None)
                record['_oai'].pop('sets', None)
                record.commit()
                db.session.commit()
                record.reindex()
                success += 1
        except Exception as err:
            print(f"Exception {uuid}: {err}")
            failed += 1
    return {"success": success, "failed": failed}
'''


parallel = 7
ids = [pid.object_uuid for pid in PersistentIdentifier.query.filter_by(pid_type='doc').all()]
for count, c in enumerate(chunk([str(val) for val in ids], len(ids) // parallel), 1):
    res = run_on_worker.delay(code, 'pop_oai', c)
    print('documents', count, len(c), res)

index_name=`uv run invenio rero es index info -i documents |grep -v percolator`
echo $index_name
uv run invenio rero es index move documents $index_name documents-document-v1.0.0-20250923 -v
uv run invenio index delete $index_name --yes-i-know

Deposits

Remove category from deposit.

from sonar.modules.deposits.api import DepositRecord
n = 0
for pid in PersistentIdentifier.query.filter_by(
            pid_type='depo').all():
    try:
        record = DepositRecord.get_record(pid.object_uuid) 
        n += 1
        changed = False
        for f in record.get('_files', []):
            if 'category' in f:
                del f['category']
                changed = True
            if 'embargo' in f:
                del f['embargo']
                changed = True
        if changed:
            record.commit()
            db.session.commit()
            record.reindex()
    except:
        print(f"Failed to update record {pid.pid_value}")

print(f"Updated {n} records")

index_name=`uv run invenio rero es index info -i deposits`
echo $index_name
uv run invenio rero es index move deposits $index_name deposits-deposit-v1.0.0-20250923 -v
uv run invenio index delete $index_name --yes-i-know

Organisations

index_name=`uv run invenio rero es index info -i organisations`
echo $index_name
uv run invenio rero es index move organisations $index_name organisations-organisation-v1.0.0-20250923 -v
uv run invenio index delete $index_name --yes-i-know

ConfigMap

• replace INVENIO_SQLALCHEMY_POOL_RECYCLE by INVENIO_SQLALCHEMY_ENGINE_OPTIONS: '{ "pool_recycle": 360 }'
• set the config in the sqlalchemy container/server by setting PGOPTIONS to -c statement_timeout=30s -c idle_in_transaction_session_timeout=60s