fix(opensearch): preserve OS index store on phase rollback during in-flight reindex (#36471)

docs/backend/OPENSEARCH_MIGRATION.md

@@ -670,6 +670,24 @@ rollback, with no impact on normal operation.
 > **Status**: Option 1 (runbook) is the only mitigation currently in place. Option 2 and 3 are
 > not yet implemented — tracked as technical debt before Phase 2 goes to production.
 
+#### Phase rollback during an in-flight full reindex (#36471)
+
+Rolling `FEATURE_FLAG_OPEN_SEARCH_PHASE` back to 0 **while a full reindex is draining the
+journal** is a distinct hazard from the mapping drift above. The phase is re-read per journal
+batch, so the remaining entries index to ES only and the OS reindex pair freezes partially
+populated. The ES switchover then completes in Phase 0.
+
+**Fixed behavior:** the Phase-0 switchover (and abort) now treats this state as an OS reindex
+abort — the active OS working/live rows survive in the store (the legacy `indicies` update is
+scoped to its own NULL-version rows), the OS reindex slots are cleared, and the partial physical
+`.os` pair is deleted from the cluster so a later boot catchup can never adopt it as active. The
+abort is logged at WARN with the deleted index names.
+
+**Operational rule:** the OS pair that survives the rollback is the *old* one — it stops
+receiving writes in Phase 0 and drifts exactly as described above. Before re-activating Phase 2,
+trigger a full reindex so OS is rebuilt in a dual-write phase. Prefer letting an in-flight
+reindex finish (or aborting it explicitly) over flipping the phase mid-drain.
+
 ---
 
 ### Fan-out routing with divergent index names — resolved (#35640)

dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImpl.java

@@ -1424,6 +1424,11 @@ public synchronized boolean fullReindexSwitchover(Connection conn, final boolean
                 } catch (Exception osEx) {
                     Logger.warn(this, "Could not mirror reindex switchover to OS store", osEx);
                 }
+            } else {
+                // Phase 0 with OS reindex slots present = the phase flag was rolled back while
+                // a dual-write full reindex was in flight (#36471). The OS pair is partial and
+                // must never survive as adoptable state.
+                abortStrandedOsReindex();
             }
 
             // Async: merge index segments and expand replicas on the newly active indices,
@@ -1523,6 +1528,65 @@ private boolean fullReindexSwitchoverOS(final boolean forceSwitch) throws Except
         return true;
     }
 
+    /**
+     * Aborts a stranded OS reindex left behind by a Phase-0 rollback during an in-flight
+     * dual-write full reindex (#36471). When the phase flag is rolled back mid-journal-drain,
+     * the OS reindex pair stops receiving writes and stays partial; the OS store still holds
+     * the reindex slots pointing at it. Left alone, those slots make {@code isInFullReindex()}
+     * report true again on a later flip to Phase 2 (triggering a switchover over null ES
+     * pointers), and the partial indices are the exact {@code .os} twins of the promoted ES
+     * names — what a boot catchup would mirror-adopt as active, silently serving a fraction
+     * of the content.
+     *
+     * <p>Clears the slots first (the safety-critical part — active working/live pointers are
+     * preserved), then deletes the partial physical indices best-effort. Never throws: this
+     * runs inside the ES switchover/abort, which must not be undone by OS housekeeping.</p>
+     */
+    private void abortStrandedOsReindex() {
+        try {
+            final Optional<VersionedIndices> osExisting =
+                    versionedIndicesAPI.loadDefaultVersionedIndices();
+            final Optional<String> reindexWorking =
+                    osExisting.flatMap(VersionedIndices::reindexWorking);
+            final Optional<String> reindexLive =
+                    osExisting.flatMap(VersionedIndices::reindexLive);
+            if (reindexWorking.isEmpty() && reindexLive.isEmpty()) {
+                return;
+            }
+            Logger.warn(this, "Migration phase was rolled back to 0 during a full reindex:"
+                    + " aborting the OS reindex — clearing the OS reindex slots and deleting the"
+                    + " partial indices [" + reindexWorking.orElse("none") + ", "
+                    + reindexLive.orElse("none") + "] so they can never be adopted as active"
+                    + " (#36471)");
+
+            final VersionedIndicesImpl.Builder osBuilder = VersionedIndicesImpl.builder();
+            osExisting.flatMap(VersionedIndices::working).ifPresent(osBuilder::working);
+            osExisting.flatMap(VersionedIndices::live).ifPresent(osBuilder::live);
+            osExisting.flatMap(VersionedIndices::siteSearch).ifPresent(osBuilder::siteSearch);
+            // reindexWorking / reindexLive intentionally omitted → cleared
+            final VersionedIndices rebuilt = osBuilder.build();
+            if (rebuilt.hasAnyIndex()) {
+                versionedIndicesAPI.saveIndices(rebuilt);
+            } else {
+                // The reindex slots were the only OS pointers (e.g. the active OS pair was
+                // deleted via the index-management flow mid-reindex). saveIndices contractually
+                // rejects an empty record — remove the version row instead, same as
+                // clearOsStorePointer (#35640), so the physical deletes below still run.
+                versionedIndicesAPI.removeVersion(VersionedIndices.OPENSEARCH_3X);
+            }
+
+            for (final Optional<String> name : List.of(reindexWorking, reindexLive)) {
+                name.ifPresent(idx -> Try.run(() -> operationsOS.indexAPI().delete(idx))
+                        .onFailure(e -> Logger.warn(this,
+                                "Could not delete partial OS reindex index " + idx
+                                        + " — delete it manually", e)));
+            }
+        } catch (Exception osEx) {
+            Logger.warn(this, "Could not abort the stranded OS reindex (#36471) — the OS store"
+                    + " may still hold reindex slots pointing at partial indices", osEx);
+        }
+    }
+
     /**
      * Optimizes (force-merges) the newly-promoted indices after a reindex switchover, targeting
      * each provider with the names it actually holds: ES with its bare names, OS with its
@@ -2899,6 +2963,10 @@ public void fullReindexAbort() {
                 } catch (Exception osEx) {
                     Logger.warn(this, "Could not clear OS reindex slots during abort", osEx);
                 }
+            } else {
+                // Same mid-reindex rollback state as the switchover path (#36471), reached when
+                // the operator aborts instead of letting the journal drain.
+                abortStrandedOsReindex();
             }
 
         } catch (Exception e) {

dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/IndiciesFactory.java

@@ -86,7 +86,12 @@ public void point(final IndiciesInfo newInfo) throws DotDataException {
         }
         DotConnect dc = new DotConnect();
         final String insertSQL = "INSERT INTO indicies VALUES(?,?)";
-        final String deleteSQL = "DELETE from indicies where index_type=? or index_name=?";
+        // Scoped to index_version IS NULL: this legacy store only owns the ES rows (the same
+        // rows loadIndicies reads). The OS migration rows carry a non-NULL index_version in the
+        // shared table and are managed by VersionedIndicesAPI — an unscoped delete-by-type wipes
+        // them on every ES switchover, which in Phase 0 orphans the OS index store (#36471).
+        final String deleteSQL =
+                "DELETE from indicies where (index_type=? or index_name=?) and index_version is null";
         for (IndexType type : IndexType.values()) {
             final String indexType = type.toString().toLowerCase();
             final String newValue = Try.of(() -> (String) PropertyUtils

dotcms-integration/src/test/java/com/dotcms/OpenSearchUpgradeSuite.java

@@ -2,6 +2,7 @@
 
 import com.dotcms.content.elasticsearch.business.MigrationPhaseStoreBootstrapIT;
 import com.dotcms.content.elasticsearch.business.ContentletIndexAPIImplPhaseSwitchIntegrationTest;
+import com.dotcms.content.elasticsearch.business.ContentletIndexAPIImplMidReindexRollbackIT;
 import com.dotcms.content.elasticsearch.business.ContentletIndexAPIImplMigrationIntegrationTest;
 import com.dotcms.content.index.opensearch.ContentFactoryIndexOperationsOSIntegrationTest;
 import com.dotcms.content.index.opensearch.ContentletIndexOperationsOSIntegrationTest;
@@ -49,6 +50,7 @@
         OSClientConfigTest.class,
         ContentletIndexAPIImplMigrationIntegrationTest.class,
         ContentletIndexAPIImplPhaseSwitchIntegrationTest.class,
+        ContentletIndexAPIImplMidReindexRollbackIT.class,
         MigrationPhaseStoreBootstrapIT.class,
         OSSearchAPIImplIntegrationTest.class,
         OSSiteSearchAPIIntegrationTest.class,