diff --git a/docs/backend/SEARCH_API_MIGRATION.md b/docs/backend/SEARCH_API_MIGRATION.md new file mode 100644 index 000000000000..071d52d83f55 --- /dev/null +++ b/docs/backend/SEARCH_API_MIGRATION.md @@ -0,0 +1,215 @@ +# Search API Migration Guide + +This guide is intended for **dotCMS plugin and integration developers** who use the +`ContentletAPI`, `ESSeachAPI`, or the `$ESContent` Velocity tool in their extensions. + +The changes described here are part of the ongoing ES → OpenSearch migration. The +deprecated methods listed below **will be removed** when dotCMS completes the cutover +to OpenSearch. Migrate before that happens to avoid compilation failures in your plugins. + +--- + +## 1. `ContentletAPI` — deprecated search methods + +### What was deprecated + +| Method | Return type | Status | +|--------|-------------|--------| +| `esSearch(String esQuery, boolean live, User user, boolean respectFrontendRoles)` | `ESSearchResults` | `@Deprecated(forRemoval = true)` | +| `esSearchRaw(String esQuery, boolean live, User user, boolean respectFrontendRoles)` | `SearchResponse` | `@Deprecated(forRemoval = true)` | + +Both methods delegate to `ESSeachAPI` directly and return Elasticsearch-specific types +(`ESSearchResults`, `org.elasticsearch.action.search.SearchResponse`). They will be +removed at OS cutover. + +### Replacements + +| Old method | New method | New return type | +|------------|------------|-----------------| +| `esSearch(...)` | `search(String query, boolean live, User user, boolean respectFrontendRoles)` | `ContentSearchResults` | +| `esSearchRaw(...)` | `searchRaw(String query, boolean live, User user, boolean respectFrontendRoles)` | `ContentSearchResponse` | + +The new methods route through the phase-aware `SearchAPI` router (ES in phases 0–1, +OpenSearch in phases 2–3) and return vendor-neutral DTOs. + +### Migration example + +```java +// Before +ContentletAPI contentletAPI = APILocator.getContentletAPI(); + +ESSearchResults results = contentletAPI.esSearch(query, false, user, false); +for (Object obj : results) { + Contentlet c = (Contentlet) obj; + // ... +} + +SearchResponse raw = contentletAPI.esSearchRaw(query, false, user, false); +SearchHit[] hits = raw.getHits().getHits(); + +// After +ContentSearchResults results = contentletAPI.search(query, false, user, false); +for (Contentlet c : results) { + // no cast needed +} + +ContentSearchResponse raw = contentletAPI.searchRaw(query, false, user, false); +List hits = raw.hits().hits(); // neutral SearchHit DTO +``` + +--- + +## 2. `ContentletAPIPreHook` / `ContentletAPIPostHook` — deprecated hook methods + +If your OSGi plugin implements either hook interface to intercept search calls, the +following methods are deprecated and will be removed: + +### `ContentletAPIPreHook` + +| Deprecated | Replacement | +|-----------|-------------| +| `boolean esSearch(String esQuery, boolean live, User user, boolean respectFrontendRoles)` | `boolean search(String query, boolean live, User user, boolean respectFrontendRoles)` | +| `boolean esSearchRaw(String esQuery, boolean live, User user, boolean respectFrontendRoles)` | `boolean searchRaw(String query, boolean live, User user, boolean respectFrontendRoles)` | + +### `ContentletAPIPostHook` + +| Deprecated | Replacement | +|-----------|-------------| +| `void esSearch(String esQuery, boolean live, User user, boolean respectFrontendRoles)` | `void search(String query, boolean live, User user, boolean respectFrontendRoles)` | +| `void esSearchRaw(String esQuery, boolean live, User user, boolean respectFrontendRoles)` | `void searchRaw(String query, boolean live, User user, boolean respectFrontendRoles)` | + +Both replacement methods have default no-op implementations in the interface, so you +only need to override them if you need to intercept those calls. + +### Migration example + +```java +// Before +public class MyPreHook implements ContentletAPIPreHook { + @Override + public boolean esSearch(String esQuery, boolean live, User user, boolean respectFrontendRoles) { + Logger.info(this, "intercepted esSearch: " + esQuery); + return true; + } +} + +// After +public class MyPreHook implements ContentletAPIPreHook { + @Override + public boolean search(String query, boolean live, User user, boolean respectFrontendRoles) { + Logger.info(this, "intercepted search: " + query); + return true; + } +} +``` + +--- + +## 3. Velocity / VTL — `$ESContent` viewtool + +The `$ESContent` viewtool (`ESContentTool`) is available in Velocity templates. Two of +its methods changed return types in this release. + +### `$ESContent.search(query)` + +| | Before | After | +|--|--------|-------| +| Return type | `ESSearchResults` (extends raw `List`) | `ContentSearchResults` (implements `List`) | +| Element type | `ContentMap` | `ContentMap` (unchanged) | + +**Impact:** Templates that iterate over the result with `#foreach` are unaffected — +the elements are still `ContentMap` objects with the same properties. + +```velocity +## This continues to work unchanged +#foreach($content in $ESContent.search($query)) + $content.title +#end +``` + +Templates that access the result as `ESSearchResults` through a Java helper or cast +will fail at runtime. Replace with `ContentSearchResults`. + +### `$ESContent.raw(query)` + +| | Before | After | +|--|--------|-------| +| Return type | `org.elasticsearch.action.search.SearchResponse` | `ContentSearchResponse` | + +**Impact:** Templates that call `.toString()` on the raw response to obtain ES +wire-format JSON (e.g. to parse it manually) will receive a different string. The new +`ContentSearchResponse.toString()` is a Java object representation, not JSON. + +```velocity +## HIGH RISK — if your template does this, it will stop receiving valid JSON +#set($json = $ESContent.raw($query).toString()) + +## Use the structured accessors instead +#set($raw = $ESContent.raw($query)) +#set($hits = $raw.hits().hits()) +#foreach($hit in $hits) + $hit.id() +#end +``` + +Useful accessors on `ContentSearchResponse`: + +| Method | Description | +|--------|-------------| +| `hits()` | Returns `SearchHits` — iterable collection of `SearchHit` | +| `hits().hits()` | `List` | +| `hits().totalHits().value()` | Total number of matching documents | +| `scrollId()` | Scroll ID for paginated requests, or `null` | +| `tookMillis()` | Query execution time in milliseconds | +| `aggregations()` | `Map>` — terms aggregations | + +--- + +## 4. Return type change: `ESSearchResults` → `ContentSearchResults` + +`ESSearchResults` (package `com.dotcms.content.elasticsearch.business`) is not yet +removed but is no longer returned by the new API methods. If your plugin declares +variables of type `ESSearchResults`, update them to `ContentSearchResults`. + +```java +// Before +ESSearchResults results = (ESSearchResults) contentletAPI.esSearch(query, live, user, roles); + +// After +ContentSearchResults results = contentletAPI.search(query, live, user, roles); +``` + +The key structural difference: + +| | `ESSearchResults` | `ContentSearchResults` | +|--|-------------------|-----------------------------| +| Implements | `List` (raw) | `List` (typed) | +| Element access | Requires `(Contentlet)` cast | Type-safe, no cast needed | +| Response metadata | `getResponse()` → `SearchResponse` (ES) | `getResponse()` → `ContentSearchResponse` (neutral) | +| Total results | `getResponse().getHits().getTotalHits().value` | `getTotalResults()` | +| Scroll ID | `getResponse().getScrollId()` | `getScrollId()` | + +--- + +## 5. Summary of new neutral DTOs + +These classes replace the Elasticsearch-specific types in the public API: + +| Old type (ES-specific) | New type (neutral) | Package | +|------------------------|--------------------|---------| +| `org.elasticsearch.action.search.SearchResponse` | `ContentSearchResponse` | `com.dotcms.content.index.domain` | +| `com.dotcms.content.elasticsearch.business.ESSearchResults` | `ContentSearchResults` | `com.dotcms.content.index.domain` | +| `org.elasticsearch.search.SearchHits` | `SearchHits` | `com.dotcms.content.index.domain` | +| `org.elasticsearch.search.SearchHit` | `SearchHit` | `com.dotcms.content.index.domain` | +| `org.elasticsearch.search.TotalHits` | `TotalHits` | `com.dotcms.content.index.domain` | + +--- + +## 6. Timeline + +| Phase | Action | +|-------|--------| +| Now (this release) | `esSearch` / `esSearchRaw` marked `@Deprecated(forRemoval = true)`. New `search` / `searchRaw` methods available. | +| OpenSearch cutover | `esSearch`, `esSearchRaw`, and ES-specific return types removed. Plugins that have not migrated will **fail to compile**. | + +Migrate as soon as possible to get the full migration window. diff --git a/dotCMS/src/main/java/com/dotcms/browser/BrowserAPIImpl.java b/dotCMS/src/main/java/com/dotcms/browser/BrowserAPIImpl.java index cb2aadd8a0d3..99b2cea2b0ce 100644 --- a/dotCMS/src/main/java/com/dotcms/browser/BrowserAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/browser/BrowserAPIImpl.java @@ -7,7 +7,7 @@ import com.dotcms.content.business.json.ContentletJsonAPI; import com.dotcms.contenttype.model.type.BaseContentType; import com.dotcms.contenttype.model.type.ContentType; -import com.dotcms.enterprise.ESSeachAPI; +import com.dotcms.content.index.SearchAPI; import com.dotcms.uuid.shorty.ShortyIdAPI; import com.dotmarketing.beans.Host; import com.dotmarketing.business.APILocator; @@ -803,7 +803,7 @@ private int countApproximateClausesInQuery(String query) { */ private Set processSingleESQuery(final BrowserQuery browserQuery, final Set inodes, final long startTime) { final boolean live = !browserQuery.showWorking; - final ESSeachAPI esSearchAPI = APILocator.getEsSearchAPI(); + final SearchAPI searchAPI = APILocator.getSearchAPI(); final List collectedInodes = new ArrayList<>(); try { @@ -815,7 +815,7 @@ private Set processSingleESQuery(final BrowserQuery browserQuery, final Logger.debug(this, String.format("Single ES query: %d inodes", inodes.size())); - esSearchAPI.esSearch(esQuery, live, browserQuery.user, false).forEach(result -> { + searchAPI.search(esQuery, live, browserQuery.user, false).forEach(result -> { final Contentlet contentlet = (Contentlet) result; collectedInodes.add(contentlet.getInode()); }); diff --git a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentletAPIImpl.java b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentletAPIImpl.java index 14f321d47e39..8a8a44b64135 100644 --- a/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentletAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentletAPIImpl.java @@ -235,8 +235,8 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import com.dotcms.content.index.domain.ContentSearchResponse; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; /** @@ -348,12 +348,14 @@ private static UniqueFieldValidationStrategyResolver getUniqueFieldValidationStr return CDIUtils.getBeanThrows(UniqueFieldValidationStrategyResolver.class); } + @Deprecated(forRemoval = true) @Override public SearchResponse esSearchRaw(String esQuery, boolean live, User user, boolean respectFrontendRoles) throws DotSecurityException, DotDataException { return APILocator.getEsSearchAPI().esSearchRaw(esQuery, live, user, respectFrontendRoles); } + @Deprecated(forRemoval = true) @Override public ESSearchResults esSearch(String esQuery, boolean live, User user, boolean respectFrontendRoles) throws DotSecurityException, DotDataException { @@ -2381,29 +2383,29 @@ private List getRelatedChildren(final Contentlet contentlet, final R final String relationshipName = rel.getRelationTypeValue().toLowerCase(); final int limit = limitParam <= 0 ? MAX_LIMIT : limitParam; - SearchResponse response; + ContentSearchResponse response; final boolean DONT_PULL_PARENTS = Boolean.FALSE; //Search for related content in existing contentlet if (UtilMethods.isSet(contentlet.getInode())) { - response = APILocator.getEsSearchAPI() - .esSearchRelated(contentlet, relationshipName, DONT_PULL_PARENTS, + response = APILocator.getSearchAPI() + .searchRelated(contentlet, relationshipName, DONT_PULL_PARENTS, WORKING_VERSION, user, respectFrontendRoles, limit, offset, null); } else { //Search for related content in other versions of the same contentlet - response = APILocator.getEsSearchAPI() - .esSearchRelated(contentlet.getIdentifier(), relationshipName, + response = APILocator.getSearchAPI() + .searchRelated(contentlet.getIdentifier(), relationshipName, DONT_PULL_PARENTS, WORKING_VERSION, user, respectFrontendRoles, limit, offset, null); } - if (response.getHits() == null) { + if (response.hits().hits().isEmpty()) { return result; } - for (final SearchHit sh : response.getHits()) { - final Map sourceMap = sh.getSourceAsMap(); + for (final com.dotcms.content.index.domain.SearchHit sh : response.hits()) { + final Map sourceMap = sh.sourceAsMap(); if (sourceMap.get(relationshipName) != null) { List relatedIdentifiers = ((ArrayList) sourceMap.get( relationshipName)); @@ -2475,25 +2477,25 @@ private List getRelatedParents(final Contentlet contentlet, final Re final String relationshipName = rel.getRelationTypeValue().toLowerCase(); final int limit = limitParam <= 0 ? MAX_LIMIT : limitParam; - SearchResponse response; + ContentSearchResponse response; final boolean PULL_PARENTS = Boolean.TRUE; //Search for related content in existing contentlet if (UtilMethods.isSet(contentlet.getInode())) { - response = APILocator.getEsSearchAPI() - .esSearchRelated(contentlet, relationshipName, PULL_PARENTS, + response = APILocator.getSearchAPI() + .searchRelated(contentlet, relationshipName, PULL_PARENTS, WORKING_VERSION, user, respectFrontendRoles, limit, offset, null); } else { - response = APILocator.getEsSearchAPI() - .esSearchRelated(contentlet.getIdentifier(), relationshipName, PULL_PARENTS, + response = APILocator.getSearchAPI() + .searchRelated(contentlet.getIdentifier(), relationshipName, PULL_PARENTS, WORKING_VERSION, user, respectFrontendRoles, limit, offset, null); } - if (response.getHits() != null) { - for (final SearchHit sh : response.getHits()) { - final Map sourceMap = sh.getSourceAsMap(); + if (!response.hits().hits().isEmpty()) { + for (final com.dotcms.content.index.domain.SearchHit sh : response.hits()) { + final Map sourceMap = sh.sourceAsMap(); final String identifier = (String) sourceMap.get("identifier"); if (identifier != null && !relatedMap.containsKey(identifier)) { final Contentlet mappedContentlet = findContentletByIdentifierAnyLanguage( diff --git a/dotCMS/src/main/java/com/dotcms/content/index/SearchAPI.java b/dotCMS/src/main/java/com/dotcms/content/index/SearchAPI.java new file mode 100644 index 000000000000..a30b5c228eed --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/index/SearchAPI.java @@ -0,0 +1,153 @@ +package com.dotcms.content.index; + +import com.dotcms.content.index.domain.ContentSearchResponse; +import com.dotcms.content.index.domain.ContentSearchResults; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.liferay.portal.model.User; + +/** + * Vendor-neutral search API for executing full-text index queries. + * + *

This interface replaces the legacy {@code ESSeachAPI} whose method signatures + * leaked {@code org.elasticsearch.action.search.SearchResponse} and + * {@code ESSearchResults} into application code. Implementations delegate to the + * active search back-end (Elasticsearch or OpenSearch) as determined by the current + * migration phase.

+ * + *

All methods accept the same Lucene/JSON query format used by the existing search + * paths, so call sites only need to change the method names and return types.

+ * + * @see SearchAPIImpl Phase-aware router (register via {@code APILocator.getSearchAPI()}) + */ +public interface SearchAPI { + + /** + * Executes a JSON search query and returns the matching contentlets loaded from the DB. + * + *

Equivalent to {@code ESSeachAPI.esSearch()} but returns a vendor-neutral + * {@link ContentSearchResults} instead of {@code ESSearchResults}.

+ * + * @param query the JSON search query + * @param live {@code true} to query the live index + * @param user the user performing the action (may be {@code null} when + * {@code respectFrontendRoles} is {@code true}) + * @param respectFrontendRoles whether front-end roles should be applied + * @return populated result list; never {@code null} + */ + ContentSearchResults search( + String query, + boolean live, + User user, + boolean respectFrontendRoles) + throws DotSecurityException, DotDataException; + + /** + * Executes a raw JSON search query and returns the index response without loading + * contentlets from the database. + * + *

Equivalent to {@code ESSeachAPI.esSearchRaw()} but returns a vendor-neutral + * {@link ContentSearchResponse} instead of {@code SearchResponse}.

+ * + * @param query the JSON search query + * @param live {@code true} to query the live index + * @param user the user performing the action + * @param respectFrontendRoles whether front-end roles should be applied + * @return raw search response; never {@code null} + */ + ContentSearchResponse searchRaw( + String query, + boolean live, + User user, + boolean respectFrontendRoles) + throws DotSecurityException, DotDataException; + + /** + * Returns related content for a given contentlet identifier (no pagination). + * + * @param contentletIdentifier identifier of the content whose relations will be searched + * @param relationshipName name of the relationship field + * @param pullParents {@code true} to search for parents, {@code false} for children + * @param live {@code true} to query the live index + * @param user the user performing the action + * @param respectFrontendRoles whether front-end roles should be applied + */ + ContentSearchResponse searchRelated( + String contentletIdentifier, + String relationshipName, + boolean pullParents, + boolean live, + User user, + boolean respectFrontendRoles) + throws DotDataException, DotSecurityException; + + /** + * Returns related content for a given contentlet (no pagination). + * + * @param contentlet the content whose relations will be searched + * @param relationshipName name of the relationship field + * @param pullParents {@code true} to search for parents, {@code false} for children + * @param live {@code true} to query the live index + * @param user the user performing the action + * @param respectFrontendRoles whether front-end roles should be applied + */ + ContentSearchResponse searchRelated( + Contentlet contentlet, + String relationshipName, + boolean pullParents, + boolean live, + User user, + boolean respectFrontendRoles) + throws DotDataException, DotSecurityException; + + /** + * Returns paginated related content for a given contentlet identifier. + * + * @param contentletIdentifier identifier of the content whose relations will be searched + * @param relationshipName name of the relationship field + * @param pullParents {@code true} to search for parents, {@code false} for children + * @param live {@code true} to query the live index + * @param user the user performing the action + * @param respectFrontendRoles whether front-end roles should be applied + * @param limit maximum number of results ({@code -1} for no limit) + * @param offset result offset for pagination + * @param sortBy sort expression, or {@code null} for default sort + */ + ContentSearchResponse searchRelated( + String contentletIdentifier, + String relationshipName, + boolean pullParents, + boolean live, + User user, + boolean respectFrontendRoles, + int limit, + int offset, + String sortBy) + throws DotDataException, DotSecurityException; + + /** + * Returns paginated related content for a given contentlet. + * + * @param contentlet the content whose relations will be searched + * @param relationshipName name of the relationship field + * @param pullParents {@code true} to search for parents, {@code false} for children + * @param live {@code true} to query the live index + * @param user the user performing the action + * @param respectFrontendRoles whether front-end roles should be applied + * @param limit maximum number of results ({@code -1} for no limit) + * @param offset result offset for pagination + * @param sortBy sort expression, or {@code null} for default sort + */ + ContentSearchResponse searchRelated( + Contentlet contentlet, + String relationshipName, + boolean pullParents, + boolean live, + User user, + boolean respectFrontendRoles, + int limit, + int offset, + String sortBy) + throws DotDataException, DotSecurityException; +} diff --git a/dotCMS/src/main/java/com/dotcms/content/index/SearchAPIImpl.java b/dotCMS/src/main/java/com/dotcms/content/index/SearchAPIImpl.java new file mode 100644 index 000000000000..3b1bdc2a7194 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/index/SearchAPIImpl.java @@ -0,0 +1,186 @@ +package com.dotcms.content.index; + +import com.dotcms.cdi.CDIUtils; +import com.dotcms.content.index.domain.ContentSearchResponse; +import com.dotcms.content.index.domain.ContentSearchResults; +import com.dotcms.content.index.elasticsearch.ESSearchAPIImpl; +import com.dotcms.content.index.opensearch.OSSearchAPIImpl; +import com.dotcms.content.model.annotation.IndexLibraryIndependent; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.liferay.portal.model.User; + +/** + * Phase-aware router implementation of {@link SearchAPI}. + * + *

Routing table

+ * 
+ * Phase                     | Read provider
+ * --------------------------|---------------
+ * 0 — not started           | ES
+ * 1 — dual-write, ES reads  | ES
+ * 2 — dual-write, OS reads  | OS  (with ES fallback on failure)
+ * 3 — OS only               | OS
+ *
+ * + *

All search operations are reads; there are no write operations in this API, so the + * router only delegates to the read-path provider determined by {@link PhaseRouter#read}.

+ * + * @see PhaseRouter + * @see ESSearchAPIImpl + * @see OSSearchAPIImpl + */ +@IndexLibraryIndependent +public class SearchAPIImpl implements SearchAPI { + + private final ESSearchAPIImpl esImpl; + private final OSSearchAPIImpl osImpl; + private final PhaseRouter router; + + public SearchAPIImpl() { + this(CDIUtils.getBeanThrows(ESSearchAPIImpl.class), + CDIUtils.getBeanThrows(OSSearchAPIImpl.class)); + } + + /** Package-private constructor for testing. */ + SearchAPIImpl(final ESSearchAPIImpl esImpl, final OSSearchAPIImpl osImpl) { + this.esImpl = esImpl; + this.osImpl = osImpl; + this.router = new PhaseRouter<>(esImpl, osImpl); + } + + /** Direct access to the ES implementation (for testing / bootstrap checks). */ + public ESSearchAPIImpl esImpl() { + return esImpl; + } + + /** Direct access to the OS implementation (for testing / bootstrap checks). */ + public OSSearchAPIImpl osImpl() { + return osImpl; + } + + // ------------------------------------------------------------------------- + // SearchAPI — all operations are reads; delegate via router.readChecked + // ------------------------------------------------------------------------- + + @Override + public ContentSearchResults search( + final String query, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + try { + return router.readChecked(impl -> + impl.search(query, live, user, respectFrontendRoles)); + } catch (final DotSecurityException | DotDataException e) { + throw e; + } catch (final Exception e) { + throw new DotDataException(e.getMessage(), e); + } + } + + @Override + public ContentSearchResponse searchRaw( + final String query, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + try { + return router.readChecked(impl -> + impl.searchRaw(query, live, user, respectFrontendRoles)); + } catch (final DotSecurityException | DotDataException e) { + throw e; + } catch (final Exception e) { + throw new DotDataException(e.getMessage(), e); + } + } + + @Override + public ContentSearchResponse searchRelated( + final String contentletIdentifier, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotDataException, DotSecurityException { + try { + return router.readChecked(impl -> impl.searchRelated( + contentletIdentifier, relationshipName, pullParents, + live, user, respectFrontendRoles)); + } catch (final DotDataException | DotSecurityException e) { + throw e; + } catch (final Exception e) { + throw new DotDataException(e.getMessage(), e); + } + } + + @Override + public ContentSearchResponse searchRelated( + final Contentlet contentlet, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotDataException, DotSecurityException { + try { + return router.readChecked(impl -> impl.searchRelated( + contentlet, relationshipName, pullParents, + live, user, respectFrontendRoles)); + } catch (final DotDataException | DotSecurityException e) { + throw e; + } catch (final Exception e) { + throw new DotDataException(e.getMessage(), e); + } + } + + @Override + public ContentSearchResponse searchRelated( + final String contentletIdentifier, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles, + final int limit, + final int offset, + final String sortBy) + throws DotDataException, DotSecurityException { + try { + return router.readChecked(impl -> impl.searchRelated( + contentletIdentifier, relationshipName, pullParents, + live, user, respectFrontendRoles, limit, offset, sortBy)); + } catch (final DotDataException | DotSecurityException e) { + throw e; + } catch (final Exception e) { + throw new DotDataException(e.getMessage(), e); + } + } + + @Override + public ContentSearchResponse searchRelated( + final Contentlet contentlet, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles, + final int limit, + final int offset, + final String sortBy) + throws DotDataException, DotSecurityException { + try { + return router.readChecked(impl -> impl.searchRelated( + contentlet, relationshipName, pullParents, + live, user, respectFrontendRoles, limit, offset, sortBy)); + } catch (final DotDataException | DotSecurityException e) { + throw e; + } catch (final Exception e) { + throw new DotDataException(e.getMessage(), e); + } + } +} diff --git a/dotCMS/src/main/java/com/dotcms/content/index/domain/AggregationBucket.java b/dotCMS/src/main/java/com/dotcms/content/index/domain/AggregationBucket.java new file mode 100644 index 000000000000..cb7c21497fce --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/index/domain/AggregationBucket.java @@ -0,0 +1,70 @@ +package com.dotcms.content.index.domain; + +import java.util.List; +import java.util.stream.Collectors; +import org.immutables.value.Value; + +/** + * Vendor-neutral representation of a single bucket in a terms aggregation. + * + *

Replaces direct use of {@code org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket} + * and {@code org.opensearch.client.opensearch._types.aggregations.StringTermsBucket} in + * application code.

+ */ +@Value.Immutable +public interface AggregationBucket { + + /** Bucket key as a String (numeric keys are converted via {@code toString()}). */ + String key(); + + /** Number of documents in this bucket. */ + long docCount(); + + static ImmutableAggregationBucket.Builder builder() { + return ImmutableAggregationBucket.builder(); + } + + // ------------------------------------------------------------------------- + // ES factories + // ------------------------------------------------------------------------- + + /** Creates a bucket from an Elasticsearch terms bucket. */ + static AggregationBucket from( + final org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket esBucket) { + return builder() + .key(esBucket.getKeyAsString()) + .docCount(esBucket.getDocCount()) + .build(); + } + + // ------------------------------------------------------------------------- + // OS factories + // ------------------------------------------------------------------------- + + /** Creates a bucket from an OpenSearch string-terms bucket. */ + static AggregationBucket fromOS( + final org.opensearch.client.opensearch._types.aggregations.StringTermsBucket osBucket) { + return builder() + .key(osBucket.key()) + .docCount(osBucket.docCount()) + .build(); + } + + /** Creates a bucket from an OpenSearch long-terms bucket. */ + static AggregationBucket fromOS( + final org.opensearch.client.opensearch._types.aggregations.LongTermsBucket osBucket) { + return builder() + .key(String.valueOf(osBucket.key())) + .docCount(osBucket.docCount()) + .build(); + } + + /** Creates a bucket from an OpenSearch double-terms bucket. */ + static AggregationBucket fromOS( + final org.opensearch.client.opensearch._types.aggregations.DoubleTermsBucket osBucket) { + return builder() + .key(String.valueOf(osBucket.key())) + .docCount(osBucket.docCount()) + .build(); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/content/index/domain/ContentSearchResponse.java b/dotCMS/src/main/java/com/dotcms/content/index/domain/ContentSearchResponse.java new file mode 100644 index 000000000000..6d15f3d437e9 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/index/domain/ContentSearchResponse.java @@ -0,0 +1,119 @@ +package com.dotcms.content.index.domain; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.immutables.value.Value; + +/** + * Vendor-neutral representation of a raw search response. + * + *

Replaces direct use of {@code org.elasticsearch.action.search.SearchResponse} in + * application code. Carries the search hits, timing metadata, scroll ID, and + * the first-level terms aggregations (the only aggregation type used in dotCMS).

+ * + *

Factory methods ({@code from(ES)}, {@code from(OS)}) map vendor types to this + * neutral DTO; they are the only places where vendor imports are allowed in this file.

+ */ +@Value.Immutable +public interface ContentSearchResponse { + + /** Neutral search hits (already vendor-independent). */ + SearchHits hits(); + + /** + * Scroll ID returned by the cluster, or {@code null} when not a scroll request. + * ES: {@code SearchResponse.getScrollId()} / OS: {@code SearchResponse.scrollId()} + */ + @Nullable + String scrollId(); + + /** Time the cluster took to execute the query, in milliseconds. */ + long tookMillis(); + + /** + * First-level terms aggregations, keyed by aggregation name. + * Only {@code terms} aggregations are mapped; other types are silently skipped. + */ + @Value.Default + default Map> aggregations() { + return Collections.emptyMap(); + } + + static ImmutableContentSearchResponse.Builder builder() { + return ImmutableContentSearchResponse.builder(); + } + + // ------------------------------------------------------------------------- + // ES factory + // ------------------------------------------------------------------------- + + static ContentSearchResponse from( + final org.elasticsearch.action.search.SearchResponse esResponse) { + + final Map> aggs = new LinkedHashMap<>(); + if (esResponse.getAggregations() != null) { + for (final org.elasticsearch.search.aggregations.Aggregation agg + : esResponse.getAggregations().asList()) { + if (agg instanceof org.elasticsearch.search.aggregations.bucket.terms.Terms) { + final org.elasticsearch.search.aggregations.bucket.terms.Terms termAgg = + (org.elasticsearch.search.aggregations.bucket.terms.Terms) agg; + aggs.put(agg.getName(), termAgg.getBuckets().stream() + .map(AggregationBucket::from) + .collect(Collectors.toList())); + } + } + } + + return builder() + .hits(esResponse.getHits() != null + ? SearchHits.from(esResponse.getHits()) + : SearchHits.empty()) + .scrollId(esResponse.getScrollId()) + .tookMillis(esResponse.getTook() != null ? esResponse.getTook().getMillis() : 0L) + .aggregations(aggs) + .build(); + } + + // ------------------------------------------------------------------------- + // OS factory + // ------------------------------------------------------------------------- + + static ContentSearchResponse from( + final org.opensearch.client.opensearch.core.SearchResponse osResponse) { + + final Map> aggs = new LinkedHashMap<>(); + if (osResponse.aggregations() != null) { + for (final Map.Entry + entry : osResponse.aggregations().entrySet()) { + final org.opensearch.client.opensearch._types.aggregations.Aggregate agg = + entry.getValue(); + if (agg.isSterms()) { + aggs.put(entry.getKey(), agg.sterms().buckets().array().stream() + .map(AggregationBucket::fromOS) + .collect(Collectors.toList())); + } else if (agg.isLterms()) { + aggs.put(entry.getKey(), agg.lterms().buckets().array().stream() + .map(AggregationBucket::fromOS) + .collect(Collectors.toList())); + } else if (agg.isDterms()) { + aggs.put(entry.getKey(), agg.dterms().buckets().array().stream() + .map(AggregationBucket::fromOS) + .collect(Collectors.toList())); + } + } + } + + return builder() + .hits(osResponse.hits() != null + ? SearchHits.from(osResponse.hits()) + : SearchHits.empty()) + .scrollId(osResponse.scrollId()) + .tookMillis(osResponse.took()) + .aggregations(aggs) + .build(); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/content/index/domain/ContentSearchResults.java b/dotCMS/src/main/java/com/dotcms/content/index/domain/ContentSearchResults.java new file mode 100644 index 000000000000..c646ad331807 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/index/domain/ContentSearchResults.java @@ -0,0 +1,127 @@ +package com.dotcms.content.index.domain; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +/** + * Vendor-neutral replacement for {@code ESSearchResults}. + * + *

Holds a {@link ContentSearchResponse} (timing, scroll, aggregations, hits) plus the + * populated objects that were loaded after the index query. The class also implements + * {@link List} — delegating all {@code List} operations to an internal list — so that + * existing callers that iterate or call {@code size()} continue to work unchanged.

+ * + *

Mirrors the shape of the legacy {@code ESSearchResults} class but without any + * Elasticsearch-specific types in the public API.

+ * + * @param the type of elements in this list — typically {@code Contentlet} when produced + * by {@code SearchAPI}, or {@code ContentMap} when produced by {@code ESContentTool} + */ +public class ContentSearchResults implements List { + + private static final long serialVersionUID = 1L; + + private final ContentSearchResponse response; + private final List contentlets; + private String query; + private String rewrittenQuery; + private long populationTook; + + public ContentSearchResults(final ContentSearchResponse response, final List contentlets) { + this.response = response; + this.contentlets = new ArrayList<>(contentlets); + } + + // ------------------------------------------------------------------------- + // Domain accessors + // ------------------------------------------------------------------------- + + public ContentSearchResponse getResponse() { + return response; + } + + public SearchHits getHits() { + return response.hits(); + } + + public long getTotalResults() { + return response.hits().totalHits().value(); + } + + public String getScrollId() { + return response.scrollId(); + } + + public long getQueryTook() { + return response.tookMillis(); + } + + public Map> getAggregations() { + return response.aggregations(); + } + + public List getContentlets() { + return contentlets; + } + + public String getQuery() { + return query; + } + + public void setQuery(final String query) { + this.query = query; + } + + public String getRewrittenQuery() { + return rewrittenQuery; + } + + public void setRewrittenQuery(final String rewrittenQuery) { + this.rewrittenQuery = rewrittenQuery; + } + + public long getPopulationTook() { + return populationTook; + } + + public void setPopulationTook(final long populationTook) { + this.populationTook = populationTook; + } + + // ------------------------------------------------------------------------- + // List delegation + // ------------------------------------------------------------------------- + + @Override public int size() { return contentlets.size(); } + @Override public boolean isEmpty() { return contentlets.isEmpty(); } + @Override public boolean contains(final Object o) { return contentlets.contains(o); } + @Override public Iterator iterator() { return contentlets.iterator(); } + @Override public Object[] toArray() { return contentlets.toArray(); } + @Override public A[] toArray(final A[] a) { return contentlets.toArray(a); } + @Override public boolean add(final T o) { return contentlets.add(o); } + @Override public boolean remove(final Object o) { return contentlets.remove(o); } + @Override public boolean containsAll(final Collection c) { return contentlets.containsAll(c); } + @Override public boolean addAll(final Collection c) { return contentlets.addAll(c); } + @Override public boolean addAll(final int index, final Collection c) { return contentlets.addAll(index, c); } + @Override public boolean removeAll(final Collection c) { return contentlets.removeAll(c); } + @Override public boolean retainAll(final Collection c) { return contentlets.retainAll(c); } + @Override public void clear() { contentlets.clear(); } + @Override public T get(final int index) { return contentlets.get(index); } + @Override public T set(final int index, final T element) { return contentlets.set(index, element); } + @Override public void add(final int index, final T element) { contentlets.add(index, element); } + @Override public T remove(final int index) { return contentlets.remove(index); } + @Override public int indexOf(final Object o) { return contentlets.indexOf(o); } + @Override public int lastIndexOf(final Object o) { return contentlets.lastIndexOf(o); } + @Override public ListIterator listIterator() { return contentlets.listIterator(); } + @Override public ListIterator listIterator(final int index) { return contentlets.listIterator(index); } + @Override public List subList(final int fromIndex, final int toIndex) { return contentlets.subList(fromIndex, toIndex); } + + @Override + public String toString() { + return "ContentSearchResults [response=" + response + "]"; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/content/index/domain/SearchHits.java b/dotCMS/src/main/java/com/dotcms/content/index/domain/SearchHits.java index 7f3deee48e0c..87f06c5e6648 100644 --- a/dotCMS/src/main/java/com/dotcms/content/index/domain/SearchHits.java +++ b/dotCMS/src/main/java/com/dotcms/content/index/domain/SearchHits.java @@ -136,6 +136,10 @@ static SearchHits errorHit() { * @return a new SearchHits instance */ static SearchHits from(org.elasticsearch.search.SearchHits esSearchHits) { + if (esSearchHits == null) { + return empty(); + } + final List hits = Arrays.stream(esSearchHits.getHits()) .map(SearchHit::from) .collect(Collectors.toList()); diff --git a/dotCMS/src/main/java/com/dotcms/content/index/elasticsearch/ESSearchAPIImpl.java b/dotCMS/src/main/java/com/dotcms/content/index/elasticsearch/ESSearchAPIImpl.java new file mode 100644 index 000000000000..743ddbf8e37e --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/index/elasticsearch/ESSearchAPIImpl.java @@ -0,0 +1,160 @@ +package com.dotcms.content.index.elasticsearch; + +import com.dotcms.content.elasticsearch.business.ESSearchResults; +import com.dotcms.content.index.SearchAPI; +import com.dotcms.content.index.domain.ContentSearchResponse; +import com.dotcms.content.index.domain.ContentSearchResults; +import com.dotcms.enterprise.ESSeachAPI; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.liferay.portal.model.User; +import javax.enterprise.context.ApplicationScoped; + +/** + * Elasticsearch implementation of {@link SearchAPI}. + * + *

Adapter over the legacy {@link com.dotcms.enterprise.priv.ESSearchAPIImpl}: all search + * operations are delegated to that implementation and the ES-specific result types + * ({@code SearchResponse}, {@code ESSearchResults}) are converted to vendor-neutral DTOs + * via {@link ContentSearchResponse#from(org.elasticsearch.action.search.SearchResponse)}.

+ * + *

Transitional — will be deleted when the ES→OS migration completes (Phase 3 cutover).

+ * + * @see com.dotcms.content.index.opensearch.OSSearchAPIImpl symmetric OS counterpart + */ +@ApplicationScoped +public class ESSearchAPIImpl implements SearchAPI { + + private final ESSeachAPI delegate; + + /** CDI constructor. */ + public ESSearchAPIImpl() { + this(new com.dotcms.enterprise.priv.ESSearchAPIImpl()); + } + + /** Package-private for testing. */ + ESSearchAPIImpl(final ESSeachAPI delegate) { + this.delegate = delegate; + } + + // ------------------------------------------------------------------------- + // SearchAPI — delegate and adapt + // ------------------------------------------------------------------------- + + @Override + public ContentSearchResults search( + final String query, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + + final ESSearchResults esResults = delegate.esSearch(query, live, user, respectFrontendRoles); + final ContentSearchResponse response = ContentSearchResponse.from(esResults.getResponse()); + final ContentSearchResults results = + new ContentSearchResults<>(response, esResults.getContentlets()); + results.setQuery(esResults.getQuery()); + results.setRewrittenQuery(esResults.getRewrittenQuery()); + results.setPopulationTook(esResults.getPopulationTook()); + return results; + } + + @Override + public ContentSearchResponse searchRaw( + final String query, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + + final org.elasticsearch.action.search.SearchResponse esResponse = + delegate.esSearchRaw(query, live, user, respectFrontendRoles); + if (esResponse == null) { + throw new DotDataException("ES search returned null — unable to load index information"); + } + return ContentSearchResponse.from(esResponse); + } + + @Override + public ContentSearchResponse searchRelated( + final String contentletIdentifier, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotDataException, DotSecurityException { + + final org.elasticsearch.action.search.SearchResponse esResponse = + delegate.esSearchRelated(contentletIdentifier, relationshipName, + pullParents, live, user, respectFrontendRoles); + if (esResponse == null) { + throw new DotDataException("ES search returned null — unable to load index information"); + } + return ContentSearchResponse.from(esResponse); + } + + @Override + public ContentSearchResponse searchRelated( + final Contentlet contentlet, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotDataException, DotSecurityException { + + final org.elasticsearch.action.search.SearchResponse esResponse = + delegate.esSearchRelated(contentlet, relationshipName, + pullParents, live, user, respectFrontendRoles); + if (esResponse == null) { + throw new DotDataException("ES search returned null — unable to load index information"); + } + return ContentSearchResponse.from(esResponse); + } + + @Override + public ContentSearchResponse searchRelated( + final String contentletIdentifier, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles, + final int limit, + final int offset, + final String sortBy) + throws DotDataException, DotSecurityException { + + final org.elasticsearch.action.search.SearchResponse esResponse = + delegate.esSearchRelated(contentletIdentifier, relationshipName, + pullParents, live, user, respectFrontendRoles, limit, offset, sortBy); + if (esResponse == null) { + throw new DotDataException("ES search returned null — unable to load index information"); + } + return ContentSearchResponse.from(esResponse); + } + + @Override + public ContentSearchResponse searchRelated( + final Contentlet contentlet, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles, + final int limit, + final int offset, + final String sortBy) + throws DotDataException, DotSecurityException { + + final org.elasticsearch.action.search.SearchResponse esResponse = + delegate.esSearchRelated(contentlet, relationshipName, + pullParents, live, user, respectFrontendRoles, limit, offset, sortBy); + if (esResponse == null) { + throw new DotDataException("ES search returned null — unable to load index information"); + } + return ContentSearchResponse.from(esResponse); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/content/index/opensearch/OSSearchAPIImpl.java b/dotCMS/src/main/java/com/dotcms/content/index/opensearch/OSSearchAPIImpl.java new file mode 100644 index 000000000000..0db6f44f4fe2 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/content/index/opensearch/OSSearchAPIImpl.java @@ -0,0 +1,393 @@ +package com.dotcms.content.index.opensearch; + +import static com.dotcms.content.index.opensearch.ContentFactoryIndexOperationsOS.addBuilderSort; + +import com.dotcms.cdi.CDIUtils; +import com.dotcms.content.index.SearchAPI; +import com.dotcms.content.index.VersionedIndices; +import com.dotcms.content.index.domain.ContentSearchResponse; +import com.dotcms.content.index.domain.ContentSearchResults; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.business.DotStateException; +import com.dotmarketing.business.Role; +import com.dotmarketing.common.model.ContentletSearch; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.StringUtils; +import com.dotmarketing.util.UtilMethods; +import com.dotmarketing.util.json.JSONArray; +import com.dotmarketing.util.json.JSONException; +import com.dotmarketing.util.json.JSONObject; +import com.liferay.portal.model.User; +import io.vavr.control.Try; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; + +/** + * OpenSearch implementation of {@link SearchAPI}. + * + *

Executes the same JSON query format used by the Elasticsearch path by deserialising the + * JSON body with {@link SearchRequest#_DESERIALIZER} and setting the resolved index from + * {@link com.dotcms.content.index.VersionedIndicesAPI}.

+ * + *

Permissions are injected using the same Lucene-based filter logic as the ES path; + * since the filter is expressed as a JSON query object, it is vendor-neutral.

+ */ +@ApplicationScoped +public class OSSearchAPIImpl implements SearchAPI { + + private final OSClientProvider clientProvider; + + /** CDI constructor. */ + @Inject + public OSSearchAPIImpl() { + this(CDIUtils.getBeanThrows(OSClientProvider.class)); + } + + /** Package-private for testing. */ + OSSearchAPIImpl(final OSClientProvider clientProvider) { + this.clientProvider = clientProvider; + } + + // ------------------------------------------------------------------------- + // SearchAPI implementation + // ------------------------------------------------------------------------- + + @Override + public ContentSearchResults search( + final String query, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + + final String normalized = query != null + ? StringUtils.lowercaseStringExceptMatchingTokens( + query, com.dotcms.content.elasticsearch.business.ESContentFactoryImpl.LUCENE_RESERVED_KEYWORDS_REGEX) + : query; + + final ContentSearchResponse resp = searchRaw(normalized, live, user, respectFrontendRoles); + final ContentSearchResults results = new ContentSearchResults<>(resp, new ArrayList<>()); + results.setQuery(normalized); + results.setRewrittenQuery(normalized); + + if (resp.hits() == null) { + return results; + } + + final long start = System.currentTimeMillis(); + final List list = new ArrayList<>(); + + for (final com.dotcms.content.index.domain.SearchHit sh : resp.hits()) { + try { + final Map sourceMap = sh.sourceAsMap(); + final ContentletSearch conwrapper = new ContentletSearch(); + conwrapper.setInode(sourceMap.get("inode").toString()); + list.add(conwrapper); + } catch (final Exception e) { + Logger.error(this, e.getMessage(), e); + } + } + + final List inodes = new ArrayList<>(); + for (final ContentletSearch conwrap : list) { + inodes.add(conwrap.getInode()); + } + + final List contentlets = + APILocator.getContentletAPIImpl().findContentlets(inodes); + for (final Contentlet contentlet : contentlets) { + if (contentlet.getInode() != null) { + results.add(contentlet); + } + } + + results.setPopulationTook(System.currentTimeMillis() - start); + return results; + } + + @Override + public ContentSearchResponse searchRaw( + final String query, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + + if (!UtilMethods.isSet(query)) { + throw new DotStateException("Search query is null"); + } + + final JSONObject completeQueryJSON; + try { + completeQueryJSON = new JSONObject(query); + completeQueryJSON.put("_source", new JSONArray("[identifier, inode]")); + } catch (final JSONException e) { + throw new DotStateException("Unable to parse the given query.", e); + } + + return executeSearch(completeQueryJSON, live, user, respectFrontendRoles, -1, -1, null); + } + + @Override + public ContentSearchResponse searchRelated( + final String contentletIdentifier, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotDataException, DotSecurityException { + + final Contentlet contentlet = + APILocator.getContentletAPI().findContentletByIdentifierAnyLanguage(contentletIdentifier); + return searchRelated(contentlet, relationshipName, pullParents, live, user, + respectFrontendRoles, -1, -1, null); + } + + @Override + public ContentSearchResponse searchRelated( + final Contentlet contentlet, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles) + throws DotDataException, DotSecurityException { + + return searchRelated(contentlet, relationshipName, pullParents, live, user, + respectFrontendRoles, -1, -1, null); + } + + @Override + public ContentSearchResponse searchRelated( + final String contentletIdentifier, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles, + final int limit, + final int offset, + final String sortBy) + throws DotDataException, DotSecurityException { + + final Contentlet contentlet = + APILocator.getContentletAPI().findContentletByIdentifierAnyLanguage(contentletIdentifier); + return searchRelated(contentlet, relationshipName, pullParents, live, user, + respectFrontendRoles, limit, offset, sortBy); + } + + @Override + public ContentSearchResponse searchRelated( + final Contentlet contentlet, + final String relationshipName, + final boolean pullParents, + final boolean live, + final User user, + final boolean respectFrontendRoles, + final int limit, + final int offset, + final String sortBy) + throws DotDataException, DotSecurityException { + + final JSONObject completeQueryJSON = + buildRelatedQuery(contentlet, relationshipName, pullParents); + return executeSearch(completeQueryJSON, false, user, respectFrontendRoles, + limit, offset, sortBy); + } + + // ------------------------------------------------------------------------- + // Private helpers + // ------------------------------------------------------------------------- + + private JSONObject buildRelatedQuery( + final Contentlet contentlet, + final String relationshipName, + final boolean pullParents) { + final JSONObject criteriaMap = new JSONObject(); + try { + if (pullParents) { + criteriaMap.put("_source", "identifier"); + criteriaMap.put("query", new JSONObject().put("match", + Map.of(relationshipName.toLowerCase(), contentlet.getIdentifier()))); + } else { + criteriaMap.put("_source", relationshipName.toLowerCase()); + criteriaMap.put("query", + new JSONObject().put("match", Map.of("inode", contentlet.getInode()))); + } + } catch (final JSONException e) { + throw new DotStateException("Unable to build related query.", e); + } + return new JSONObject(criteriaMap.toString()); + } + + /** + * Executes a search against the active OpenSearch index, applying permissions and sorting. + * + *

Uses {@link SearchRequest#_DESERIALIZER} to parse the full JSON body (query, aggs, + * _source, from, size, sort, etc.) and then overlays the index resolved from + * {@link com.dotcms.content.index.VersionedIndicesAPI}.

+ */ + private ContentSearchResponse executeSearch( + final JSONObject queryJson, + final boolean live, + final User user, + final boolean respectFrontendRoles, + final int limit, + final int offset, + final String sortBy) + throws DotSecurityException, DotDataException { + + final String indexToHit = resolveIndex(live); + + if (user == null && !respectFrontendRoles) { + throw new DotSecurityException( + "You must specify a user if you are not respecting frontend roles"); + } + + List roles = new ArrayList<>(); + boolean isAdmin = false; + if (user != null) { + if (!APILocator.getRoleAPI().doesUserHaveRole(user, + APILocator.getRoleAPI().loadCMSAdminRole())) { + roles = APILocator.getRoleAPI().loadRolesForUser(user.getUserId()); + } else { + isAdmin = true; + } + } + + final StringBuffer perms = new StringBuffer(); + if (!isAdmin && !queryJson.has("permissions:")) { + APILocator.getContentletAPIImpl() + .addPermissionsToQuery(perms, user, roles, respectFrontendRoles); + } + + if (perms.length() > 0) { + try { + final JSONObject permissionsFilter = new JSONObject().put("query_string", + new JSONObject().put("query", perms.toString().trim())); + JSONArray boolFilters = new JSONArray("[" + permissionsFilter + "]"); + + if (queryJson.has("query")) { + final JSONObject currentQueryJSON = + new JSONObject(queryJson.getJSONObject("query").toString()); + boolFilters = + new JSONArray("[" + permissionsFilter + "," + currentQueryJSON + "]"); + } + + final JSONObject filteredJSON = new JSONObject().put("bool", + new JSONObject().put("must", new JSONObject().put("bool", + new JSONObject().put("must", boolFilters)))); + queryJson.put("query", filteredJSON); + } catch (final JSONException e) { + throw new DotStateException("Unable to apply permissions to OS query.", e); + } + } + + // Override pagination from parameters + try { + if (limit > 0) { + queryJson.put("size", limit); + } + if (offset > 0) { + queryJson.put("from", offset); + } + } catch (final JSONException e) { + throw new DotStateException("Unable to set pagination params.", e); + } + + final OpenSearchClient client = clientProvider.getClient(); + final JsonpMapper mapper = client._transport().jsonpMapper(); + + try { + // Parse body fields from JSON using the SearchRequest deserializer + final SearchRequest bodyTemplate; + try (final InputStream is = new ByteArrayInputStream( + queryJson.toString().getBytes(StandardCharsets.UTF_8)); + final jakarta.json.stream.JsonParser parser = mapper.jsonProvider() + .createParser(is)) { + bodyTemplate = SearchRequest._DESERIALIZER.deserialize(parser, mapper); + } + + // Build the final request with the resolved index added + final SearchRequest.Builder requestBuilder = new SearchRequest.Builder() + .index(indexToHit); + + if (bodyTemplate.query() != null) { + requestBuilder.query(bodyTemplate.query()); + } + if (bodyTemplate.aggregations() != null && !bodyTemplate.aggregations().isEmpty()) { + requestBuilder.aggregations(bodyTemplate.aggregations()); + } + if (bodyTemplate.source() != null) { + requestBuilder.source(bodyTemplate.source()); + } + if (bodyTemplate.from() != null) { + requestBuilder.from(bodyTemplate.from()); + } + if (bodyTemplate.size() != null) { + requestBuilder.size(bodyTemplate.size()); + } + if (bodyTemplate.sort() != null && !bodyTemplate.sort().isEmpty()) { + requestBuilder.sort(bodyTemplate.sort()); + } + if (bodyTemplate.highlight() != null) { + requestBuilder.highlight(bodyTemplate.highlight()); + } + if (bodyTemplate.postFilter() != null) { + requestBuilder.postFilter(bodyTemplate.postFilter()); + } + if (bodyTemplate.trackTotalHits() != null) { + requestBuilder.trackTotalHits(bodyTemplate.trackTotalHits()); + } + + // sortBy parameter overrides / extends body sort + if (UtilMethods.isSet(sortBy)) { + addBuilderSort(sortBy, requestBuilder); + } + + final SearchResponse response = + client.search(requestBuilder.build(), Object.class); + return ContentSearchResponse.from(response); + + } catch (final IOException e) { + throw new DotStateException("OS search execution failed: " + e.getMessage(), e); + } + } + + private String resolveIndex(final boolean live) { + final Optional optional = Try + .of(() -> APILocator.getVersionedIndicesAPI().loadDefaultVersionedIndices()) + .getOrElse(Optional.empty()); + + if (optional.isEmpty()) { + throw new com.dotmarketing.exception.DotRuntimeException( + "Unable to load versioned indices for OS search"); + } + + final VersionedIndices indices = optional.get(); + if (live) { + return indices.live().orElseThrow( + () -> new com.dotmarketing.exception.DotRuntimeException( + "No live index found for OS search")); + } + return indices.working().orElseThrow( + () -> new com.dotmarketing.exception.DotRuntimeException( + "No working index found for OS search")); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/contenttype/business/ContentTypeAPIImpl.java b/dotCMS/src/main/java/com/dotcms/contenttype/business/ContentTypeAPIImpl.java index b79a2509855f..05fb8b9d2f50 100644 --- a/dotCMS/src/main/java/com/dotcms/contenttype/business/ContentTypeAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/contenttype/business/ContentTypeAPIImpl.java @@ -66,7 +66,8 @@ import io.vavr.control.Try; import java.util.HashSet; import java.util.Set; -import org.elasticsearch.action.search.SearchResponse; +import com.dotcms.content.index.domain.AggregationBucket; +import com.dotcms.content.index.domain.ContentSearchResponse; import java.util.ArrayList; import java.util.Date; @@ -736,21 +737,15 @@ public List recentlyUsed(BaseContentType type, int numberToShow) th query = query.replace("{2}", String.valueOf(limit)); try { - SearchResponse raw = APILocator.getEsSearchAPI().esSearchRaw(query.toLowerCase(), false, user, false); + final ContentSearchResponse raw = + APILocator.getSearchAPI().searchRaw(query.toLowerCase(), false, user, false); - - JSONObject jo = new JSONObject(raw.toString()).getJSONObject("aggregations").getJSONObject("recent-contents"); - JSONArray ja = jo.getJSONArray("buckets"); - List ret = new ArrayList<>(); - for (int i = 0; i < ja.size(); i++) { - JSONObject joe = ja.getJSONObject(i); - String var = joe.getString("key"); - - ret.add(find(var)); + final List ret = new ArrayList<>(); + for (final AggregationBucket bucket : + raw.aggregations().getOrDefault("recent-contents", List.of())) { + ret.add(find(bucket.key())); } - - return ImmutableList.copyOf(ret); } catch (Exception e) { throw new DotStateException(e); @@ -781,21 +776,14 @@ public Map getEntriesByContentTypes(final String siteId) throws Do String query = queryBuilder.toString(); try { - SearchResponse raw = APILocator.getEsSearchAPI().esSearchRaw(query.toLowerCase(), false, user, false); + final ContentSearchResponse raw = + APILocator.getSearchAPI().searchRaw(query.toLowerCase(), false, user, false); - JSONObject jo = new JSONObject(raw.toString()).getJSONObject("aggregations").getJSONObject("sterms#entries"); - JSONArray ja = jo.getJSONArray("buckets"); - - Map result = new HashMap<>(); - - for (int i = 0; i < ja.size(); i++) { - JSONObject jsonObject = ja.getJSONObject(i); - String contentTypeName = jsonObject.getString("key"); - long count = jsonObject.getLong("doc_count"); - - result.put(contentTypeName, count); + final Map result = new HashMap<>(); + for (final AggregationBucket bucket : + raw.aggregations().getOrDefault("entries", java.util.List.of())) { + result.put(bucket.key(), bucket.docCount()); } - return result; } catch (Exception e) { throw new DotStateException(e); diff --git a/dotCMS/src/main/java/com/dotcms/rendering/velocity/viewtools/ESContentTool.java b/dotCMS/src/main/java/com/dotcms/rendering/velocity/viewtools/ESContentTool.java index e0455e168d2c..5c6bcd969228 100644 --- a/dotCMS/src/main/java/com/dotcms/rendering/velocity/viewtools/ESContentTool.java +++ b/dotCMS/src/main/java/com/dotcms/rendering/velocity/viewtools/ESContentTool.java @@ -1,47 +1,43 @@ package com.dotcms.rendering.velocity.viewtools; import com.dotcms.content.elasticsearch.business.ESSearchResults; +import com.dotcms.content.index.SearchAPI; +import com.dotcms.content.index.domain.ContentSearchResponse; +import com.dotcms.content.index.domain.ContentSearchResults; import com.dotmarketing.beans.Host; import com.dotmarketing.business.APILocator; -import com.dotmarketing.business.web.UserWebAPI; import com.dotmarketing.business.web.WebAPILocator; import com.dotmarketing.exception.DotDataException; import com.dotmarketing.exception.DotSecurityException; -import com.dotmarketing.portlets.contentlet.business.ContentletAPI; import com.dotmarketing.portlets.contentlet.model.Contentlet; import com.dotmarketing.util.Logger; import com.dotmarketing.util.PageMode; import com.dotcms.rendering.velocity.viewtools.content.ContentMap; +import org.elasticsearch.action.search.SearchResponse; + import java.util.ArrayList; import java.util.List; import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpSession; import org.apache.velocity.context.Context; import org.apache.velocity.tools.view.context.ViewContext; import org.apache.velocity.tools.view.tools.ViewTool; -import org.elasticsearch.action.search.SearchResponse; import com.liferay.portal.model.User; public class ESContentTool implements ViewTool { - private UserWebAPI userAPI; private HttpServletRequest req; private User user = null; private Context context; - ContentletAPI esapi = APILocator.getContentletAPI(); private Host currentHost; private PageMode mode; + @Override public void init(Object initData) { - userAPI = WebAPILocator.getUserWebAPI(); - - - this.context = ((ViewContext) initData).getVelocityContext(); this.req = ((ViewContext) initData).getRequest(); @@ -53,32 +49,45 @@ public void init(Object initData) { }catch(Exception e){ Logger.error(this, "Error finding current host", e); } - } - - - - public ESSearchResults search(String esQuery) throws DotSecurityException, DotDataException{ - - ESSearchResults cons = esapi.esSearch(esQuery, mode.showLive, user, true); - List maps = new ArrayList<>(); - - - for(Object x : cons){ - Contentlet con = (Contentlet)x; - - maps.add(new ContentMap(con, user, !mode.showLive,currentHost,context)); + + public ContentSearchResults search(final String esQuery) throws DotSecurityException, DotDataException { + final SearchAPI searchAPI = APILocator.getSearchAPI(); + final ContentSearchResults cons = searchAPI.search(esQuery, mode.showLive, user, true); + final List maps = new ArrayList<>(); + + for (final Contentlet con : cons) { + maps.add(new ContentMap(con, user, !mode.showLive, currentHost, context)); } - - return new ESSearchResults(cons.getResponse(), maps); + + return new ContentSearchResults<>(cons.getResponse(), maps); + } + + public ContentSearchResponse raw(final String esQuery) throws DotSecurityException, DotDataException { + return APILocator.getSearchAPI().searchRaw(esQuery, mode.showLive, user, true); } - - - public SearchResponse raw(String esQuery) throws DotSecurityException, DotDataException{ - - return esapi.esSearchRaw(esQuery, mode.showLive, user, true); - + + /** + * @deprecated Use {@link #search(String)} for vendor-neutral access. + * This method returns Elasticsearch-specific types and will be removed in v26.08.04. + * Velocity templates using {@code $results.hits}, {@code $results.aggregations}, + * or {@code $results.response} must migrate to the neutral equivalents exposed by + * {@link ContentSearchResults}. + */ + @Deprecated(forRemoval = true) + @SuppressWarnings("deprecation") + public ESSearchResults esSearch(final String esQuery) throws DotSecurityException, DotDataException { + return APILocator.getContentletAPI().esSearch(esQuery, mode.showLive, user, true); + } + + /** + * @deprecated Use {@link #raw(String)} for vendor-neutral access. + * This method returns an Elasticsearch-specific type and will be removed in v26.08.04. + */ + @Deprecated(forRemoval = true) + @SuppressWarnings("deprecation") + public SearchResponse esRaw(final String esQuery) throws DotSecurityException, DotDataException { + return APILocator.getContentletAPI().esSearchRaw(esQuery, mode.showLive, user, true); } - -} \ No newline at end of file +} diff --git a/dotCMS/src/main/java/com/dotcms/rest/api/v1/page/PageResource.java b/dotCMS/src/main/java/com/dotcms/rest/api/v1/page/PageResource.java index 6297731ec3cd..f5c4bc58ff3d 100644 --- a/dotCMS/src/main/java/com/dotcms/rest/api/v1/page/PageResource.java +++ b/dotCMS/src/main/java/com/dotcms/rest/api/v1/page/PageResource.java @@ -39,7 +39,7 @@ import org.glassfish.jersey.server.JSONP; import com.dotcms.api.web.HttpServletRequestThreadLocal; -import com.dotcms.content.elasticsearch.business.ESSearchResults; +import com.dotcms.content.index.domain.ContentSearchResults; import com.dotcms.contenttype.business.ContentTypeAPI; import com.dotcms.contenttype.model.type.BaseContentType; import com.dotcms.contenttype.model.type.ContentType; @@ -1057,7 +1057,7 @@ public Response searchPage( final String esQuery = getPageByPathESQuery(path); - final ESSearchResults esresult = esapi.esSearch(esQuery, live, user, live); + final ContentSearchResults esresult = esapi.search(esQuery, live, user, live); final Set> contentletMaps = applyFilters(onlyLiveSites, esresult) .stream() .map(contentlet -> { @@ -1261,7 +1261,7 @@ private String getPageByPathESQuery(final String pathParam) { private Collection applyFilters( final boolean workingSite, - final ESSearchResults esresult) throws DotDataException { + final ContentSearchResults esresult) throws DotDataException { final Collection contentlets = this.removeMultiLangVersion(esresult); return workingSite ? filterByWorkingSite(contentlets) : contentlets; diff --git a/dotCMS/src/main/java/com/dotcms/rest/elasticsearch/ESContentResourcePortlet.java b/dotCMS/src/main/java/com/dotcms/rest/elasticsearch/ESContentResourcePortlet.java index fe534fe17607..f6f7b84551b7 100644 --- a/dotCMS/src/main/java/com/dotcms/rest/elasticsearch/ESContentResourcePortlet.java +++ b/dotCMS/src/main/java/com/dotcms/rest/elasticsearch/ESContentResourcePortlet.java @@ -259,6 +259,8 @@ public Response searchRaw(@Context HttpServletRequest request) { try { String esQuery = IOUtils.toString(request.getInputStream()); + // FIXME(OS-cutover): esSearchRaw returns ES JSON wire format via SearchResponse.toString(). + // Migrate to searchRaw() + Jackson serialization when Phase 3 OS cutover makes ES unavailable. return responseResource.response(esapi.esSearchRaw(esQuery, mode.showLive, user, mode.showLive).toString()); } catch (Exception e) { diff --git a/dotCMS/src/main/java/com/dotcms/workflow/helper/WorkflowHelper.java b/dotCMS/src/main/java/com/dotcms/workflow/helper/WorkflowHelper.java index 43304c54ad5b..918d685222f9 100644 --- a/dotCMS/src/main/java/com/dotcms/workflow/helper/WorkflowHelper.java +++ b/dotCMS/src/main/java/com/dotcms/workflow/helper/WorkflowHelper.java @@ -84,10 +84,8 @@ import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.lang.time.StopWatch; import org.apache.velocity.context.Context; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; +import com.dotcms.content.index.domain.AggregationBucket; +import com.dotcms.content.index.domain.ContentSearchResponse; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -189,13 +187,12 @@ private BulkActionView findBulkActionByQuery(final String luceneQuery, final String query = String.format(ES_WFSTEP_AGGREGATES_QUERY, queryWithDatesFormatted); //We should only be considering Working content. - final SearchResponse response = LicenseManager.getInstance().isCommunity()? - this.contentletAPI.esSearch(query, - false, user, false).getResponse(): - this.contentletAPI - .esSearchRaw(StringUtils.lowercaseStringExceptMatchingTokens(query, + final ContentSearchResponse response = LicenseManager.getInstance().isCommunity()? + this.contentletAPI.search(query, false, user, false).getResponse(): + this.contentletAPI.searchRaw( + StringUtils.lowercaseStringExceptMatchingTokens(query, ESContentFactoryImpl.LUCENE_RESERVED_KEYWORDS_REGEX), - false, user, false); + false, user, false); //Query must be sent lowercase. It's a must. Logger.debug(getClass(), () -> "luceneQuery: " + sanitizedQuery); @@ -229,21 +226,16 @@ private BulkActionView findBulkActionByContentlets(final List contentlet * @throws DotSecurityException */ @CloseDBIfOpened - private BulkActionView buildBulkActionView (final SearchResponse response, + private BulkActionView buildBulkActionView (final ContentSearchResponse response, final User user) throws DotDataException, DotSecurityException { final Set archivedSchemes = workflowAPI.findArchivedSchemes().stream().map(WorkflowScheme::getId).collect(Collectors.toSet()); - final Aggregations aggregations = response.getAggregations(); final Map stepCounts = new HashMap<>(); - for (final Aggregation aggregation : aggregations.asList()) { - - if (aggregation instanceof ParsedStringTerms) { - ((ParsedStringTerms) aggregation) - .getBuckets().forEach( - bucket -> stepCounts.put(bucket.getKeyAsString(), bucket.getDocCount()) - ); + for (final Map.Entry> entry : response.aggregations().entrySet()) { + for (final AggregationBucket bucket : entry.getValue()) { + stepCounts.put(bucket.key(), bucket.docCount()); } } diff --git a/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java b/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java index 2517e8ed9530..f107e7a4f3f8 100644 --- a/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java +++ b/dotCMS/src/main/java/com/dotmarketing/business/APILocator.java @@ -906,11 +906,26 @@ public static ServerActionAPI getServerActionAPI() { * Creates a single instance of the {@link ESSeachAPI} class. * * @return The {@link ESSeachAPI} class. + * @deprecated Use {@link #getSearchAPI()} for vendor-neutral search access. */ + @Deprecated public static ESSeachAPI getEsSearchAPI () { return (ESSeachAPI) getInstance( APIIndex.ES_SEARCH_API ); } + /** + * Returns the vendor-neutral {@link com.dotcms.content.index.SearchAPI} router. + * + *

Routes search operations to the active provider (Elasticsearch or OpenSearch) + * based on the current migration phase. Prefer this over the deprecated + * {@link #getEsSearchAPI()} for all new call sites.

+ * + * @return the {@link com.dotcms.content.index.SearchAPI} instance. + */ + public static com.dotcms.content.index.SearchAPI getSearchAPI() { + return (com.dotcms.content.index.SearchAPI) getInstance(APIIndex.SEARCH_API); + } + /** * Creates a single instance of the {@link RulesAPI} class. * @@ -1435,7 +1450,8 @@ enum APIIndex ANALYTICS_CUSTOM_ATTRIBUTE_API, VERSIONED_INDICES_API, OPENSEARCH_INDEX_API, - CONTENT_MAPPING_API + CONTENT_MAPPING_API, + SEARCH_API ; Object create() { @@ -1539,6 +1555,7 @@ Object create() { case VERSIONED_INDICES_API: return CDIUtils.getBeanThrows(VersionedIndicesAPI.class); case OPENSEARCH_INDEX_API: return new OSIndexAPIImpl(); case CONTENT_MAPPING_API: return new ESMappingAPIImpl(); + case SEARCH_API: return new com.dotcms.content.index.SearchAPIImpl(); } throw new AssertionError("Unknown API index: " + this); } diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPI.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPI.java index 9a90d76519f6..b455b3b86581 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPI.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPI.java @@ -2516,32 +2516,60 @@ void validateContentletNoRels(Contentlet contentlet, */ void publishAssociated(Contentlet contentlet, boolean isNew, boolean isNewVersion) throws DotSecurityException, DotDataException, DotStateException; + /** + * Executes a raw JSON search query and returns a vendor-neutral response without loading contentlets. + * Use this method (not the Lucene-based overloads) when the query is an ES/OS JSON query body. + * + * @param query the JSON search query + * @param live {@code true} to query the live index + * @param user the user performing the action + * @param respectFrontendRoles whether front-end roles should be applied + * @return vendor-neutral {@link com.dotcms.content.index.domain.ContentSearchResponse} + * @see #esSearchRaw(String, boolean, User, boolean) + */ + default com.dotcms.content.index.domain.ContentSearchResponse searchRaw( + final String query, final boolean live, final User user, + final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + return com.dotmarketing.business.APILocator.getSearchAPI() + .searchRaw(query, live, user, respectFrontendRoles); + } + + /** + * Executes a JSON search query, loads the matching contentlets from the DB and returns them. + * Use this method (not the Lucene-based overloads) when the query is an ES/OS JSON query body. + * + * @param query the JSON search query + * @param live {@code true} to query the live index + * @param user the user performing the action + * @param respectFrontendRoles whether front-end roles should be applied + * @return vendor-neutral {@link com.dotcms.content.index.domain.ContentSearchResults} + * @see #esSearch(String, boolean, User, boolean) + */ + default com.dotcms.content.index.domain.ContentSearchResults search( + final String query, final boolean live, final User user, + final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + return com.dotmarketing.business.APILocator.getSearchAPI() + .search(query, live, user, respectFrontendRoles); + } + /** * This will only return the list of inodes as hits, and does not load the contentlets from cache. *
NOTE: dotCMS Enterprise only feature. * - * @param esQuery - * @param live - * @param user - * @param respectFrontendRoles - * @return - * @throws DotSecurityException - * @throws DotDataException + * @deprecated Use {@link #searchRaw(String, boolean, User, boolean)} for vendor-neutral access. */ + @Deprecated(forRemoval = true) public org.elasticsearch.action.search.SearchResponse esSearchRaw ( String esQuery, boolean live, User user, boolean respectFrontendRoles ) throws DotSecurityException, DotDataException; /** * Executes a given Elastic Search query. *
NOTE: dotCMS Enterprise only feature. * - * @param esQuery - * @param live - * @param user - * @param respectFrontendRoles - * @return - * @throws DotSecurityException - * @throws DotDataException + * @deprecated Use {@link #search(String, boolean, User, boolean)} for vendor-neutral access. */ + @Deprecated(forRemoval = true) public ESSearchResults esSearch ( String esQuery, boolean live, User user, boolean respectFrontendRoles ) throws DotSecurityException, DotDataException; /** diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIInterceptor.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIInterceptor.java index edbc03a51506..f9598632a049 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIInterceptor.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIInterceptor.java @@ -2,6 +2,8 @@ import com.dotcms.business.CloseDBIfOpened; import com.dotcms.content.index.IndexContentletScroll; +import com.dotcms.content.index.domain.ContentSearchResponse; +import com.dotcms.content.index.domain.ContentSearchResults; import com.dotcms.content.elasticsearch.business.ESSearchResults; import com.dotcms.content.elasticsearch.business.SearchCriteria; import com.dotcms.contenttype.model.type.ContentType; @@ -3291,6 +3293,45 @@ public SearchResponse esSearchRaw(String esQuery, boolean live, User user, return ret; } + @Override + public ContentSearchResults search(final String query, final boolean live, + final User user, final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + for (ContentletAPIPreHook pre : preHooks) { + if (!pre.search(query, live, user, respectFrontendRoles)) { + final String msg = String.format(PREHOOK_FAILED_MESSAGE, pre.getClass().getName()); + Logger.error(this, msg); + throw new DotRuntimeException(msg); + } + } + final ContentSearchResults ret = conAPI.search(query, live, user, respectFrontendRoles); + for (ContentletAPIPostHook post : postHooks) { + post.search(query, live, user, respectFrontendRoles); + } + return ret; + } + + @Override + public ContentSearchResponse searchRaw(final String query, final boolean live, + final User user, final boolean respectFrontendRoles) + throws DotSecurityException, DotDataException { + if (LicenseManager.getInstance().isCommunity()) { + throw new DotStateException("Need an enterprise license to run this functionality."); + } + for (ContentletAPIPreHook pre : preHooks) { + if (!pre.searchRaw(query, live, user, respectFrontendRoles)) { + final String msg = String.format(PREHOOK_FAILED_MESSAGE, pre.getClass().getName()); + Logger.error(this, msg); + throw new DotRuntimeException(msg); + } + } + final ContentSearchResponse ret = conAPI.searchRaw(query, live, user, respectFrontendRoles); + for (ContentletAPIPostHook post : postHooks) { + post.searchRaw(query, live, user, respectFrontendRoles); + } + return ret; + } + @Override public void updateUserReferences(User userToReplace, String replacementUserId, User user) throws DotDataException, DotSecurityException { diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIPostHook.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIPostHook.java index 906515fb5c97..83ddbe967a07 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIPostHook.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIPostHook.java @@ -1700,29 +1700,25 @@ public default void publishAssociated(Contentlet contentlet, boolean isNew, bool public default void publishAssociated(Contentlet contentlet, boolean isNew) throws DotSecurityException, DotDataException, DotContentletStateException, DotStateException{} /** - * - * @param esQuery - * @param live - * @param user - * @param respectFrontendRoles - * @throws DotSecurityException - * @throws DotDataException + * @deprecated Use {@link #searchRaw(String, boolean, User, boolean)} instead. */ + @Deprecated(forRemoval = true) public default void esSearchRaw(String esQuery, boolean live, User user, boolean respectFrontendRoles) throws DotSecurityException, DotDataException{} /** - * - * @param esQuery - * @param live - * @param user - * @param respectFrontendRoles - * @throws DotSecurityException - * @throws DotDataException + * @deprecated Use {@link #search(String, boolean, User, boolean)} instead. */ + @Deprecated(forRemoval = true) public default void esSearch(String esQuery, boolean live, User user, boolean respectFrontendRoles) throws DotSecurityException, DotDataException{} + public default void search(String query, boolean live, User user, + boolean respectFrontendRoles) throws DotSecurityException, DotDataException {} + + public default void searchRaw(String query, boolean live, User user, + boolean respectFrontendRoles) throws DotSecurityException, DotDataException {} + /** - * + * * @param buffy * @param user * @param roles diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIPreHook.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIPreHook.java index 5e4488e43631..6959542bff87 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIPreHook.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/ContentletAPIPreHook.java @@ -1998,35 +1998,33 @@ public default boolean publishAssociated(Contentlet contentlet, boolean isNew) t } /** - * - * @param esQuery - * @param live - * @param user - * @param respectFrontendRoles - * @return - * @throws DotSecurityException - * @throws DotDataException + * @deprecated Use {@link #searchRaw(String, boolean, User, boolean)} instead. */ + @Deprecated(forRemoval = true) public default boolean esSearchRaw(String esQuery, boolean live, User user, boolean respectFrontendRoles) throws DotSecurityException, DotDataException{ return true; } /** - * - * @param esQuery - * @param live - * @param user - * @param respectFrontendRoles - * @return - * @throws DotSecurityException - * @throws DotDataException + * @deprecated Use {@link #search(String, boolean, User, boolean)} instead. */ + @Deprecated(forRemoval = true) public default boolean esSearch(String esQuery, boolean live, User user, boolean respectFrontendRoles) throws DotSecurityException, DotDataException{ return true; } + public default boolean search(String query, boolean live, User user, + boolean respectFrontendRoles) throws DotSecurityException, DotDataException { + return true; + } + + public default boolean searchRaw(String query, boolean live, User user, + boolean respectFrontendRoles) throws DotSecurityException, DotDataException { + return true; + } + /** - * + * * @param buffy * @param user * @param roles diff --git a/dotcms-integration/src/test/java/com/dotcms/OpenSearchUpgradeSuite.java b/dotcms-integration/src/test/java/com/dotcms/OpenSearchUpgradeSuite.java index 0a8b93f95fe0..a275c4f9811a 100644 --- a/dotcms-integration/src/test/java/com/dotcms/OpenSearchUpgradeSuite.java +++ b/dotcms-integration/src/test/java/com/dotcms/OpenSearchUpgradeSuite.java @@ -9,6 +9,7 @@ import com.dotcms.content.index.opensearch.OSIndexAPIImplIntegrationTest; import com.dotcms.content.index.opensearch.OSClientConfigTest; import com.dotcms.content.index.opensearch.OSClientProviderIntegrationTest; +import com.dotcms.content.index.opensearch.OSSearchAPIImplIntegrationTest; import com.dotcms.junit.MainBaseSuite; import org.junit.runner.RunWith; import org.junit.runners.Suite.SuiteClasses; @@ -40,7 +41,8 @@ ContentFactoryIndexOperationsOSIntegrationTest.class, OSClientProviderIntegrationTest.class, OSClientConfigTest.class, - ContentletIndexAPIImplMigrationIT.class + ContentletIndexAPIImplMigrationIT.class, + OSSearchAPIImplIntegrationTest.class }) public class OpenSearchUpgradeSuite { } \ No newline at end of file diff --git a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ES6UpgradeTest.java b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ES6UpgradeTest.java index 8174a17a9f61..8d6b80c3e3d6 100644 --- a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ES6UpgradeTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ES6UpgradeTest.java @@ -18,6 +18,7 @@ import com.dotmarketing.portlets.categories.model.Category; import com.dotmarketing.portlets.contentlet.model.Contentlet; import com.dotmarketing.util.DateUtil; +import com.dotcms.content.index.domain.ContentSearchResults; import com.dotmarketing.util.Logger; import com.google.common.base.CaseFormat; import com.liferay.portal.model.User; @@ -93,8 +94,8 @@ public void testElasticSearchJson(final Object objectFile) json = json.replaceAll(TO_REPLACE_HOST_ID, site.getIdentifier()); Logger.info(this, json); - final ESSearchResults results = APILocator.getContentletAPI() - .esSearch(json, false, systemUser, false); + final ContentSearchResults results = APILocator.getContentletAPI() + .search(json, false, systemUser, false); Assert.assertNotNull(results); @@ -103,7 +104,7 @@ public void testElasticSearchJson(final Object objectFile) if (json.contains("agg")) { //This is an aggregation - Assert.assertFalse(results.getAggregations().asList().isEmpty()); + Assert.assertFalse(results.getAggregations().isEmpty()); } else { //Contentlets Assert.assertFalse(results.isEmpty()); diff --git a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESMappingAPITest.java b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESMappingAPITest.java index dce4aa240e7a..035d2d550c83 100644 --- a/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESMappingAPITest.java +++ b/dotcms-integration/src/test/java/com/dotcms/content/elasticsearch/business/ESMappingAPITest.java @@ -93,7 +93,9 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.elasticsearch.action.search.SearchResponse; +import com.dotcms.content.index.domain.AggregationBucket; +import com.dotcms.content.index.domain.ContentSearchResponse; +import com.dotcms.content.index.domain.ContentSearchResults; import org.junit.BeforeClass; import org.junit.Test; @@ -1046,10 +1048,9 @@ public void Test_Create_ContentType_With_KeyValue_Field_Test_Query_Expect_Succes + " }" + "}", queryString); - final ESSearchResults searchResults = contentletAPI.esSearch(wrappedQuery, false, user, false); + final ContentSearchResults searchResults = contentletAPI.search(wrappedQuery, false, user, false); assertFalse(searchResults.isEmpty()); - for (final Object searchResult : searchResults) { - final Contentlet contentlet = (Contentlet) searchResult; + for (final Contentlet contentlet : searchResults) { final Map map = (Map)contentlet.getMap().get("myKeyValueField"); assertEquals(map.get("key1"),"val1"); @@ -1074,17 +1075,16 @@ public void Test_Create_ContentType_With_KeyValue_Field_Test_Query_Expect_Succes + " } " + "}", flattenQueryString, aggregationString); - final SearchResponse raw = contentletAPI.esSearchRaw( + final ContentSearchResponse raw = contentletAPI.searchRaw( StringUtils.lowercaseStringExceptMatchingTokens(wrappedQueryWithAggregations, ESContentFactoryImpl.LUCENE_RESERVED_KEYWORDS_REGEX), false, user, false); - final JSONArray jsonArray = new JSONObject(raw.toString()).getJSONObject("aggregations") - .getJSONObject("sterms#tag").getJSONArray("buckets"); + final java.util.List buckets = raw.aggregations().get("tag"); + assertNotNull("aggregations must contain 'tag' key", buckets); - for(int i=0; i < jsonArray.length(); i++){ - final JSONObject object = (JSONObject)jsonArray.get(i); + for (int i = 0; i < buckets.size(); i++) { final int keyVal = i + 1; - assertEquals(String.format("key%d_val%d",keyVal, keyVal ),object.get("key")); + assertEquals(String.format("key%d_val%d", keyVal, keyVal), buckets.get(i).key()); } } @@ -1114,7 +1114,7 @@ public void Test_Create_FileAsset_Query_by_MetaData_Expect_Success() + " }" + "}", queryString); - final ESSearchResults searchResults = contentletAPI.esSearch(wrappedQuery, false, user, false); + final ContentSearchResults searchResults = contentletAPI.search(wrappedQuery, false, user, false); assertFalse(searchResults.isEmpty()); } @@ -1145,7 +1145,7 @@ public void Test_Create_FileAsset_With_Metadata_KeyValue_Then_Query() + " }" + " } " + "}", flattenQueryString.toLowerCase()); - final ESSearchResults searchResults = contentletAPI.esSearch(wrappedQuery, false, user, false); + final ContentSearchResults searchResults = contentletAPI.search(wrappedQuery, false, user, false); assertFalse(searchResults.isEmpty()); } diff --git a/dotcms-integration/src/test/java/com/dotcms/content/index/opensearch/OSSearchAPIImplIntegrationTest.java b/dotcms-integration/src/test/java/com/dotcms/content/index/opensearch/OSSearchAPIImplIntegrationTest.java new file mode 100644 index 000000000000..c9170182627e --- /dev/null +++ b/dotcms-integration/src/test/java/com/dotcms/content/index/opensearch/OSSearchAPIImplIntegrationTest.java @@ -0,0 +1,430 @@ +package com.dotcms.content.index.opensearch; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.dotcms.DataProviderWeldRunner; +import com.dotcms.IntegrationTestBase; +import com.dotcms.content.index.VersionedIndices; +import com.dotcms.content.index.VersionedIndicesAPI; +import com.dotcms.content.index.VersionedIndicesImpl; +import com.dotcms.content.index.domain.ContentSearchResponse; +import com.dotcms.content.index.domain.ContentSearchResults; +import com.dotcms.content.index.domain.IndexBulkRequest; +import com.dotcms.util.IntegrationTestInitService; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.common.db.DotConnect; +import com.dotmarketing.util.Logger; +import com.liferay.portal.model.User; +import java.util.List; +import java.util.UUID; +import javax.enterprise.context.ApplicationScoped; +import javax.inject.Inject; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.indices.RefreshRequest; + +/** + * Integration tests for {@link OSSearchAPIImpl} exercised against a live OpenSearch 3.x container. + * + *

Each test creates minimal live/working indices in OpenSearch, registers them in + * {@link VersionedIndicesAPI} so {@link OSSearchAPIImpl#resolveIndex} can find them, then + * verifies that the search path returns well-formed {@link ContentSearchResponse} / + * {@link ContentSearchResults} objects (structure and non-null invariants). The tests search + * against empty indices — no content is indexed — so the result sets are always empty, but the + * full execution path (permissions, pagination, JSON deserialisation) is exercised.

+ * + *

Requires the {@code opensearch-upgrade} Docker container running on + * {@code http://localhost:9201} with security disabled. + * Registered in {@link com.dotcms.OpenSearchUpgradeSuite}.

+ * + *

Run with: + * 

+ *   ./mvnw verify -pl :dotcms-integration \
+ *       -Dcoreit.test.skip=false \
+ *       -Dopensearch.upgrade.test=true
+ *
+ *

+ * + * @author Fabrizzio Araya + */ +@ApplicationScoped +@RunWith(DataProviderWeldRunner.class) +public class OSSearchAPIImplIntegrationTest extends IntegrationTestBase { + + private static final String RUN_ID = + UUID.randomUUID().toString().replace("-", "").substring(0, 8); + + private static final String IDX_LIVE = "live_search_" + RUN_ID; + private static final String IDX_WORKING = "working_search_" + RUN_ID; + + /** Unique identifier and inode for the document indexed in _source rewrite tests. */ + private static final String TEST_DOC_IDENTIFIER = "test-identifier-" + RUN_ID; + private static final String TEST_DOC_INODE = "test-inode-" + RUN_ID; + private static final String TEST_DOC_ID = TEST_DOC_IDENTIFIER + "_1_default"; + private static final String TEST_DOC_JSON = + "{\"identifier\":\"" + TEST_DOC_IDENTIFIER + "\"," + + "\"inode\":\"" + TEST_DOC_INODE + "\"," + + "\"title\":\"OSSearchAPIImpl source-rewrite test\"," + + "\"language_id\":1," + + "\"contenttype\":\"testtype\"}"; + + /** + * The version constant used by {@link VersionedIndicesAPI#loadDefaultVersionedIndices()}. + * Test indices must be registered under this version for {@code OSSearchAPIImpl.resolveIndex()} + * to find them. + */ + private static final String DEFAULT_OS_VERSION = VersionedIndices.OPENSEARCH_3X; + + // ── CDI-injected beans ────────────────────────────────────────────────── + @Inject + private OSSearchAPIImpl osSearchAPI; + + @Inject + private OSIndexAPIImpl osIndexAPI; + + @Inject + private ContentletIndexOperationsOS opsOS; + + @Inject + private OSClientProvider clientProvider; + + @Inject + private VersionedIndicesAPI versionedIndicesAPI; + + private User systemUser; + + // ======================================================================= + // Lifecycle + // ======================================================================= + + @BeforeClass + public static void prepare() throws Exception { + IntegrationTestInitService.getInstance().init(); + } + + @Before + public void setUp() throws Exception { + cleanupTestOsIndices(); + cleanupVersionedRows(); + + systemUser = APILocator.getUserAPI().getSystemUser(); + + // Create OS indices and register them under the default OPENSEARCH_3X version + // so resolveIndex() (which calls loadDefaultVersionedIndices()) finds them. + osIndexAPI.createIndex(IDX_LIVE, 1); + osIndexAPI.createIndex(IDX_WORKING, 1); + + // createIndex() stores the cluster-prefixed name in OpenSearch; resolveIndex() + // must use the same prefixed name, otherwise the search hits a non-existent index. + final String fullLive = osIndexAPI.getNameWithClusterIDPrefix(IDX_LIVE); + final String fullWorking = osIndexAPI.getNameWithClusterIDPrefix(IDX_WORKING); + + versionedIndicesAPI.saveIndices( + VersionedIndicesImpl.builder() + .version(DEFAULT_OS_VERSION) + .live(fullLive) + .working(fullWorking) + .build()); + + // Ensure resolveIndex() reads the freshly saved rows, not a cached prior state. + versionedIndicesAPI.clearCache(); + } + + @After + public void tearDown() { + cleanupTestOsIndices(); + cleanupVersionedRows(); + } + + // ======================================================================= + // Tests – searchRaw + // ======================================================================= + + /** + * Given scenario: A valid match-all JSON query against an empty working index. + * Expected: {@link OSSearchAPIImpl#searchRaw} returns a non-null {@link ContentSearchResponse} + * with a non-null {@link com.dotcms.content.index.domain.SearchHits} and zero total hits. + */ + @Test + public void test_searchRaw_matchAll_shouldReturnEmptyResultsWithNonNullStructure() + throws Exception { + + final String matchAll = "{\"query\":{\"match_all\":{}}}"; + + final ContentSearchResponse response = + osSearchAPI.searchRaw(matchAll, false, systemUser, false); + + assertNotNull("searchRaw must return a non-null ContentSearchResponse", response); + assertNotNull("hits must not be null on an empty-index response", response.hits()); + assertNotNull("aggregations map must not be null", response.aggregations()); + Logger.info(this, + "✅ test_searchRaw_matchAll_shouldReturnEmptyResultsWithNonNullStructure passed" + + " – totalHits=" + response.hits().totalHits().value()); + } + + /** + * Given scenario: A terms aggregation query against an empty working index. + * Expected: {@link ContentSearchResponse#aggregations()} contains the key {@code "entries"} + * (from the JSON {@code "aggs": {"entries": {"terms": ...}}}) and that bucket list + * is empty (no documents). + */ + @Test + public void test_searchRaw_withTermsAgg_shouldReturnAggregationKey() throws Exception { + final String aggQuery = + "{\"size\":0,\"aggs\":{\"entries\":{\"terms\":{\"field\":\"contenttype_dotraw\",\"size\":100}}}}"; + + final ContentSearchResponse response = + osSearchAPI.searchRaw(aggQuery, false, systemUser, false); + + assertNotNull("searchRaw must return a non-null response", response); + assertTrue( + "aggregations map must contain the 'entries' key even when result is empty", + response.aggregations().containsKey("entries")); + assertTrue("entries bucket must be empty for an empty index", + response.aggregations().get("entries").isEmpty()); + Logger.info(this, + "✅ test_searchRaw_withTermsAgg_shouldReturnAggregationKey passed"); + } + + /** + * Given scenario: A match-all query against the live index. + * Expected: No exception is thrown and a valid response is returned. + */ + @Test + public void test_searchRaw_liveIndex_shouldNotThrow() throws Exception { + final String matchAll = "{\"query\":{\"match_all\":{}}}"; + + final ContentSearchResponse response = + osSearchAPI.searchRaw(matchAll, true, systemUser, false); + + assertNotNull("searchRaw against live index must return non-null response", response); + assertNotNull("hits must be non-null for live index query", response.hits()); + Logger.info(this, "✅ test_searchRaw_liveIndex_shouldNotThrow passed"); + } + + /** + * Given scenario: A null query string is passed to {@link OSSearchAPIImpl#searchRaw}. + * Expected: A {@link com.dotmarketing.business.DotStateException} is thrown. + */ + @Test(expected = com.dotmarketing.business.DotStateException.class) + public void test_searchRaw_nullQuery_shouldThrowDotStateException() throws Exception { + osSearchAPI.searchRaw(null, false, systemUser, false); + } + + /** + * Given scenario: An empty string query is passed to {@link OSSearchAPIImpl#searchRaw}. + * Expected: A {@link com.dotmarketing.business.DotStateException} is thrown. + */ + @Test(expected = com.dotmarketing.business.DotStateException.class) + public void test_searchRaw_emptyQuery_shouldThrowDotStateException() throws Exception { + osSearchAPI.searchRaw("", false, systemUser, false); + } + + /** + * Given scenario: {@code user} is {@code null} and {@code respectFrontendRoles} is false. + * Expected: A {@link com.dotmarketing.exception.DotSecurityException} is thrown. + */ + @Test(expected = com.dotmarketing.exception.DotSecurityException.class) + public void test_searchRaw_nullUserNoFrontendRoles_shouldThrowDotSecurityException() + throws Exception { + osSearchAPI.searchRaw("{\"query\":{\"match_all\":{}}}", false, null, false); + } + + // ======================================================================= + // Tests – search (full contentlet population) + // ======================================================================= + + /** + * Given scenario: A match-all JSON query with no documents in the index. + * Expected: {@link OSSearchAPIImpl#search} returns a non-null {@link ContentSearchResults} + * that is empty and reports zero total hits. + */ + @Test + public void test_search_matchAll_emptyIndex_shouldReturnEmptyContentSearchResults() + throws Exception { + + final String matchAll = "{\"query\":{\"match_all\":{}}}"; + + final ContentSearchResults results = + osSearchAPI.search(matchAll, false, systemUser, false); + + assertNotNull("search must return non-null ContentSearchResults", results); + assertTrue("ContentSearchResults must be empty when index has no documents", + results.isEmpty()); + assertNotNull("getResponse must return non-null ContentSearchResponse", + results.getResponse()); + Logger.info(this, + "✅ test_search_matchAll_emptyIndex_shouldReturnEmptyContentSearchResults passed"); + } + + /** + * Given scenario: A match-all query is run with {@code respectFrontendRoles=true} and a null + * user — the anonymous-user path. + * Expected: No security exception; an empty but valid result is returned. + */ + @Test + public void test_search_respectFrontendRoles_nullUser_shouldNotThrow() throws Exception { + final String matchAll = "{\"query\":{\"match_all\":{}}}"; + + final ContentSearchResults results = + osSearchAPI.search(matchAll, false, null, true); + + assertNotNull("search with respectFrontendRoles=true must return non-null results", results); + Logger.info(this, + "✅ test_search_respectFrontendRoles_nullUser_shouldNotThrow passed"); + } + + // ======================================================================= + // Tests – pagination + // ======================================================================= + + /** + * Given scenario: A search with explicit limit and offset against an empty index. + * Expected: No exception; result is empty and response structure is valid. + */ + @Test + public void test_searchRelated_withLimitAndOffset_emptyIndex_shouldReturnValidStructure() + throws Exception { + + final String matchAll = "{\"query\":{\"match_all\":{}}}"; + // Use a fake identifier/inode so the method constructs the query and fires against OS + // (it will find 0 results because the index is empty) + final com.dotmarketing.portlets.contentlet.model.Contentlet fakeContent = + new com.dotmarketing.portlets.contentlet.model.Contentlet(); + fakeContent.setIdentifier("fake-id-" + RUN_ID); + fakeContent.setInode("fake-inode-" + RUN_ID); + + final ContentSearchResponse response = osSearchAPI.searchRelated( + fakeContent, "fakeRelationship", false, false, systemUser, false, 10, 0, null); + + assertNotNull("searchRelated must return non-null response", response); + assertNotNull("hits must be non-null", response.hits()); + Logger.info(this, + "✅ test_searchRelated_withLimitAndOffset_emptyIndex_shouldReturnValidStructure passed"); + } + + // ======================================================================= + // Tests – _source rewrite (identifier + inode always present) + // ======================================================================= + + /** + * Given scenario: A document with both {@code identifier} and {@code inode} fields is indexed. + * A plain match-all query (no {@code _source} clause) is executed via + * {@link OSSearchAPIImpl#searchRaw}. + * Expected: {@link OSSearchAPIImpl} rewrites the query to {@code "_source":[identifier,inode]} + * and both fields are returned in the hit's {@code sourceAsMap}. + */ + @Test + public void test_searchRaw_withIndexedDocument_sourceRewrite_shouldIncludeIdentifierAndInode() + throws Exception { + + final String fullWorking = osIndexAPI.getNameWithClusterIDPrefix(IDX_WORKING); + indexTestDocument(fullWorking); + + final String matchAll = "{\"query\":{\"match_all\":{}}}"; + final ContentSearchResponse response = + osSearchAPI.searchRaw(matchAll, false, systemUser, false); + + assertNotNull("searchRaw must return a non-null response", response); + assertEquals("Exactly one hit expected", 1, response.hits().totalHits().value()); + + final com.dotcms.content.index.domain.SearchHit hit = + response.hits().iterator().next(); + assertEquals("identifier must be present in sourceAsMap", + TEST_DOC_IDENTIFIER, hit.sourceAsMap().get("identifier")); + assertEquals("inode must be present in sourceAsMap — _source rewrite must include it", + TEST_DOC_INODE, hit.sourceAsMap().get("inode")); + + Logger.info(this, "✅ test_searchRaw_withIndexedDocument_sourceRewrite_shouldIncludeIdentifierAndInode passed"); + } + + /** + * Given scenario: A document with both {@code identifier} and {@code inode} fields is indexed. + * A query that explicitly restricts {@code _source} to only {@code ["identifier"]} + * is executed via {@link OSSearchAPIImpl#searchRaw}. + * Expected: The impl overwrites the caller's {@code _source} with {@code [identifier, inode]}, + * so {@code inode} is still present in the hit's {@code sourceAsMap} despite not + * being requested by the caller. + */ + @Test + public void test_searchRaw_userDefinedSource_isOverwrittenToIncludeInode() + throws Exception { + + final String fullWorking = osIndexAPI.getNameWithClusterIDPrefix(IDX_WORKING); + indexTestDocument(fullWorking); + + // Caller requests only identifier — inode intentionally omitted. + final String queryWithSource = + "{\"query\":{\"match_all\":{}},\"_source\":[\"identifier\"]}"; + final ContentSearchResponse response = + osSearchAPI.searchRaw(queryWithSource, false, systemUser, false); + + assertNotNull(response); + assertTrue("Response must have at least one hit", response.hits().totalHits().value() > 0); + + final com.dotcms.content.index.domain.SearchHit hit = + response.hits().iterator().next(); + assertNotNull("inode must still be present after _source overwrite", + hit.sourceAsMap().get("inode")); + + Logger.info(this, "✅ test_searchRaw_userDefinedSource_isOverwrittenToIncludeInode passed"); + } + + // ======================================================================= + // Helpers + // ======================================================================= + + /** + * Indexes {@link #TEST_DOC_JSON} into the given index and refreshes it so the + * document is immediately visible to searches. + */ + private void indexTestDocument(final String fullIndexName) throws Exception { + final IndexBulkRequest req = opsOS.createBulkRequest(); + opsOS.addIndexOp(req, fullIndexName, TEST_DOC_ID, TEST_DOC_JSON); + opsOS.putToIndex(req); + refreshTestIndex(fullIndexName); + } + + private void refreshTestIndex(final String fullIndexName) { + try { + final OpenSearchClient client = clientProvider.getClient(); + client.indices().refresh(RefreshRequest.of(r -> r.index(fullIndexName))); + } catch (final Exception e) { + Logger.warn(this, "refreshTestIndex: error refreshing '" + fullIndexName + + "': " + e.getMessage()); + } + } + + private synchronized void cleanupTestOsIndices() { + for (final String idx : List.of(IDX_LIVE, IDX_WORKING)) { + try { + if (osIndexAPI.indexExists(idx)) { + osIndexAPI.delete(idx); + } + } catch (Exception e) { + Logger.warn(this, "Cleanup: error removing OS index '" + idx + "': " + e.getMessage()); + } + } + } + + private void cleanupVersionedRows() { + try { + // Remove only the test-scoped index entries by their RUN_ID-tagged index names. + // Deleting by index_name avoids removing the entire OPENSEARCH_3X version row, + // which would break other tests running in parallel. + new DotConnect() + .setSQL("DELETE FROM indicies WHERE index_name LIKE ?") + .addParam("%" + RUN_ID + "%") + .loadResult(); + } catch (Exception e) { + Logger.warn(this, "Cleanup: error removing versioned DB rows: " + e.getMessage()); + } + } + +} diff --git a/dotcms-integration/src/test/java/com/dotcms/contenttype/business/ContentTypeDestroyAPIImplTest.java b/dotcms-integration/src/test/java/com/dotcms/contenttype/business/ContentTypeDestroyAPIImplTest.java index 8de5ac5a8e9c..c0cf0253a874 100644 --- a/dotcms-integration/src/test/java/com/dotcms/contenttype/business/ContentTypeDestroyAPIImplTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/contenttype/business/ContentTypeDestroyAPIImplTest.java @@ -3,7 +3,7 @@ import com.dotcms.IntegrationTestBase; import com.dotcms.JUnit4WeldRunner; import com.dotcms.business.CloseDBIfOpened; -import com.dotcms.content.elasticsearch.business.ESSearchResults; +import com.dotcms.content.index.domain.ContentSearchResults; import com.dotcms.contenttype.business.ContentTypeDestroyAPIImpl.ContentletVersionInfo; import com.dotcms.contenttype.model.field.Field; import com.dotcms.contenttype.model.type.ContentType; @@ -149,7 +149,7 @@ public void Destroy_General_Test() throws DotDataException, DotSecurityException Assert.assertEquals(0, count); final Set inodes = contentTypeAndInode._2(); - //Test no content is left hang around + //Test no content is left to hang around for (String inode:inodes) { count = new DotConnect().setSQL("select count(*) as x from contentlet where inode = ? ").addParam(inode).getInt("x"); Assert.assertEquals(0, count); @@ -164,7 +164,7 @@ public void Destroy_General_Test() throws DotDataException, DotSecurityException Assert.assertTrue(searchIndex(anyInode.get(), false).isEmpty()); } - private ESSearchResults searchIndex(final ContentType contentType, final boolean live ) + private ContentSearchResults searchIndex(final ContentType contentType, final boolean live ) throws DotDataException, DotSecurityException { final String esQuery = String.format("{\n" @@ -175,12 +175,12 @@ private ESSearchResults searchIndex(final ContentType contentType, final boolean + " }\n" + "}", contentType.variable()); - return APILocator.getEsSearchAPI() - .esSearch(esQuery, live, APILocator.systemUser(), false); + return APILocator.getSearchAPI() + .search(esQuery, live, APILocator.systemUser(), false); } - private ESSearchResults searchIndex(final String inode, final boolean live ) + private ContentSearchResults searchIndex(final String inode, final boolean live ) throws DotDataException, DotSecurityException { final String esQuery = String.format("{\n" @@ -191,8 +191,8 @@ private ESSearchResults searchIndex(final String inode, final boolean live ) + " }\n" + "}", inode); - return APILocator.getEsSearchAPI() - .esSearch(esQuery, live, APILocator.systemUser(), false); + return APILocator.getSearchAPI() + .search(esQuery, live, APILocator.systemUser(), false); } /** * Given Scenario: We create a Content Type with a relationship field diff --git a/dotcms-integration/src/test/java/com/dotcms/rest/api/v1/page/PageResourceTest.java b/dotcms-integration/src/test/java/com/dotcms/rest/api/v1/page/PageResourceTest.java index 7b084786c56e..ab5d0065b683 100644 --- a/dotcms-integration/src/test/java/com/dotcms/rest/api/v1/page/PageResourceTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/rest/api/v1/page/PageResourceTest.java @@ -24,7 +24,9 @@ import com.dotcms.JUnit4WeldRunner; import com.dotcms.api.web.HttpServletRequestThreadLocal; import com.dotcms.api.web.HttpServletResponseThreadLocal; -import com.dotcms.content.elasticsearch.business.ESSearchResults; +import com.dotcms.content.index.domain.ContentSearchResponse; +import com.dotcms.content.index.domain.ContentSearchResults; +import com.dotcms.content.index.domain.SearchHits; import com.dotcms.contenttype.business.ContentTypeAPI; import com.dotcms.contenttype.model.field.TextField; import com.dotcms.contenttype.model.type.ContentType; @@ -137,7 +139,6 @@ import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; import javax.ws.rs.core.Response; -import org.elasticsearch.action.search.SearchResponse; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -563,12 +564,12 @@ public void testPathParam() throws DotSecurityException, DotDataException { final String path = pagePath; - final SearchResponse searchResponse = mock(SearchResponse.class); - final Contentlet contentlet = pageAsset; final List contentlets = list(contentlet); - final ESSearchResults results = new ESSearchResults(searchResponse, contentlets); + final ContentSearchResults results = new ContentSearchResults<>( + ContentSearchResponse.builder().hits(SearchHits.empty()).tookMillis(0).build(), + contentlets); final String query = String.format("{" + "query: {" + "query_string: {" @@ -578,7 +579,7 @@ public void testPathParam() + "}", path.replace("/", "\\\\/")); - when(esapi.esSearch(query, false, user, false)).thenReturn(results); + when(esapi.search(query, false, user, false)).thenReturn(results); final Response response = pageResource.searchPage(request, new EmptyHttpResponse(), path, false, true); RestUtilTest.verifySuccessResponse(response); @@ -603,10 +604,10 @@ public void testPathParamWithHost() throws DotSecurityException, DotDataException { final String path = String.format("//%s/%s/%s", hostName, folderName, pageName); - final SearchResponse searchResponse = mock(SearchResponse.class); - final List contentlets = list(pageAsset); - final ESSearchResults results = new ESSearchResults(searchResponse, contentlets); + final ContentSearchResults results = new ContentSearchResults<>( + ContentSearchResponse.builder().hits(SearchHits.empty()).tookMillis(0).build(), + contentlets); String preparedPagePath = String.format("%s/%s",folderName,pageName).replace("/", "\\\\/"); final String query = String.format("{" + "query: {" @@ -616,7 +617,7 @@ public void testPathParamWithHost() + "}" + "}", preparedPagePath, host.getHostname()); - when(esapi.esSearch(query, false, user, false)).thenReturn(results); + when(esapi.search(query, false, user, false)).thenReturn(results); final Response response = pageResource.searchPage(request, new EmptyHttpResponse(), path, false, true); RestUtilTest.verifySuccessResponse(response); diff --git a/dotcms-integration/src/test/java/com/dotmarketing/portlets/contentlet/business/ContentletAPITest.java b/dotcms-integration/src/test/java/com/dotmarketing/portlets/contentlet/business/ContentletAPITest.java index eb979795b51c..4331138f46f9 100644 --- a/dotcms-integration/src/test/java/com/dotmarketing/portlets/contentlet/business/ContentletAPITest.java +++ b/dotcms-integration/src/test/java/com/dotmarketing/portlets/contentlet/business/ContentletAPITest.java @@ -191,7 +191,7 @@ import org.apache.velocity.context.InternalContextAdapterImpl; import org.apache.velocity.runtime.parser.node.SimpleNode; import org.awaitility.Awaitility; -import org.elasticsearch.action.search.SearchResponse; +import com.dotcms.content.index.domain.ContentSearchResponse; import org.jetbrains.annotations.NotNull; import org.junit.Assert; import org.junit.Ignore; @@ -8305,7 +8305,7 @@ public void test_variant_present_in_document_id() throws Exception { + " }," + "}"; - final SearchResponse responseDefaultVariant = APILocator.getContentletAPI().esSearchRaw( + final ContentSearchResponse responseDefaultVariant = APILocator.getContentletAPI().searchRaw( StringUtils.lowercaseStringExceptMatchingTokens(queryContentOnDefaultVariant, ESContentFactoryImpl.LUCENE_RESERVED_KEYWORDS_REGEX), false, APILocator.systemUser(), false); @@ -8313,7 +8313,7 @@ public void test_variant_present_in_document_id() throws Exception { assertEquals(contentDefaultVariant.getIdentifier() + "_" + contentDefaultVariant.getLanguageId() + "_" + contentDefaultVariant.getVariantId(), - responseDefaultVariant.getHits().iterator().next().getId()); + responseDefaultVariant.hits().iterator().next().id()); final String queryContentOnNewVariant = "{" + "query: {" @@ -8323,14 +8323,14 @@ public void test_variant_present_in_document_id() throws Exception { + " }," + "}"; - final SearchResponse responseNewVariant = APILocator.getContentletAPI().esSearchRaw( + final ContentSearchResponse responseNewVariant = APILocator.getContentletAPI().searchRaw( StringUtils.lowercaseStringExceptMatchingTokens(queryContentOnNewVariant, ESContentFactoryImpl.LUCENE_RESERVED_KEYWORDS_REGEX), false, APILocator.systemUser(), false); assertEquals(contentNewVariant.getIdentifier() + "_" + contentNewVariant.getLanguageId() + "_" + contentNewVariant.getVariantId(), - responseNewVariant.getHits().iterator().next().getId()); + responseNewVariant.hits().iterator().next().id()); } @DataProvider