Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/issue-15531-fixing-mutipletimes-…
Browse files Browse the repository at this point in the history 
…refresh'
  • Loading branch information
@wezell
wezell committed Nov 27, 2018
2 parents 5d35ed0 + f5a9555 commit 1ea323d
Show file tree
Hide file tree
Showing 7 changed files with 364 additions and 231 deletions.
46 changes: 29 additions & 17 deletions dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentletAPIImpl.java
View file
Expand Up @@ -175,21 +175,22 @@
public class ESContentletAPIImpl implements ContentletAPI {

private static final String CAN_T_CHANGE_STATE_OF_CHECKED_OUT_CONTENT = "Can't change state of checked out content or where inode is not set. Use Search or Find then use method";
private static final String CANT_GET_LOCK_ON_CONTENT ="Only the CMS Admin or the user who locked the contentlet can lock/unlock it";
private static final String FAILED_TO_DELETE_UNARCHIVED_CONTENT = "Failed to delete unarchived content. Content must be archived first before it can be deleted.";
private static final String NEVER_EXPIRE = "NeverExpire";
private static final String CANT_GET_LOCK_ON_CONTENT = "Only the CMS Admin or the user who locked the contentlet can lock/unlock it";
private static final String FAILED_TO_DELETE_UNARCHIVED_CONTENT = "Failed to delete unarchived content. Content must be archived first before it can be deleted.";
private static final String NEVER_EXPIRE = "NeverExpire";
private static final String CHECKIN_IN_PROGRESS = "__checkin_in_progress__";

private final ESContentletIndexAPI indexAPI;
private final ESContentFactoryImpl contentFactory;
private final PermissionAPI permissionAPI;
private final CategoryAPI categoryAPI;
private final RelationshipAPI relationshipAPI;
private final FieldAPI fieldAPI;
private final LanguageAPI languageAPI;
private final PermissionAPI permissionAPI;
private final CategoryAPI categoryAPI;
private final RelationshipAPI relationshipAPI;
private final FieldAPI fieldAPI;
private final LanguageAPI languageAPI;
private final DistributedJournalAPI<String> distributedJournalAPI;
private final TagAPI tagAPI;
private final TagAPI tagAPI;

private int MAX_LIMIT = 10000;
private static final int MAX_LIMIT = 10000;

private static final String backupPath = ConfigUtils.getBackupPath() + File.separator + "contentlets";

Expand Down Expand Up @@ -2202,7 +2203,7 @@ public void refresh(Structure structure) throws DotReindexStateException {
* @throws DotReindexStateException
* @throws DotDataException
*/
private void refreshNoDeps(Contentlet contentlet) throws DotReindexStateException,
private void refreshNoDeps(final Contentlet contentlet) throws DotReindexStateException,
DotDataException {
indexAPI.addContentToIndex(contentlet, false);
CacheLocator.getContentletCache().add(contentlet.getInode(), contentlet);
Expand Down Expand Up @@ -2517,8 +2518,10 @@ public void deleteRelatedContent(final Contentlet contentlet, final Relationship
}
}

// Refresh the parent
refreshNoDeps(contentlet);
// Refresh the parent only if the contentlet is not already in the checkin
if (contentlet.get(CHECKIN_IN_PROGRESS) == null || !contentlet.getBoolProperty(CHECKIN_IN_PROGRESS)) {
refreshNoDeps(contentlet);
}
}

@WrapInTransaction
Expand Down Expand Up @@ -3125,6 +3128,7 @@ private Contentlet checkin(Contentlet contentlet, ContentletRelationships conten
}

if (createNewVersion || (!createNewVersion && (contentRelationships != null || cats != null))) {
contentlet.setBoolProperty(CHECKIN_IN_PROGRESS, Boolean.TRUE);
moveContentDependencies(workingContentlet, contentlet, contentRelationships, cats, user, respectFrontendRoles);
}

Expand Down Expand Up @@ -3530,8 +3534,6 @@ else if (incomingFile.exists() ){
FileAsset asset = APILocator.getFileAssetAPI().fromContentlet(contentlet);
}



ActivityLogger.logInfo(getClass(), "Content Saved", "StartDate: " +contentPushPublishDate+ "; "
+ "EndDate: " +contentPushExpireDate + "; User:" + user.getUserId() + "; ContentIdentifier: " + contentlet.getIdentifier(), contentlet.getHost());

Expand All @@ -3542,11 +3544,21 @@ else if (incomingFile.exists() ){
}

return contentlet;
}finally{
contentlet.cleanup();
} finally {

this.cleanup(contentlet);
}
}

private void cleanup(final Contentlet contentlet) {

if (contentlet.getMap().containsKey(CHECKIN_IN_PROGRESS)) {
contentlet.getMap().remove(CHECKIN_IN_PROGRESS);
}

contentlet.cleanup();
}

private void updateTemplateInAllLanguageVersions(final Contentlet contentlet, final User user)
throws DotDataException, DotSecurityException{

Expand Down
134 changes: 76 additions & 58 deletions dotCMS/src/main/java/com/dotcms/content/elasticsearch/business/ESContentletIndexAPI.java
View file
Expand Up @@ -5,6 +5,7 @@
import com.dotcms.content.business.DotMappingException;
import com.dotcms.content.elasticsearch.business.IndiciesAPI.IndiciesInfo;
import com.dotcms.content.elasticsearch.util.ESClient;
import com.dotcms.exception.ExceptionUtil;
import com.dotcms.tika.TikaUtils;
import com.dotcms.util.CollectionsUtils;
import com.dotmarketing.business.APILocator;
Expand Down Expand Up @@ -61,9 +62,11 @@ public class ESContentletIndexAPI implements ContentletIndexAPI{
private static final String TIMEOUT_INDEX_WAIT_FOR = "TIMEOUT_INDEX_WAIT_FOR";
private static final int TIME_INDEX_FORCE_DEFAULT = 30000;
private static final String TIMEOUT_INDEX_FORCE = "TIMEOUT_INDEX_FORCE";

private static final String SELECT_CONTENTLET_VERSION_INFO = "select working_inode,live_inode from contentlet_version_info where identifier=?";
private static DistributedJournalAPI<String> journalAPI = null;
private static final ESIndexAPI esIndexApi = new ESIndexAPI();
private static final ESMappingAPIImpl mappingAPI = new ESMappingAPIImpl();
private static final ESIndexAPI esIndexApi = new ESIndexAPI();
private static final ESMappingAPIImpl mappingAPI = new ESMappingAPIImpl();

public static final SimpleDateFormat timestampFormatter=new SimpleDateFormat("yyyyMMddHHmmss");

Expand Down Expand Up @@ -318,30 +321,30 @@ public void addContentToIndex(final Contentlet content, final boolean deps, bool
}

@WrapInTransaction
public void addContentToIndex(final Contentlet content,
public void addContentToIndex(final Contentlet parentContenlet,
final boolean includeDependencies,
final boolean indexBeforeCommit,
final boolean reindexOnly,
final BulkRequestBuilder bulk) throws DotHibernateException {

if (null == content || !UtilMethods.isSet(content.getIdentifier())) {
if (null == parentContenlet || !UtilMethods.isSet(parentContenlet.getIdentifier())) {
return;
}

// http://jira.dotmarketing.net/browse/DOTCMS-6886
// check for related content to reindex
List<Contentlet> contentDependencies = null;
final boolean indexIsNotDefer = IndexPolicy.DEFER != content.getIndexPolicy();
final boolean indexIsNotDefer = IndexPolicy.DEFER != parentContenlet.getIndexPolicy();
final List<Contentlet> contentToIndex = new ArrayList<>();

contentToIndex.add(content);
contentToIndex.add(parentContenlet);

try {

if(includeDependencies){

final List<Contentlet> dependencies = loadDeps(content);
dependencies.forEach(contentlet -> contentlet.setIndexPolicy(content.getIndexPolicyDependencies()));
final List<Contentlet> dependencies = loadDeps(parentContenlet);
dependencies.forEach(contentlet -> contentlet.setIndexPolicy(parentContenlet.getIndexPolicyDependencies()));
if (indexIsNotDefer) {
contentDependencies = new ArrayList<>(dependencies);
} else {
Expand All @@ -353,7 +356,7 @@ public void addContentToIndex(final Contentlet content,

if (indexIsNotDefer) {

this.handleIndexNotDefer(content, reindexOnly, bulk, contentDependencies, contentToIndex, false);
this.handleIndexNotDefer(parentContenlet, reindexOnly, bulk, contentDependencies, contentToIndex, false);
} else {

this.indexContentList(contentToIndex, bulk, reindexOnly);
Expand All @@ -362,11 +365,11 @@ public void addContentToIndex(final Contentlet content,

if (indexIsNotDefer) {

this.handleIndexNotDefer(content, reindexOnly, bulk, contentDependencies, contentToIndex, true);
this.handleIndexNotDefer(parentContenlet, reindexOnly, bulk, contentDependencies, contentToIndex, true);
} else {
// add a commit listener to index the contentlet if the entire
// transaction finish clean
HibernateUtil.addCommitListener(content.getInode() + ReindexRunnable.Action.ADDING,
HibernateUtil.addCommitListener(parentContenlet.getInode() + ReindexRunnable.Action.ADDING,
new AddReindexRunnable(contentToIndex, ReindexRunnable.Action.ADDING, bulk, reindexOnly));
}
}
Expand Down Expand Up @@ -571,46 +574,62 @@ private void indexContentletList(BulkRequestBuilder req, List<Contentlet> conten
boolean alwaysRegenerateMetadata = Config
.getBooleanProperty("always.regenerate.metadata.on.reindex", false);

for(Contentlet con : contentToIndexSet) {
String id=con.getIdentifier()+"_"+con.getLanguageId();
IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();
Gson gson=new Gson(); // todo why do we create a new Gson everytime
String mapping=null;
for(final Contentlet contentlet : contentToIndexSet) {

final String id=contentlet.getIdentifier()+"_"+contentlet.getLanguageId();
Logger.debug(this, ()->"\n*********----------- Indexing : " + Thread.currentThread().getName()
+ ", id: " + contentlet.getIdentifier() + ", identityHashCode: " + System.identityHashCode(contentlet));
Logger.debug(this, ()->"*********----------- " + DbConnectionFactory.getConnection());
Logger.debug(this, ()->"*********----------- " + ExceptionUtil.getCurrentStackTraceAsString(Config.getIntProperty("stacktracelimit", 10)) + "\n");

final IndiciesInfo info = APILocator.getIndiciesAPI().loadIndicies();
final Gson gson = new Gson(); // todo why do we create a new Gson everytime
String mapping = null;

try {

if (con.isLive() || con.isWorking()) {
if (contentlet.isLive() || contentlet.isWorking()) {
if (alwaysRegenerateMetadata) {
new TikaUtils().generateMetaData(con, true);
new TikaUtils().generateMetaData(contentlet, true);
} else if (regenerateMissingMetadata) {
new TikaUtils().generateMetaData(con);
new TikaUtils().generateMetaData(contentlet);
}
}

if (con.isWorking()) {
mapping=gson.toJson(mappingAPI.toMap(con));
if (contentlet.isWorking()) {

mapping = gson.toJson(mappingAPI.toMap(contentlet));

if(!reindexOnly)
req.add(new IndexRequest(info.working, "content", id)
.source(mapping, XContentType.JSON));
if(info.reindex_working!=null)
req.add(new IndexRequest(info.reindex_working, "content", id)
.source(mapping, XContentType.JSON));
if (!reindexOnly) {
req.add(new IndexRequest(info.working, "content", id)
.source(mapping, XContentType.JSON));
}

if (info.reindex_working!=null) {
req.add(new IndexRequest(info.reindex_working, "content", id)
.source(mapping, XContentType.JSON));
}
}

if(con.isLive()) {
if(mapping==null)
mapping=gson.toJson(mappingAPI.toMap(con));
if (contentlet.isLive()) {
if(mapping==null) {
mapping = gson.toJson(mappingAPI.toMap(contentlet));
}

if(!reindexOnly)
req.add(new IndexRequest(info.live, "content", id)
.source(mapping, XContentType.JSON));
if(info.reindex_live!=null)
req.add(new IndexRequest(info.reindex_live, "content", id)
.source(mapping, XContentType.JSON));
if(!reindexOnly) {
req.add(new IndexRequest(info.live, "content", id)
.source(mapping, XContentType.JSON));
}

if(info.reindex_live!=null) {
req.add(new IndexRequest(info.reindex_live, "content", id)
.source(mapping, XContentType.JSON));
}
}
}
catch(DotMappingException ex) {
Logger.error(this, "Can't get a mapping for contentlet with id_lang:" + id + " Content data: " + con.getMap(), ex);

contentlet.markAsReindexed();
} catch(DotMappingException ex) {
Logger.error(this, "Can't get a mapping for contentlet with id_lang:" + id + " Content data: " + contentlet.getMap(), ex);
throw ex;
}
}
Expand All @@ -619,32 +638,31 @@ private void indexContentletList(BulkRequestBuilder req, List<Contentlet> conten

@CloseDBIfOpened
@SuppressWarnings("unchecked")
public List<Contentlet> loadDeps(Contentlet content) throws DotDataException, DotSecurityException {
List<Contentlet> contentToIndex=new ArrayList<Contentlet>();
List<String> depsIdentifiers=mappingAPI.dependenciesLeftToReindex(content);
for(String ident : depsIdentifiers) {
public List<Contentlet> loadDeps(final Contentlet parentContentlet) throws DotDataException, DotSecurityException {

final List<Contentlet> contentToIndex = new ArrayList<Contentlet>();
final List<String> depsIdentifiers = this.mappingAPI.dependenciesLeftToReindex(parentContentlet);
for(final String identifier : depsIdentifiers) {

// get working and live version for all languages based on the identifier
// String sql = "select distinct inode from contentlet join contentlet_version_info " +
// " on (inode=live_inode or inode=working_inode) and contentlet.identifier=?";
String sql = "select working_inode,live_inode from contentlet_version_info where identifier=?";

DotConnect dc = new DotConnect();
dc.setSQL(sql);
dc.addParam(ident);
List<Map<String,String>> ret = dc.loadResults();
List<String> inodes = new ArrayList<String>();
for(Map<String,String> m : ret) {
String workingInode = m.get("working_inode");
String liveInode = m.get("live_inode");
final List<Map<String,String>> versionInfoMapResults =
new DotConnect().setSQL(SELECT_CONTENTLET_VERSION_INFO).addParam(identifier).loadResults();
final List<String> inodes = new ArrayList<>();
for(final Map<String,String> versionInfoMap : versionInfoMapResults) {

final String workingInode = versionInfoMap.get("working_inode");
final String liveInode = versionInfoMap.get("live_inode");
inodes.add(workingInode);
if(UtilMethods.isSet(liveInode) && !workingInode.equals(liveInode)){
inodes.add(liveInode);
}
}

for(String inode : inodes) {
Contentlet con=APILocator.getContentletAPI().find(inode, APILocator.getUserAPI().getSystemUser(), false);
contentToIndex.add(con);
for(final String inode : inodes) {

final Contentlet contentlet = APILocator.getContentletAPI()
.find(inode, APILocator.getUserAPI().getSystemUser(), false);
contentToIndex.add(contentlet);
}
}
return contentToIndex;
Expand Down

0 comments on commit 1ea323d

Please sign in to comment.