Skip to content

Commit

Permalink
#24093 Working on the migration task to populate missing contentlet a…
Browse files Browse the repository at this point in the history 
…s JSON data
  • Loading branch information
@jgambarios
jgambarios committed Mar 24, 2023
1 parent 87ee138 commit 5aa3a75
Show file tree
Hide file tree
Showing 3 changed files with 147 additions and 61 deletions.
31 changes: 29 additions & 2 deletions .../integration-test/java/com/dotcms/util/content/json/PopulateContentletAsJSONUtilTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,44 @@
package com.dotcms.util.content.json;

import com.dotcms.IntegrationTestBase;
import com.dotcms.util.IntegrationTestInitService;
import com.dotmarketing.exception.DotDataException;
import org.apache.felix.framework.OSGIUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import java.io.IOException;
import java.sql.SQLException;

public class PopulateContentletAsJSONUtilTest extends IntegrationTestBase {

@BeforeClass
public static void prepare() throws Exception {
//Setting web app environment
IntegrationTestInitService.getInstance().init();

if (!OSGIUtil.getInstance().isInitialized()) {
OSGIUtil.getInstance().initializeFramework();
}
}

@Test
public void Test_Run() throws SQLException, DotDataException, IOException {
PopulateContentletAsJSONUtil.getInstance().populate("Host");
public void Test_populate_host() throws SQLException, DotDataException, IOException {
PopulateContentletAsJSONUtil.getInstance().populateForAssetSubType("Host");
}

@Test
public void Test_populate_All_excluding_host() throws SQLException, DotDataException, IOException {
PopulateContentletAsJSONUtil.getInstance().populateExcludingAssetSubType("Host");
}

/**
* Remove the content type and workflows created
*/
@AfterClass
public static void cleanup() {

}

}
173 changes: 117 additions & 56 deletions dotCMS/src/main/java/com/dotcms/util/content/json/PopulateContentletAsJSONUtil.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.dotcms.util.content.json;

import com.dotcms.business.CloseDBIfOpened;
import com.dotcms.business.WrapInTransaction;
import com.dotcms.content.business.json.ContentletJsonAPI;
import com.dotcms.content.business.json.ContentletJsonHelper;
Expand All @@ -10,6 +9,7 @@
import com.dotmarketing.business.APILocator;
import com.dotmarketing.common.db.DotConnect;
import com.dotmarketing.db.DbConnectionFactory;
import com.dotmarketing.db.HibernateUtil;
import com.dotmarketing.exception.DotDataException;
import com.dotmarketing.exception.DotRuntimeException;
import com.dotmarketing.portlets.contentlet.model.Contentlet;
Expand Down Expand Up @@ -42,29 +42,37 @@ public class PopulateContentletAsJSONUtil {

private final ContentletJsonAPI contentletJsonAPI;

// Query to find all the contentlets that are Hosts and have a null contentlet_as_json
private final String SUBTYPE_WITH_NO_JSON = "select c.*\n" +
"from contentlet c\n" +
" JOIN identifier i ON i.id = c.identifier\n" +
" JOIN contentlet_version_info cv ON i.id = cv.identifier\n" +
" AND (c.inode = cv.working_inode OR c.inode = cv.live_inode)\n" +
"WHERE i.asset_subtype = '%s' AND c.contentlet_as_json IS NULL;";

// Query to find all the contentlets that are NOT Hosts and have a null contentlet_as_json
private final String CONTENTS_WITH_NO_JSON = "select c.*\n" +
"from contentlet c\n" +
" JOIN identifier i ON i.id = c.identifier\n" +
" JOIN contentlet_version_info cv ON i.id = cv.identifier\n" +
" AND (c.inode = cv.working_inode OR c.inode = cv.live_inode)\n" +
"WHERE i.asset_type <> 'contentlet' AND c.contentlet_as_json IS NULL;";
// Query to find all the contentlets for a given asset_subtype and have a null contentlet_as_json
private final String SUBTYPE_WITH_NO_JSON = "select c.*" +
"from contentlet c" +
" JOIN identifier i ON i.id = c.identifier" +
" JOIN contentlet_version_info cv ON i.id = cv.identifier" +
" AND (c.inode = cv.working_inode OR c.inode = cv.live_inode)" +
" WHERE i.asset_subtype = '%s' AND c.contentlet_as_json IS NULL;";

// Query to find all the contentlets that have a null contentlet_as_json
private final String CONTENTS_WITH_NO_JSON = "select c.*" +
"from contentlet c" +
" JOIN identifier i ON i.id = c.identifier" +
" JOIN contentlet_version_info cv ON i.id = cv.identifier" +
" AND (c.inode = cv.working_inode OR c.inode = cv.live_inode)" +
" WHERE i.asset_type = 'contentlet' AND c.contentlet_as_json IS NULL;";

// Query to find all the contentlets that are NOT of a given asset_subtype and have a null contentlet_as_json
private final String CONTENTS_WITH_NO_JSON_AND_EXCLUDE = "select c.*" +
"from contentlet c" +
" JOIN identifier i ON i.id = c.identifier" +
" JOIN contentlet_version_info cv ON i.id = cv.identifier" +
" AND (c.inode = cv.working_inode OR c.inode = cv.live_inode)" +
" WHERE i.asset_subtype <> '%s' AND i.asset_type = 'contentlet' AND c.contentlet_as_json IS NULL;";

// Query to update the contentlet_as_json column of the contentlet table
private final String UPDATE_CONTENTLET_AS_JSON = "UPDATE contentlet SET contentlet_as_json = ? WHERE inode = ?";

// Cursor related queries
private final String DECLARE_CURSOR = "DECLARE missingContentletAsJSONCursor CURSOR FOR %s";
private final String FETCH_CURSOR_POSTGRES = "FETCH FORWARD 100 FROM missingContentletAsJSONCursor";
private final String FETCH_CURSOR_MSSQL = "FETCH NEXT 100 FROM missingContentletAsJSONCursor";
private final String FETCH_CURSOR_POSTGRES = "FETCH FORWARD %s FROM missingContentletAsJSONCursor";
private final String FETCH_CURSOR_MSSQL = "FETCH NEXT %s FROM missingContentletAsJSONCursor";
private final String CLOSE_CURSOR = "CLOSE missingContentletAsJSONCursor";

private static class SingletonHolder {
Expand All @@ -91,47 +99,65 @@ private interface CheckedConsumer<T, E extends Exception> {

/**
* Finds all the contentlets that need to be updated with the contentlet_as_json column for a given
* optional assetSubtype (Content Type).
* assetSubtype (Content Type).
*
* @param assetSubtype Optional assetSubtype (Content Type) to filter the contentlets to process, if null then all
* @param assetSubtype Asset subtype (Content Type) to filter the contentlets to process, if null then all
* the contentlets will be processed.
* @throws SQLException
* @throws DotDataException
* @throws IOException
*/
@WrapInTransaction
public void populate(@Nullable String assetSubtype) throws SQLException, DotDataException, IOException {

final File populateJSONTaskDataFile = File.createTempFile("rows-task-230320", "tmp");

// First we need to find all the contentlets to process and write them into a file
findAndStoreToDisk(assetSubtype, populateJSONTaskDataFile);
public void populateForAssetSubType(String assetSubtype) throws SQLException, DotDataException, IOException {
populate(assetSubtype, null);
}

// Now we need to process the file and each record on it
processFile(populateJSONTaskDataFile);
/**
* Finds all the contentlets that need to be updated with the contentlet_as_json column excluding the contentles
* of a given assetSubtype (Content Type).
*
* @param assetSubtype Asset subtype (Content Type) use to exclude contentlets of that given type from the query.
* @throws SQLException
* @throws DotDataException
* @throws IOException
*/
public void populateExcludingAssetSubType(String assetSubtype) throws SQLException, DotDataException, IOException {
populate(null, assetSubtype);
}

/**
* This method processes a file that contains all the contentlets that need to be updated with the contentlet_as_json
* Finds all the contentlets that need to be updated with the contentlet_as_json column for the given
* assetSubtype and excludingAssetSubtype.
*
* @param taskDataFile
* @param assetSubtype Optional asset subtype (Content Type) to filter the contentlets to process, if null then all
* the contentlets will be processed unless the excludingAssetSubtype is provided.
* @param excludingAssetSubtype Optional asset subtype (Content Type) use to exclude contentlets from the query
* @throws SQLException
* @throws DotDataException
* @throws IOException
*/
@WrapInTransaction
private void processFile(final File taskDataFile) throws IOException {
private void populate(@Nullable String assetSubtype, @Nullable String excludingAssetSubtype) throws SQLException, DotDataException, IOException {

try (final Stream<String> streamLines = Files.lines(taskDataFile.toPath())) {
final File populateJSONTaskDataFile = File.createTempFile("rows-task-230320", "tmp");

streamLines.map(line -> {
try {
return parseLine(line);
} catch (Exception e) {
Logger.error("Error parsing line: " + line, e);
return null;
}
})// Map each line to Tuple (inode, contentlet_as_json)
.filter(Objects::nonNull)
.forEach(wrapCheckedConsumer(this::updateContentlet));// Update each contentlet in the DB
try {

HibernateUtil.startTransaction();

// First we need to find all the contentlets to process and write them into a file
findAndStoreToDisk(assetSubtype, excludingAssetSubtype, populateJSONTaskDataFile);

} finally {
HibernateUtil.closeSessionSilently();
}

try {

HibernateUtil.startTransaction();

// Now we need to process the file and each record on it
processFile(populateJSONTaskDataFile);
} finally {
HibernateUtil.closeSessionSilently();
}
}

Expand All @@ -146,8 +172,9 @@ private void processFile(final File taskDataFile) throws IOException {
* @throws DotDataException
* @throws IOException
*/
@CloseDBIfOpened
@WrapInTransaction
private void findAndStoreToDisk(@Nullable String assetSubtype,
@Nullable String excludingAssetSubtype,
final File populateJSONTaskDataFile) throws
SQLException, DotDataException, IOException {

Expand All @@ -156,34 +183,47 @@ private void findAndStoreToDisk(@Nullable String assetSubtype,
try (final Connection conn = DbConnectionFactory.getConnection();
var stmt = conn.createStatement()) {

// Declaring the cursor
if (Strings.isNullOrEmpty(assetSubtype)) {
stmt.execute(String.format(DECLARE_CURSOR, CONTENTS_WITH_NO_JSON));
if (Strings.isNullOrEmpty(excludingAssetSubtype)) {
stmt.execute(String.format(DECLARE_CURSOR, CONTENTS_WITH_NO_JSON));
} else {
var selectQuery = String.format(CONTENTS_WITH_NO_JSON_AND_EXCLUDE, excludingAssetSubtype);
stmt.execute(String.format(DECLARE_CURSOR, selectQuery));
}
} else {
var selectQuery = String.format(SUBTYPE_WITH_NO_JSON, assetSubtype);
stmt.execute(String.format(DECLARE_CURSOR, selectQuery));
}

boolean hasMoreRows = true;
boolean hasRows;

do {

hasRows = false;

// Fetching batches of 100 records
var batchSize = 100;
if (DbConnectionFactory.isMsSql()) {
stmt.execute(FETCH_CURSOR_MSSQL);
stmt.execute(String.format(FETCH_CURSOR_MSSQL, batchSize));
} else {
stmt.execute(FETCH_CURSOR_POSTGRES);
stmt.execute(String.format(FETCH_CURSOR_POSTGRES, batchSize));
}

try (ResultSet rs = stmt.getResultSet()) {

// Process the batch of rows
while (rs.next()) {

hasRows = true;

// Now we want to write the found Contentlets into a file for a later processing
var dotConnect = new DotConnect();
dotConnect.fromResultSet(rs);

var jsonDataArray = Optional.ofNullable(dotConnect.loadObjectResults())
var loadedResults = dotConnect.loadObjectResults();

var jsonDataArray = Optional.ofNullable(loadedResults)
.map(results ->
TransformerLocator.createContentletTransformer(results).asList()
)// Transform the results into a list of contentlets
Expand All @@ -200,15 +240,12 @@ private void findAndStoreToDisk(@Nullable String assetSubtype,
fileWriter.newLine();
}
}

// Check if there are more rows to fetch
hasMoreRows = rs.getRow() > 0;
}

// Flush the writer to the file
fileWriter.flush();
} while (hasRows);

} while (hasMoreRows);
// Flush the writer to the file
fileWriter.flush();

// Close the cursor
stmt.execute(CLOSE_CURSOR);
Expand All @@ -217,6 +254,30 @@ private void findAndStoreToDisk(@Nullable String assetSubtype,
}
}

/**
* This method processes a file that contains all the contentlets that need to be updated with the contentlet_as_json
*
* @param taskDataFile
* @throws IOException
*/
@WrapInTransaction
private void processFile(final File taskDataFile) throws IOException {

try (final Stream<String> streamLines = Files.lines(taskDataFile.toPath())) {

streamLines.map(line -> {
try {
return parseLine(line);
} catch (Exception e) {
Logger.error("Error parsing line: " + line, e);
return null;
}
})// Map each line to Tuple (inode, contentlet_as_json)
.filter(Objects::nonNull)
.forEach(wrapCheckedConsumer(this::updateContentlet));// Update each contentlet in the DB
}
}

/**
* Parses the given line and returns a tuple with the inode and the json representation of the contentlet.
*
Expand Down
4 changes: 1 addition & 3 deletions .../src/main/java/com/dotmarketing/startup/runonce/Task230320FixMissingContentletAsJSON.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.dotmarketing.startup.runonce;

import com.dotcms.business.WrapInTransaction;
import com.dotcms.util.content.json.PopulateContentletAsJSONUtil;
import com.dotmarketing.exception.DotDataException;
import com.dotmarketing.exception.DotRuntimeException;
Expand All @@ -20,7 +19,6 @@ public boolean forceRun() {
}

@Override
@WrapInTransaction
public void executeUpgrade() throws DotDataException, DotRuntimeException {

Logger.info(this, "Running upgrade Task230320FixMissingContentletAsJSON");
Expand All @@ -29,7 +27,7 @@ public void executeUpgrade() throws DotDataException, DotRuntimeException {
stopWatch.start();

try {
PopulateContentletAsJSONUtil.getInstance().populate("Host");
PopulateContentletAsJSONUtil.getInstance().populateForAssetSubType("Host");
} catch (SQLException e) {
throw new DotDataException(e.getMessage(), e);
} catch (IOException e) {
Expand Down

0 comments on commit 5aa3a75

Please sign in to comment.