#24093 include in 23.01.2 second attempt

dotCMS/src/integration-test/java/com/dotcms/MainSuite.java

@@ -161,6 +161,7 @@
 import com.dotmarketing.quartz.DotStatefulJobTest;
 import com.dotmarketing.quartz.job.CleanUpFieldReferencesJobTest;
 import com.dotmarketing.quartz.job.IntegrityDataGenerationJobTest;
+import com.dotmarketing.quartz.job.PopulateContentletAsJSONJobTest;
 import com.dotmarketing.startup.StartupTasksExecutorTest;
 import com.dotmarketing.startup.runalways.Task00050LoadAppsSecretsTest;
 import com.dotmarketing.startup.runonce.Task05195CreatesDestroyActionAndAssignDestroyDefaultActionsToTheSystemWorkflowTest;
@@ -651,8 +652,9 @@
         VisitorsCurrentURLConditionletTest.class,
         VisitorsGeolocationConditionletTest.class,
         ManifestUtilTest.class,
-        ZipUtilTest.class
-//        PopulateContentletAsJSONUtilTest.class
+        ZipUtilTest.class,
+        PopulateContentletAsJSONUtilTest.class,
+        PopulateContentletAsJSONJobTest.class
 
 })
 public class MainSuite {

dotCMS/src/integration-test/java/com/dotmarketing/quartz/job/PopulateContentletAsJSONJobTest.java

@@ -0,0 +1,65 @@
+package com.dotmarketing.quartz.job;
+
+import com.dotcms.IntegrationTestBase;
+import com.dotcms.util.IntegrationTestInitService;
+import com.dotmarketing.init.DotInitScheduler;
+import com.dotmarketing.quartz.QuartzUtils;
+import io.vavr.control.Try;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.quartz.SchedulerException;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class PopulateContentletAsJSONJobTest extends IntegrationTestBase {
+
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        IntegrationTestInitService.getInstance().init();
+        DotInitScheduler.start();
+        deleteJob();
+    }
+
+    @AfterClass
+    public static void afterClass() throws Exception {
+        deleteJob();
+    }
+
+    /**
+     * <b>Method to test:</b> {@link PopulateContentletAsJSONJob#fireJob(String)}
+     * <p>
+     * <b>Given sceneario:</b> This test will register a {@link PopulateContentletAsJSONJob} job and make sure it was
+     * scheduled.
+     * <p>
+     * <b>Expected result:</b> No errors should be thrown and the job should be scheduled.
+     */
+    @Test
+    public void test_fireJob() {
+
+        final var jobName = PopulateContentletAsJSONJob.getJobName();
+        final var groupName = PopulateContentletAsJSONJob.getJobGroupName();
+
+        try {
+            // Make sure the job can be schedule without errors
+            PopulateContentletAsJSONJob.fireJob("Host");
+        } catch (Exception e) {
+            fail("Unable to fire job: " + e.getMessage());
+        }
+
+        // Make sure the job was scheduled
+        final var scheduler = QuartzUtils.getScheduler();
+
+        // Checking the job was created
+        var jobDetail = Try.of(() -> scheduler.getJobDetail(jobName, groupName))
+                .onFailure(e -> {
+                    fail(String.format("Error retrieving job detail [%s, %s]: %s", jobName, groupName, e.getMessage()));
+                }).getOrNull();
+        assertNotNull(jobDetail);
+    }
+
+    private static void deleteJob() throws SchedulerException {
+        PopulateContentletAsJSONJob.removeJob();
+    }
+}

dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/transform/ContentletTransformer.java

@@ -106,9 +106,7 @@ private static Contentlet transform(final Map<String, Object> map)  {
         contentlet.setModDate((Date) map.get("mod_date"));
         contentlet.setModUser((String) map.get("mod_user"));
         contentlet.setOwner((String) map.get("owner"));
-        contentlet.setProperty(Contentlet.TITTLE_KEY, map.get(Contentlet.TITTLE_KEY));
         contentlet.setSortOrder(ConversionUtils.toInt(map.get("sort_order"),0));
-
         contentlet.setLanguageId(ConversionUtils.toLong(map.get("language_id"), 0L));
 
         try {
@@ -247,11 +245,15 @@ private static void populateWysiwyg(final Map<String, Object> map, Contentlet co
      */
     private static void populateFields(final Contentlet contentlet, final Map<String, Object> originalMap)
             throws DotDataException, DotSecurityException {
+
         final Map<String, Object> fieldsMap = new HashMap<>();
         final String inode = (String) originalMap.get(INODE);
         final String identifier = (String) originalMap.get(IDENTIFIER);
         final String contentTypeId = (String) originalMap.get(STRUCTURE_INODE);
 
+        // Populate the title
+        contentlet.setProperty(Contentlet.TITTLE_KEY, originalMap.get(Contentlet.TITTLE_KEY));
+
         final ContentType contentType = APILocator.getContentTypeAPI(APILocator.systemUser())
                 .find(contentTypeId);
         final List<Field> fields = new LegacyFieldTransformer(contentType.fields())
@@ -278,6 +280,11 @@ private static void populateFields(final Contentlet contentlet, final Map<String
                             && FileAssetAPI.FILE_NAME_FIELD
                             .equals(field.getVelocityVarName())) {
                         value = APILocator.getIdentifierAPI().find(identifier).getAssetName();
+                    } else if (UtilMethods.isSet(identifier)
+                            && contentType instanceof FileAssetContentType
+                            && FileAssetAPI.META_DATA_FIELD.equals(field.getVelocityVarName())) {
+                        // We can ignore this metadata field, metadata will be generated directly from the asset
+                        value = Collections.emptyMap();
                     } else {
                         if (LegacyFieldTypes.BINARY.legacyValue()
                                 .equals(field.getFieldType())) {

dotCMS/src/main/java/com/dotmarketing/quartz/job/PopulateContentletAsJSONJob.java

@@ -5,11 +5,14 @@
 import com.dotmarketing.exception.DotDataException;
 import com.dotmarketing.exception.DotRuntimeException;
 import com.dotmarketing.quartz.DotStatefulJob;
+import com.dotmarketing.quartz.QuartzUtils;
+import com.dotmarketing.util.Config;
 import com.dotmarketing.util.Logger;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
-import org.quartz.Trigger;
+import io.vavr.Lazy;
+import io.vavr.control.Try;
+import org.quartz.*;
 
 import java.io.IOException;
 import java.io.Serializable;
@@ -21,26 +24,35 @@
  */
 public class PopulateContentletAsJSONJob extends DotStatefulJob {
 
-    public static final String EXCLUDING_ASSET_SUB_TYPE = "excludingAssetSubType";
+    private static final String EXCLUDING_ASSET_SUB_TYPE = "excludingAssetSubType";
+    private static final String CONFIG_PROPERTY_HOURS_INTERVAL = "populateContentletAsJSONJob.hours.interval";
+    private static final Lazy<Integer> HOURS_INTERVAL = Lazy.of(() -> Config.getIntProperty(
+            CONFIG_PROPERTY_HOURS_INTERVAL, 4));
 
     @Override
     public void run(JobExecutionContext jobContext) throws JobExecutionException {
 
-        final Trigger trigger = jobContext.getTrigger();
-        final Map<String, Serializable> executionData = getExecutionData(trigger, PopulateContentletAsJSONJob.class);
+        final var jobDataMap = jobContext.getJobDetail().getJobDataMap();
 
         final String excludingAssetSubType;
-        if (executionData.containsKey(EXCLUDING_ASSET_SUB_TYPE)) {
-            excludingAssetSubType = (String) executionData.get(EXCLUDING_ASSET_SUB_TYPE);
+        if (jobDataMap.containsKey(EXCLUDING_ASSET_SUB_TYPE)) {
+            excludingAssetSubType = (String) jobDataMap.get(EXCLUDING_ASSET_SUB_TYPE);
         } else {
             excludingAssetSubType = null;
         }
 
         try {
+            // Executing the populate contentlet as JSON logic
             new PopulateContentletAsJSONUtil().populateExcludingAssetSubType(excludingAssetSubType);
+            // Removing the job if everything went well
+            removeJob();
         } catch (SQLException | DotDataException | IOException e) {
             Logger.error(this, "Error executing Contentlet as JSON population job", e);
             throw new DotRuntimeException(e);
+        } catch (SchedulerException e) {
+            Logger.error(this, String.format("Unable to remove [%s] job",
+                    PopulateContentletAsJSONJob.class.getName()), e);
+            throw new DotRuntimeException(e);
         }
     }
 
@@ -49,14 +61,69 @@ public void run(JobExecutionContext jobContext) throws JobExecutionException {
      */
     public static void fireJob(final String excludingAssetSubType) {
 
-        final ImmutableMap<String, Serializable> nextExecutionData = ImmutableMap
-                .of(EXCLUDING_ASSET_SUB_TYPE, excludingAssetSubType);
+        final var jobName = getJobName();
+        final var groupName = getJobGroupName();
+
+        final var scheduler = QuartzUtils.getScheduler();
+
+        // Checking if the job already exists
+        var jobDetail = Try.of(() -> scheduler.getJobDetail(jobName, groupName))
+                .onFailure(e -> {
+                    Logger.error(PopulateContentletAsJSONJob.class,
+                            String.format("Error retrieving job detail [%s, %s]", jobName, groupName), e);
+                }).getOrNull();
+
+        if (jobDetail != null) {
+            Logger.info(PopulateContentletAsJSONJob.class,
+                    String.format("Job [%s, %s] already exists, skipping creation", jobName, groupName));
+            return;
+        }
+
+        // Creating the job
+        final var jobDataMap = new JobDataMap();
+        jobDataMap.put(EXCLUDING_ASSET_SUB_TYPE, excludingAssetSubType);
+
+        jobDetail = new JobDetail(
+                jobName, groupName, PopulateContentletAsJSONJob.class
+        );
+
+        jobDetail.setJobDataMap(jobDataMap);
+        jobDetail.setDurability(false);
+        jobDetail.setVolatility(false);
+        jobDetail.setRequestsRecovery(true);
+
+        // This trigger will fire the job every 4 hours
+        final var trigger = TriggerUtils.makeHourlyTrigger(HOURS_INTERVAL.get());
+        trigger.setName(jobName);
+        trigger.setGroup(groupName);
+        trigger.setJobName(jobName);
+        trigger.setJobGroup(groupName);
+        trigger.setMisfireInstruction(SimpleTrigger.MISFIRE_INSTRUCTION_FIRE_NOW);
+
         try {
-            DotStatefulJob.enqueueTrigger(nextExecutionData, PopulateContentletAsJSONJob.class);
+            scheduler.scheduleJob(jobDetail, trigger);
         } catch (Exception e) {
-            Logger.error(HostAssetsJobProxy.class, "Error scheduling populate content as JSON job", e);
+            Logger.error(PopulateContentletAsJSONJob.class, "Error scheduling populate content as JSON job", e);
             throw new DotRuntimeException("Error scheduling populate content as JSON job", e);
         }
     }
 
+    /**
+     * Removes the PopulateContentletAsJSONJob from the scheduler
+     */
+    @VisibleForTesting
+    static void removeJob() throws SchedulerException {
+        QuartzUtils.removeJob(getJobName(), getJobGroupName());
+    }
+
+    @VisibleForTesting
+    static String getJobName() {
+        return PopulateContentletAsJSONJob.class.getSimpleName();
+    }
+
+    @VisibleForTesting
+    static String getJobGroupName() {
+        return getJobName() + "_Group";
+    }
+
 }

dotCMS/src/main/java/com/dotmarketing/util/TaskLocatorUtil.java

@@ -366,7 +366,7 @@ public static List<Class<?>> getStartupRunAlwaysTaskClasses() {
 		//UT that were backported to LTS
 	public static List<Class<?>> getBackportedUpgradeTaskClasses() {
 		final List<Class<?>> ret = new ArrayList<Class<?>>();
-//		ret.add(Task230320FixMissingContentletAsJSON.class);
+		ret.add(Task230320FixMissingContentletAsJSON.class);
 		return ret.stream().sorted(classNameComparator).collect(Collectors.toList());
 	}