IntegrationBundle bug fixes (#13454)

* Merge pull request #1213 from acquia/MAUT-5072 Segments are very slow to build * Adding the index also to the entity * Merge pull request #889 from mautic-inc/MAUT-3552 Impossible for a marketer to know why an object did not sync due to validation issues * Merge pull request #1000 from mautic-inc/MAUT-4181 MAUT-4181: IntegrationsBundle: don't sync other object types if IDs for particular objects are specified * Fixes after rebase to M5 * Merge pull request #1060 from mautic-inc/MAUT-4362 MAUT-4362: Scoring - SFDC clearing out new data from ACS * Merge pull request #1093 from mautic-inc/MAUT-4640 MAUT-4640: The field change log queue is not processed entirely during a single integration sync * Chages needed after rebase to M5 * Merge pull request #1486 from acquia/MAUT-6230-syncing-sf MAUT-6230 / Removed last sync date time from ObjectDAO and Integratio… * Merge pull request #1706 from acquia/MAUT-6264 MAUT-6264 "Push to Salesforce" Campaign Action showing wrong count * Merge pull request #1758 from acquia/MAUT-7063-Salesforce-Integration-referenceValueDAO-is-causing-sync-issues MAUT-7063: salesforce integration reference value dao is causing sync issues * Merge pull request #1802 from acquia/MAUT-7700 MAUT-7700 - Fixing incorrect timeline URL for salesforce * Fix new line issue * Fixes after rebase to M5 * Merge pull request #1968 from acquia/MAUT-5271 MAUT-5271: Plugin Credentials Change Bug Fix * Changes needed after rebase to M5 --------- Co-authored-by: Anton Vlasenko <43744263+anton-vlasenko@users.noreply.github.com> Co-authored-by: Alan Hartless <alan@devkardia.com> Co-authored-by: lukassykora <lukas.sykora@acquia.com> Co-authored-by: Rohit Pavaskar <66303837+rohitp19@users.noreply.github.com> Co-authored-by: Robert Parker <diaboloshogunate@gmail.com> Co-authored-by: Miroslav Fedeleš <miroslav.fedeles@gmail.com> Co-authored-by: Saurabh Gupta <saurabh.gupta@acquia.com> Co-authored-by: Saurabh Gupta <48244990+dadarya0@users.noreply.github.com>
mautic · Feb 26, 2024 · 8e5b815 · 8e5b815
1 parent cb5b3c7
commit 8e5b815
Show file tree

Hide file tree

Showing 34 changed files with 856 additions and 92 deletions.
diff --git a/app/bundles/CampaignBundle/Entity/FailedLeadEventLogRepository.php b/app/bundles/CampaignBundle/Entity/FailedLeadEventLogRepository.php
@@ -2,11 +2,28 @@
 
 namespace Mautic\CampaignBundle\Entity;
 
+use Doctrine\DBAL\ArrayParameterType;
 use Mautic\CoreBundle\Entity\CommonRepository;
 
 /**
  * @extends CommonRepository<FailedLeadEventLog>
  */
 class FailedLeadEventLogRepository extends CommonRepository
 {
+    /**
+     * @param array<string|int> $ids
+     */
+    public function deleteByIds(array $ids): void
+    {
+        if (!$ids) {
+            return;
+        }
+
+        $this->_em->getConnection()
+            ->createQueryBuilder()
+            ->delete(MAUTIC_TABLE_PREFIX.'campaign_lead_event_failed_log')
+            ->where('log_id IN (:ids)')
+            ->setParameter('ids', $ids, ArrayParameterType::STRING)
+            ->executeStatement();
+    }
 }
diff --git a/app/bundles/CampaignBundle/EventListener/CampaignEventLogCleanupSubscriber.php b/app/bundles/CampaignBundle/EventListener/CampaignEventLogCleanupSubscriber.php
@@ -0,0 +1,41 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mautic\CampaignBundle\EventListener;
+
+use Mautic\CampaignBundle\CampaignEvents;
+use Mautic\CampaignBundle\Entity\FailedLeadEventLogRepository;
+use Mautic\CampaignBundle\Entity\LeadEventLog;
+use Mautic\CampaignBundle\Event\ExecutedBatchEvent;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+
+final class CampaignEventLogCleanupSubscriber implements EventSubscriberInterface
+{
+    public function __construct(private FailedLeadEventLogRepository $failedLeadEventLogRepository)
+    {
+    }
+
+    public static function getSubscribedEvents(): array
+    {
+        return [
+            CampaignEvents::ON_EVENT_EXECUTED_BATCH => ['onEventBatchExecuted', -100],
+        ];
+    }
+
+    /**
+     * Deletes failed log entries for all successful event logs.
+     */
+    public function onEventBatchExecuted(ExecutedBatchEvent $event): void
+    {
+        $ids = $event->getExecuted()
+            ->map(fn (LeadEventLog $eventLog) => $eventLog->getId())
+            ->getValues();
+
+        if (!$ids) {
+            return;
+        }
+
+        $this->failedLeadEventLogRepository->deleteByIds($ids);
+    }
+}
diff --git a/app/bundles/CoreBundle/Tests/Unit/Doctrine/ArrayTypeTest.php b/app/bundles/CoreBundle/Tests/Unit/Doctrine/ArrayTypeTest.php
@@ -102,6 +102,27 @@ public function testGivenStdClassWhenConvertsToDatabaseValueThenGetEncodedData()
         );
     }
 
+    public function testGivenObjectWithPrivatePropertyWhenConvertsToDatabaseValue(): void
+    {
+        $value = [
+            'fields' => [
+                'field_account_executive_o' => [
+                    null,
+                    new \Mautic\IntegrationsBundle\Sync\DAO\Value\ReferenceValueDAO(),
+                ],
+            ],
+            'dateModified' => [
+                '2022-05-02T21:39:27+00:00',
+                '2022-05-03T14:22:33+00:00',
+            ],
+        ];
+
+        $serialized   = $this->arrayType->convertToDatabaseValue($value, $this->platform);
+        $unserialized = $this->arrayType->convertToPHPValue($serialized, $this->platform);
+
+        $this->assertEquals($value, $unserialized);
+    }
+
     public function testGivenObjectWithPrivatePropertyWhenConvertsToPHPValueThenGetsArrayWithoutObject(): void
     {
         $array = [

diff --git a/app/bundles/FormBundle/EventListener/FormSubscriber.php b/app/bundles/FormBundle/EventListener/FormSubscriber.php
@@ -217,7 +217,7 @@ public function onFormSubmitActionRepost(Events\SubmissionEvent $event): void
             $matchedFields[$key] = $field['alias'];
 
             // decode html chars and quotes before posting to next form
-            $payload[$key]       = htmlspecialchars_decode($value, ENT_QUOTES);
+            $payload[$key]       = html_entity_decode(htmlspecialchars_decode($value, ENT_QUOTES), ENT_QUOTES);
         }
 
         $event->setPostSubmitPayload($payload);

diff --git a/app/bundles/FormBundle/Tests/EventListener/FormSubscriberTest.php b/app/bundles/FormBundle/Tests/EventListener/FormSubscriberTest.php
@@ -54,17 +54,18 @@ protected function setUp(): void
     public function testOnFormSubmitActionRepost(): void
     {
         $postData = [
-            'first_name' => "Test's Name",
-            'notes'      => 'A & B < dy >',
+            'first_name' => "Test's Name un être> and être",
+            'notes'      => 'A & B < dy >
+New line',
             'formId'     => '1',
             'return'     => '',
             'formName'   => 'form190122',
             'messenger'  => '1',
         ];
 
         $resultData = [
-            'first_name' => 'Test&#39;s Name',
-            'notes'      => 'A &#38; B &#60; dy &#62;',
+            'first_name' => 'Test&#39;s Name un &ecirc;tre&gt; and être',
+            'notes'      => 'A &#38; B &#60; dy &#62;&#10;New line',
         ];
 
         $request         = new Request();

diff --git a/app/bundles/IntegrationsBundle/Config/config.php b/app/bundles/IntegrationsBundle/Config/config.php
@@ -212,6 +212,7 @@
                     'mautic.integrations.sync.data_exchange.mautic.full_object_report_builder',
                     'mautic.integrations.sync.data_exchange.mautic.partial_object_report_builder',
                     'mautic.integrations.sync.data_exchange.mautic.order_executioner',
+                    'mautic.integrations.helper.sync_date',
                 ],
             ],
             'mautic.integrations.sync.integration_process.object_change_generator' => [
@@ -287,6 +288,7 @@
                     'mautic.integrations.sync.notification.handler_container',
                     'mautic.integrations.helper.sync_integrations',
                     'mautic.integrations.helper.config_integrations',
+                    'translator',
                 ],
             ],
             'mautic.integrations.sync.notification.writer' => [

diff --git a/app/bundles/IntegrationsBundle/Controller/ConfigController.php b/app/bundles/IntegrationsBundle/Controller/ConfigController.php
@@ -9,6 +9,7 @@
 use Mautic\IntegrationsBundle\Event\ConfigAuthUrlEvent;
 use Mautic\IntegrationsBundle\Event\ConfigSaveEvent;
 use Mautic\IntegrationsBundle\Event\FormLoadEvent;
+use Mautic\IntegrationsBundle\Event\KeysSaveEvent;
 use Mautic\IntegrationsBundle\Exception\IntegrationNotFoundException;
 use Mautic\IntegrationsBundle\Form\Type\IntegrationConfigType;
 use Mautic\IntegrationsBundle\Helper\ConfigIntegrationsHelper;
@@ -105,11 +106,16 @@ private function submitForm(
         }
 
         // Get the fields before the form binds partial data due to pagination
-        $settings      = $this->integrationConfiguration->getFeatureSettings();
-        $fieldMappings = $settings['sync']['fieldMappings'] ?? [];
+        $settings          = $this->integrationConfiguration->getFeatureSettings();
+        $fieldMappings     = $settings['sync']['fieldMappings'] ?? [];
+        $oldApiKeys        = $this->integrationConfiguration->getApiKeys();
 
         // Submit the form
         $form->handleRequest($request);
+
+        $configEvent = new KeysSaveEvent($this->integrationConfiguration, $oldApiKeys);
+        $this->dispatcher->dispatch($configEvent, IntegrationEvents::INTEGRATION_API_KEYS_BEFORE_SAVE);
+
         if ($this->integrationObject instanceof ConfigFormSyncInterface) {
             $integration   = $this->integrationObject->getName();
             $settings      = $this->integrationConfiguration->getFeatureSettings();

diff --git a/app/bundles/IntegrationsBundle/Entity/FieldChangeRepository.php b/app/bundles/IntegrationsBundle/Entity/FieldChangeRepository.php
@@ -39,7 +39,7 @@ public function deleteEntitiesForObjectByColumnName(int $objectId, string $objec
     /**
      * Takes an object id & type and deletes all entities that match.
      */
-    public function deleteEntitiesForObject(int $objectId, string $objectType, ?string $integration = null): void
+    public function deleteEntitiesForObject(int $objectId, string $objectType, ?string $integration = null, \DateTimeInterface $toDateTime = null): void
     {
         $qb = $this->getEntityManager()->getConnection()->createQueryBuilder();
 
@@ -51,8 +51,13 @@ public function deleteEntitiesForObject(int $objectId, string $objectType, ?stri
             $qb->setParameter('integration', $integration);
         }
 
+        if (null !== $toDateTime) {
+            $expr = $expr->with($qb->expr()->lte('modified_at', ':toDateTime'));
+            $qb->setParameter('toDateTime', $toDateTime->format('Y-m-d H:i:s'));
+        }
+
         $qb->setParameter('objectType', $objectType)
-            ->setParameter('objectId', (int) $objectId);
+            ->setParameter('objectId', $objectId);
 
         $qb
             ->delete(MAUTIC_TABLE_PREFIX.'sync_object_field_change_report')
@@ -116,7 +121,9 @@ public function findChangesBefore(string $integration, string $objectType, \Date
             )
             ->setParameter('integration', $integration)
             ->setParameter('objectType', $objectType)
-            ->orderBy('f.modified_at'); // Newer updated fields must override older updated fields
+            // 1. We must sort by f.object_id. Otherwise values stored in PartialObjectReportBuilder::lastProcessedTrackedId will be incorrect.
+            // 2. Newer updated fields must override older updated fields
+            ->orderBy('f.object_id, f.modified_at', 'ASC');
 
         return $qb->executeQuery()->fetchAllAssociative();
     }

diff --git a/app/bundles/IntegrationsBundle/Entity/ObjectMapping.php b/app/bundles/IntegrationsBundle/Entity/ObjectMapping.php
@@ -68,6 +68,7 @@ public static function loadMetadata(ORM\ClassMetadata $metadata): void
             ->setCustomRepositoryClass(ObjectMappingRepository::class)
             ->addIndex(['integration', 'integration_object_name', 'integration_object_id', 'integration_reference_id'], 'integration_object')
             ->addIndex(['integration', 'integration_object_name', 'integration_reference_id', 'integration_object_id'], 'integration_reference')
+            ->addIndex(['integration', 'internal_object_name', 'last_sync_date'], 'integration_integration_object_name_last_sync_date')
             ->addIndex(['integration', 'last_sync_date'], 'integration_last_sync_date');
 
         $builder->addId();

diff --git a/app/bundles/IntegrationsBundle/Event/KeysSaveEvent.php b/app/bundles/IntegrationsBundle/Event/KeysSaveEvent.php
@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mautic\IntegrationsBundle\Event;
+
+use Mautic\PluginBundle\Entity\Integration;
+use Symfony\Contracts\EventDispatcher\Event;
+
+final class KeysSaveEvent extends Event
+{
+    /**
+     * @var array<string,string>
+     */
+    private array $newKeys;
+
+    /**
+     * @param array<string,string> $oldKeys
+     */
+    public function __construct(private Integration $integrationConfiguration, private array $oldKeys)
+    {
+        $this->newKeys = $integrationConfiguration->getApiKeys();
+    }
+
+    public function getIntegrationConfiguration(): Integration
+    {
+        return $this->integrationConfiguration;
+    }
+
+    /**
+     * @return array<string,string>
+     */
+    public function getOldKeys(): array
+    {
+        return $this->oldKeys;
+    }
+
+    /**
+     * @return array<string,string>
+     */
+    public function getNewKeys(): array
+    {
+        return $this->newKeys;
+    }
+}
diff --git a/app/bundles/IntegrationsBundle/IntegrationEvents.php b/app/bundles/IntegrationsBundle/IntegrationEvents.php
@@ -205,4 +205,13 @@ final class IntegrationEvents
      * @var string
      */
     public const INTEGRATION_BATCH_SYNC_COMPLETED_MAUTIC_TO_INTEGRATION = 'mautic.integration.INTEGRATION_BATCH_SYNC_COMPLETED_MAUTIC_TO_INTEGRATION';
+
+    /**
+     * This event is dispatched when api keys is updated/inserted.
+     *
+     * The event listener receives a Mautic\IntegrationsBundle\Event\KeysSaveEvent instance.
+     *
+     * @var string
+     */
+    public const INTEGRATION_API_KEYS_BEFORE_SAVE = 'mautic.integration.INTEGRATION_API_KEYS_BEFORE_SAVE';
 }
diff --git a/app/bundles/IntegrationsBundle/Sync/DAO/Sync/ObjectIdsDAO.php b/app/bundles/IntegrationsBundle/Sync/DAO/Sync/ObjectIdsDAO.php
@@ -69,4 +69,12 @@ public function getObjectIdsFor(string $objectType): array
 
         return $this->objects[$objectType];
     }
+
+    /**
+     * @return string[]
+     */
+    public function getObjectTypes(): array
+    {
+        return array_keys($this->objects);
+    }
 }
diff --git a/app/bundles/IntegrationsBundle/Sync/DAO/Sync/Request/ObjectDAO.php b/app/bundles/IntegrationsBundle/Sync/DAO/Sync/Request/ObjectDAO.php
@@ -27,6 +27,9 @@ public function __construct(
          * Date/Time the sync started.
          */
         private ?\DateTimeInterface $toDateTime = null,
+        /**
+         * @deprecated Not used. To be removed in Mautic 6. Use SyncDateHelper instead
+         */
         private ?\DateTimeInterface $objectLastSyncDateTime = null
     ) {
     }
@@ -77,6 +80,9 @@ public function getToDateTime(): ?\DateTimeInterface
         return $this->toDateTime;
     }
 
+    /**
+     * @deprecated Not used. To be removed in Mautic 6. Use SyncDateHelper instead
+     */
     public function getObjectLastSyncDateTime(): ?\DateTimeInterface
     {
         return $this->objectLastSyncDateTime;

diff --git a/app/bundles/IntegrationsBundle/Sync/DAO/Value/ReferenceValueDAO.php b/app/bundles/IntegrationsBundle/Sync/DAO/Value/ReferenceValueDAO.php
@@ -34,4 +34,20 @@ public function __toString(): string
     {
         return (string) $this->value;
     }
+
+    /** @return array<string, mixed> */
+    public function __serialize(): array
+    {
+        return [
+            'value' => $this->value,
+            'types' => $this->type,
+        ];
+    }
+
+    /** @param array<string, mixed> $data */
+    public function __unserialize(array $data): void
+    {
+        $this->value = $data['value'] ?? null;
+        $this->type  = $data['type'] ?? null;
+    }
 }
diff --git a/app/bundles/IntegrationsBundle/Sync/Helper/SyncDateHelper.php b/app/bundles/IntegrationsBundle/Sync/Helper/SyncDateHelper.php
@@ -20,6 +20,8 @@ class SyncDateHelper
      */
     private array $lastObjectSyncDates = [];
 
+    private ?\DateTimeInterface $internalSyncStartDateTime = null;
+
     public function __construct(
         private Connection $connection
     ) {
@@ -59,7 +61,7 @@ public function getSyncFromDateTime(string $integration, string $object): \DateT
         return $this->lastObjectSyncDates[$key];
     }
 
-    public function getSyncToDateTime(): \DateTimeInterface
+    public function getSyncToDateTime(): ?\DateTimeInterface
     {
         if ($this->syncToDateTime) {
             return $this->syncToDateTime;
@@ -110,4 +112,39 @@ public function getLastSyncDateForObject(string $integration, string $object): ?
 
         return $lastSync;
     }
+
+    public function getInternalSyncStartDateTime(): ?\DateTimeInterface
+    {
+        return $this->internalSyncStartDateTime;
+    }
+
+    public function setInternalSyncStartDateTime(): void
+    {
+        if ($this->internalSyncStartDateTime) {
+            return;
+        }
+
+        $this->internalSyncStartDateTime = $this->calculateInternalSyncStartDateTime();
+    }
+
+    private function calculateInternalSyncStartDateTime(): \DateTimeInterface
+    {
+        $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC'));
+        // If there is no syncToDateTime value use "now"
+        if (!$this->getSyncToDateTime()) {
+            return $now;
+        }
+
+        // Clone it so that we don't modify the initial object
+        $syncToDateTime = clone $this->getSyncToDateTime();
+
+        // We should compare in UTC timezone
+        if (method_exists($syncToDateTime, 'setTimezone')) {
+            $syncToDateTime->setTimezone(new \DateTimeZone('UTC'));
+        }
+
+        // If syncToDate is less than now then use syncToDate, because otherwise we may delete
+        // changes that aren't supposed to be deleted from the sync_object_field_change_report table
+        return min($now, $syncToDateTime);
+    }
 }