Merge remote-tracking branch 'upstream/staging' into fix-url-validation

.htaccess

@@ -96,3 +96,22 @@
         </IfModule>
     </IfModule>
 </IfModule>
+
+# Denie access via HTTP requests to all PHP files.
+<Files "*.php">
+    Require all denied
+</Files>
+
+# Except those whitelisted bellow.
+<Files "index.php">
+    Require all granted
+</Files>
+<Files "index_dev.php">
+    Require all granted
+</Files>
+<Files "filemanager.php">
+    Require all granted
+</Files>
+<Files "upgrade.php">
+    Require all granted
+</Files>

.travis.yml

@@ -1,3 +1,4 @@
+dist: trusty
 language: php
 
 services:

README.md

@@ -44,9 +44,9 @@ Installing from source is only recommended if you are comfortable using the comm
 
 ## Requirements
 
-#### Contributors Agreement 
+#### Contributor Agreement 
 
-By contributing to this project, you accept and agree to the [Contributors Agreement](https://www.mautic.org/contributors-agreement) in its entirety.
+By contributing to this project, you accept and agree to the [Contributor Agreement](https://www.mautic.org/contributor-agreement) in its entirety.
 
 #### Development / Build process requirements
 
@@ -64,7 +64,7 @@ By contributing to this project, you accept and agree to the [Contributors Agree
 	- recommended: `openssl`, `opcache` / `apcu` / `memcached`
 	- recommended for development: `xdebug`
 3. Recommended memory limit: minimally 256 MB for testing, 512 MB and more for production.
-4. Disabling `ONLY_FULL_GROUP_BY` on the mySQL server.
+4. Recommended MySQL defaults can be set by running the queries `SET GLOBAL innodb_default_row_format=DYNAMIC; SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));`
 
 ## Installation
 

app/AppKernel.php

@@ -34,14 +34,14 @@ class AppKernel extends Kernel
      *
      * @const integer
      */
-    const MINOR_VERSION = 15;
+    const MINOR_VERSION = 16;
 
     /**
      * Patch version number.
      *
      * @const integer
      */
-    const PATCH_VERSION = 2;
+    const PATCH_VERSION = 0;
 
     /**
      * Extra version identifier.
@@ -83,7 +83,7 @@ public function __construct($environment, $debug)
     public function handle(Request $request, $type = HttpKernelInterface::MASTER_REQUEST, $catch = true)
     {
         if (strpos($request->getRequestUri(), 'installer') !== false || !$this->isInstalled()) {
-            define('MAUTIC_INSTALLER', 1);
+            defined('MAUTIC_INSTALLER') or define('MAUTIC_INSTALLER', 1);
         }
 
         if (defined('MAUTIC_INSTALLER')) {

app/bundles/CampaignBundle/Assets/js/campaign.js

@@ -48,7 +48,7 @@ Mautic.campaignOnLoad = function (container, response) {
             var thisSelect = mQuery(event.target).attr('id');
             Mautic.campaignBuilderUpdateEventListTooltips(thisSelect, false);
 
-            mQuery('#'+thisSelect+'_chosen .chosen-search input').on('keydown.toolip', function () {
+            mQuery('#'+thisSelect+'_chosen .chosen-search input').on('keydown.tooltip', function () {
                 // Destroy tooltips that are filtered out
                 Mautic.campaignBuilderUpdateEventListTooltips(thisSelect, true);
             }).on('keyup.tooltip', function() {
@@ -64,7 +64,7 @@ Mautic.campaignOnLoad = function (container, response) {
             var thisSelect = mQuery(event.target).attr('id');
             Mautic.campaignBuilderUpdateEventListTooltips(thisSelect, true);
 
-            mQuery('#'+thisSelect+'_chosen .chosen-search input').off('keyup.toolip')
+            mQuery('#'+thisSelect+'_chosen .chosen-search input').off('keyup.tooltip')
                 .off('keydown.tooltip');
         });
 

app/bundles/CampaignBundle/Entity/Campaign.php

@@ -585,7 +585,7 @@ public function getForms()
      */
     public function addForm(Form $form)
     {
-        $this->forms[] = $form;
+        $this->forms[$form->getId()] = $form;
 
         $this->changes['forms']['added'][$form->getId()] = $form->getName();
 

app/bundles/CampaignBundle/Entity/LeadRepository.php

@@ -211,7 +211,12 @@ public function getInactiveContacts($campaignId, $decisionId, $parentDecisionId,
         $q = $this->getSlaveConnection($limiter)->createQueryBuilder();
         $q->select('l.lead_id, l.date_added')
             ->from(MAUTIC_TABLE_PREFIX.'campaign_leads', 'l')
-            ->where($q->expr()->eq('l.campaign_id', ':campaignId'))
+            ->where(
+                $q->expr()->andX(
+                    $q->expr()->eq('l.campaign_id', ':campaignId'),
+                    $q->expr()->eq('l.manually_removed', 0)
+                )
+            )
             // Order by ID so we can query by greater than X contact ID when batching
             ->orderBy('l.lead_id')
             ->setMaxResults($limiter->getBatchLimit())
@@ -291,7 +296,12 @@ public function getInactiveContactCount($campaignId, array $decisionIds, Contact
             $q = $this->getSlaveConnection()->createQueryBuilder();
             $q->select('count(*)')
                 ->from(MAUTIC_TABLE_PREFIX.'campaign_leads', 'l')
-                ->where($q->expr()->eq('l.campaign_id', ':campaignId'))
+                ->where(
+                    $q->expr()->andX(
+                        $q->expr()->eq('l.campaign_id', ':campaignId'),
+                        $q->expr()->eq('l.manually_removed', 0)
+                    )
+                )
                 // Order by ID so we can query by greater than X contact ID when batching
                 ->orderBy('l.lead_id')
                 ->setParameter('campaignId', (int) $campaignId);

app/bundles/CampaignBundle/EventListener/CampaignActionJumpToEventSubscriber.php

@@ -95,19 +95,32 @@ public function onCampaignBuild(CampaignBuilderEvent $event)
      * Process campaign.jump_to_event actions.
      *
      * @param PendingEvent $campaignEvent
+     *
+     * @throws \Mautic\CampaignBundle\Executioner\Dispatcher\Exception\LogNotProcessedException
+     * @throws \Mautic\CampaignBundle\Executioner\Dispatcher\Exception\LogPassedAndFailedException
+     * @throws \Mautic\CampaignBundle\Executioner\Exception\CannotProcessEventException
+     * @throws \Mautic\CampaignBundle\Executioner\Scheduler\Exception\NotSchedulableException
      */
     public function onJumpToEvent(PendingEvent $campaignEvent)
     {
         $event      = $campaignEvent->getEvent();
         $jumpTarget = $this->getJumpTargetForEvent($event, 'e.id');
 
         if ($jumpTarget === null) {
-            $campaignEvent->passWithError($jumpTarget, $this->translator->trans('mautic.campaign.campaign.jump_to_event.target_not_exist'));
+            // Target event has been removed.
+            $pending  = $campaignEvent->getPending();
+            $contacts = $campaignEvent->getContacts();
+            foreach ($contacts as $logId => $contact) {
+                // Pass with an error for the UI.
+                $campaignEvent->passWithError(
+                    $pending->get($logId),
+                    $this->translator->trans('mautic.campaign.campaign.jump_to_event.target_not_exist')
+                );
+            }
+        } else {
+            $this->eventExecutioner->executeForContacts($jumpTarget, $campaignEvent->getContacts());
+            $campaignEvent->passRemaining();
         }
-
-        $this->eventExecutioner->executeForContacts($jumpTarget, $campaignEvent->getContacts());
-
-        $campaignEvent->passRemaining();
     }
 
     /**

app/bundles/CampaignBundle/EventListener/LeadSubscriber.php

@@ -99,7 +99,7 @@ public function onLeadListBatchChange(ListChangeEvent $event)
 
                     $removeLeads = [];
                     foreach ($leads as $l) {
-                        $lists = (isset($leadLists[$l])) ? $leadLists[$l] : [];
+                        $lists = (isset($leadLists[$l['id']])) ? $leadLists[$l['id']] : [];
                         if (array_intersect(array_keys($lists), $campaignLists[$c['id']])) {
                             continue;
                         } else {

app/bundles/CampaignBundle/Executioner/ContactFinder/ScheduledContactFinder.php

@@ -63,9 +63,14 @@ public function hydrateContacts(ArrayCollection $logs)
 
         $contacts = $this->leadRepository->getContactCollection($contactIds);
 
-        foreach ($logs as $log) {
+        foreach ($logs as $key => $log) {
             $contactId = $log->getLead()->getId();
-            $contact   = $contacts->get($contactId);
+            if (!$contact = $contacts->get($contactId)) {
+                // the contact must have been deleted mid execution so remove this log from memory
+                $logs->remove($key);
+
+                continue;
+            }
 
             $log->setLead($contact);
         }

app/bundles/CampaignBundle/Executioner/EventExecutioner.php

@@ -394,7 +394,7 @@ private function scheduleEvents(ArrayCollection $events, ArrayCollection $contac
 
             $this->logger->debug(
                 'CAMPAIGN: Event ID# '.$event->getId().
-                ' to be executed on '.$executionDate->format('Y-m-d H:i:s')
+                ' to be executed on '.$executionDate->format('Y-m-d H:i:s e')
             );
 
             // Check if we need to schedule this if it is not an inactivity check

app/bundles/CampaignBundle/Executioner/InactiveExecutioner.php

@@ -352,7 +352,7 @@ private function executeLogsForInactiveEvents(ArrayCollection $events, ArrayColl
 
             $this->logger->debug(
                 'CAMPAIGN: Event ID# '.$event->getId().
-                ' to be executed on '.$eventExecutionDate->format('Y-m-d H:i:s')
+                ' to be executed on '.$eventExecutionDate->format('Y-m-d H:i:s e')
             );
 
             if ($this->scheduler->shouldSchedule($eventExecutionDate, $executionDate)) {

app/bundles/CampaignBundle/Executioner/KickoffExecutioner.php

@@ -217,8 +217,8 @@ private function executeOrScheduleEvent()
                     $executionDate = $this->scheduler->getExecutionDateTime($event, $now);
                     $this->logger->debug(
                         'CAMPAIGN: Event ID# '.$event->getId().
-                        ' to be executed on '.$executionDate->format('Y-m-d H:i:s').
-                        ' compared to '.$now->format('Y-m-d H:i:s')
+                        ' to be executed on '.$executionDate->format('Y-m-d H:i:s e').
+                        ' compared to '.$now->format('Y-m-d H:i:s e')
                     );
 
                     // Adjust the hour based on contact timezone if applicable

app/bundles/CampaignBundle/Executioner/RealTimeExecutioner.php

@@ -210,7 +210,7 @@ private function executeAssociatedEvents(ArrayCollection $children, \DateTime $n
             $executionDate = $this->scheduler->getExecutionDateTime($child, $now);
             $this->logger->debug(
                 'CAMPAIGN: Event ID# '.$child->getId().
-                ' to be executed on '.$executionDate->format('Y-m-d H:i:s')
+                ' to be executed on '.$executionDate->format('Y-m-d H:i:s e')
             );
 
             if ($this->scheduler->shouldSchedule($executionDate, $now)) {

app/bundles/CampaignBundle/Executioner/ScheduledExecutioner.php

@@ -346,8 +346,8 @@ private function validateSchedule(ArrayCollection $logs, \DateTime $now, $schedu
             $executionDate = $this->scheduler->validateExecutionDateTime($log, $now);
             $this->logger->debug(
                 'CAMPAIGN: Log ID #'.$log->getID().
-                ' to be executed on '.$executionDate->format('Y-m-d H:i:s').
-                ' compared to '.$now->format('Y-m-d H:i:s')
+                ' to be executed on '.$executionDate->format('Y-m-d H:i:s e').
+                ' compared to '.$now->format('Y-m-d H:i:s e')
             );
 
             if ($this->scheduler->shouldSchedule($executionDate, $now)) {

app/bundles/CampaignBundle/Executioner/Scheduler/Mode/Interval.php

@@ -138,14 +138,21 @@ public function groupContactsByDate(Event $event, ArrayCollection $contacts, \Da
 
         // Get the difference between now and the date we're supposed to be executing
         $compareFromDateTime = $compareFromDateTime ? clone $compareFromDateTime : new \DateTime('now');
-        $compareFromDateTime->setTimezone($this->getDefaultTimezone());
-
-        $diff    = $compareFromDateTime->diff($executionDate);
-        $diff->f = 0; // we don't care about microseconds
+        $diff                = $compareFromDateTime->diff($executionDate);
+        $diff->f             = 0; // we don't care about microseconds
 
         /** @var Lead $contact */
         foreach ($contacts as $contact) {
-            $groupExecutionDate = $this->getGroupExecutionDateTime($event->getId(), $contact, $diff, $compareFromDateTime, $hour, $startTime, $endTime, $daysOfWeek);
+            $groupExecutionDate = $this->getGroupExecutionDateTime(
+                $event->getId(),
+                $contact,
+                $diff,
+                $compareFromDateTime,
+                $hour,
+                $startTime,
+                $endTime,
+                $daysOfWeek
+            );
             if (!isset($groupedExecutionDates[$groupExecutionDate->getTimestamp()])) {
                 $groupedExecutionDates[$groupExecutionDate->getTimestamp()] = new GroupExecutionDateDAO($groupExecutionDate);
             }
@@ -202,16 +209,46 @@ private function getGroupExecutionDateTime(
         \DateTime $endTime = null,
         array $daysOfWeek = []
     ) {
+        $this->logger->debug(
+            sprintf('CAMPAIGN: Comparing calculated executed time for event ID %s and contact ID %s with %s', $eventId, $contact->getId(), $compareFromDateTime->format('Y-m-d H:i:s e'))
+        );
+
         if ($hour) {
+            $this->logger->debug(
+                sprintf('CAMPAIGN: Scheduling event ID %s for contact ID %s based on hour of %s', $eventId, $contact->getId(), $hour->format('H:i e'))
+            );
             $groupDateTime = $this->getExecutionDateTimeFromHour($contact, $hour, $diff, $eventId, $compareFromDateTime);
         } elseif ($startTime && $endTime) {
+            $this->logger->debug(
+                sprintf(
+                    'CAMPAIGN: Scheduling event ID %s for contact ID %s based on hour range of %s to %s',
+                    $eventId,
+                    $contact->getId(),
+                    $startTime->format('H:i e'),
+                    $endTime->format('H:i e')
+                )
+            );
+
             $groupDateTime = $this->getExecutionDateTimeBetweenHours($contact, $startTime, $endTime, $diff, $eventId, $compareFromDateTime);
         } else {
+            $this->logger->debug(
+                sprintf('CAMPAIGN: Scheduling event ID %s for contact ID %s without hour restrictions.', $eventId, $contact->getId())
+            );
+
             $groupDateTime = clone $compareFromDateTime;
             $groupDateTime->add($diff);
         }
 
         if ($daysOfWeek) {
+            $this->logger->debug(
+                sprintf(
+                    'CAMPAIGN: Scheduling event ID %s for contact ID %s based on DOW restrictions of %s',
+                    $eventId,
+                    $contact->getId(),
+                    implode(',', $daysOfWeek)
+                )
+            );
+
             // Schedule for the next day of the week if applicable
             while (!in_array((int) $groupDateTime->format('w'), $daysOfWeek)) {
                 $groupDateTime->modify('+1 day');
@@ -262,6 +299,7 @@ private function getExecutionDateTimeFromHour(Lead $contact, \DateTime $hour, \D
         }
 
         $groupExecutionDate = clone $compareFromDateTime;
+        $groupExecutionDate->setTimezone($this->getDefaultTimezone());
         $groupExecutionDate->add($diff);
 
         $groupExecutionDate->setTime($groupHour->format('H'), $groupHour->format('i'));
@@ -323,6 +361,7 @@ private function getExecutionDateTimeBetweenHours(
 
         if (!isset($groupExecutionDate)) {
             $groupExecutionDate = clone $compareFromDateTime;
+            $groupExecutionDate->setTimezone($this->getDefaultTimezone());
             $groupExecutionDate->add($diff);
         }
 

app/bundles/CampaignBundle/Tests/Command/ValidateEventCommandTest.php

@@ -60,4 +60,35 @@ public function testEventsAreExecutedForInactiveEventWithMultipleContact()
         $this->assertCount(3, $byEvent[7]); // inactive event executed
         $this->assertCount(0, $byEvent[10]); // the positive path should be 0
     }
+
+    public function testContactsRemovedFromTheCampaignAreNotExecutedForInactiveEvents()
+    {
+        $this->runCommand('mautic:campaigns:trigger', ['-i' => 1, '--contact-ids' => '1,2,3']);
+
+        // Wait 20 seconds then execute the campaign again to send scheduled events
+        sleep(20);
+        $this->runCommand('mautic:campaigns:trigger', ['-i' => 1, '--contact-ids' => '1,2,3']);
+
+        // No open email decisions should be recorded yet
+        $byEvent = $this->getCampaignEventLogs([3]);
+        $this->assertCount(0, $byEvent[3]);
+
+        // Wait 20 seconds to go beyond the inaction timeframe
+        sleep(20);
+
+        // Remove a contact from the campaign
+        $this->db->createQueryBuilder()->update(MAUTIC_TABLE_PREFIX.'campaign_leads')
+            ->set('manually_removed', 1)
+            ->where('lead_id = 1')
+            ->execute();
+
+        // Now they should be inactive
+        $this->runCommand('mautic:campaigns:validate', ['--decision-id' => 3, '--contact-ids' => '1,2,3']);
+
+        // Only two contacts should have been considered inactive because one was marked as manually removed
+        $byEvent = $this->getCampaignEventLogs([3, 7, 10]);
+        $this->assertCount(2, $byEvent[3]); // decision recorded
+        $this->assertCount(2, $byEvent[7]); // inactive event executed
+        $this->assertCount(0, $byEvent[10]); // the positive path should be 0
+    }
 }