From 23a5169ecbd4a67575ca9f82643fc9eed87197d5 Mon Sep 17 00:00:00 2001 From: Dmitry Khrysev Date: Wed, 29 Mar 2017 23:12:04 +0300 Subject: [PATCH] CRM-7972: Slow Ecommerce widget leads to 40 seconds login (#9087) --- .../AbstractPrecalculatedVisitProvider.php | 128 ++++++++++++++++ .../PrecalculatedTrackingVisitProvider.php | 104 +++++++++++++ .../PrecalculatedWebsiteVisitProvider.php | 70 +++++++++ .../Resources/config/services.yml | 28 ++++ .../LoadTrackingWebsiteToMagentoChannel.php | 41 +++++ ...PrecalculatedTrackingVisitProviderTest.php | 79 ++++++++++ ...ecalculatedTrackingWebsiteProviderTest.php | 69 +++++++++ ...PrecalculatedTrackingVisitProviderTest.php | 142 ++++++++++++++++++ .../PrecalculatedWebsiteVisitProviderTest.php | 109 ++++++++++++++ 9 files changed, 770 insertions(+) create mode 100644 src/Oro/Bridge/MarketingCRM/Provider/AbstractPrecalculatedVisitProvider.php create mode 100644 src/Oro/Bridge/MarketingCRM/Provider/PrecalculatedTrackingVisitProvider.php create mode 100644 src/Oro/Bridge/MarketingCRM/Provider/PrecalculatedWebsiteVisitProvider.php create mode 100644 src/Oro/Bridge/MarketingCRM/Tests/Functional/DataFixtures/LoadTrackingWebsiteToMagentoChannel.php create mode 100644 src/Oro/Bridge/MarketingCRM/Tests/Functional/Provider/PrecalculatedTrackingVisitProviderTest.php create mode 100644 src/Oro/Bridge/MarketingCRM/Tests/Functional/Provider/PrecalculatedTrackingWebsiteProviderTest.php create mode 100644 src/Oro/Bridge/MarketingCRM/Tests/Unit/Provider/PrecalculatedTrackingVisitProviderTest.php create mode 100644 src/Oro/Bridge/MarketingCRM/Tests/Unit/Provider/PrecalculatedWebsiteVisitProviderTest.php diff --git a/src/Oro/Bridge/MarketingCRM/Provider/AbstractPrecalculatedVisitProvider.php b/src/Oro/Bridge/MarketingCRM/Provider/AbstractPrecalculatedVisitProvider.php new file mode 100644 index 00000000000..f48af3f0455 --- /dev/null +++ b/src/Oro/Bridge/MarketingCRM/Provider/AbstractPrecalculatedVisitProvider.php @@ -0,0 +1,128 @@ +registry = $registry; + $this->aclHelper = $aclHelper; + $this->configManager = $configManager; + } + + /** + * @param QueryBuilder $queryBuilder + * @return int + */ + protected function getSingleIntegerResult(QueryBuilder $queryBuilder) + { + try { + return (int)$this->aclHelper->apply($queryBuilder)->getSingleScalarResult(); + } catch (NoResultException $ex) { + return 0; + } + } + + /** + * @param QueryBuilder $queryBuilder + * @param \DateTime $from + * @param \DateTime $to + */ + protected function applyDateLimit(QueryBuilder $queryBuilder, \DateTime $from, \DateTime $to) + { + if ($from && $to && $this->getDate($from) === $this->getDate($to)) { + $queryBuilder->andWhere($queryBuilder->expr()->eq('t.actionDate', ':date')) + ->setParameter('date', $this->getDate($from)); + } else { + if ($from) { + $queryBuilder + ->andWhere($queryBuilder->expr()->gte('t.actionDate', ':from')) + ->setParameter('from', $this->getDate($from)); + } + if ($to) { + $queryBuilder + ->andWhere($queryBuilder->expr()->lte('t.actionDate', ':to')) + ->setParameter('to', $this->getDate($to)); + } + } + } + + /** + * @return QueryBuilder + */ + protected function createUniqueVisitQueryBuilder() + { + $queryBuilder = $this + ->getUniqueTrackingVisitRepository() + ->createQueryBuilder('t'); + + return $queryBuilder; + } + + /** + * @return bool + */ + protected function isPrecalculatedStatisticEnabled() + { + return $this->configManager->get('oro_tracking.precalculated_statistic_enabled'); + } + + /** + * @return EntityRepository + */ + private function getUniqueTrackingVisitRepository() + { + return $this->registry->getManagerForClass(UniqueTrackingVisit::class) + ->getRepository(UniqueTrackingVisit::class); + } + + /** + * @param \DateTime $dateTime + * @return string + */ + private function getDate(\DateTime $dateTime) + { + /** @var Connection $connection */ + $connection = $this->registry->getConnection(); + $dateFormat = $connection->getDatabasePlatform()->getDateFormatString(); + + return $dateTime->format($dateFormat); + } +} diff --git a/src/Oro/Bridge/MarketingCRM/Provider/PrecalculatedTrackingVisitProvider.php b/src/Oro/Bridge/MarketingCRM/Provider/PrecalculatedTrackingVisitProvider.php new file mode 100644 index 00000000000..e0836156564 --- /dev/null +++ b/src/Oro/Bridge/MarketingCRM/Provider/PrecalculatedTrackingVisitProvider.php @@ -0,0 +1,104 @@ +trackingVisitProvider = $trackingVisitProvider; + } + + /** + * Return total number of visits, last visit date and visits per month + * filtered by customers + * + * @param Customer[] $customers + * + * @return array + */ + public function getAggregates(array $customers) + { + return $this->trackingVisitProvider->getAggregates($customers); + } + + /** + * @param \DateTime $from + * @param \DateTime $to + * + * @return int + */ + public function getDeeplyVisitedCount(\DateTime $from = null, \DateTime $to = null) + { + if (!$this->isPrecalculatedStatisticEnabled()) { + return $this->trackingVisitProvider->getDeeplyVisitedCount($from, $to); + } + + if (!$this->isFeaturesEnabled()) { + return 0; + } + + $queryBuilder = $this->getVisitCountQueryBuilder($from, $to); + $queryBuilder->andHaving('COUNT(t.userIdentifier) > 1'); + + return $this->getSingleIntegerResult($queryBuilder); + } + + /** + * @param \DateTime $from + * @param \DateTime $to + * + * @return int + */ + public function getVisitedCount(\DateTime $from = null, \DateTime $to = null) + { + if (!$this->isPrecalculatedStatisticEnabled()) { + return $this->trackingVisitProvider->getVisitedCount($from, $to); + } + + if (!$this->isFeaturesEnabled()) { + return 0; + } + + $queryBuilder = $this->getVisitCountQueryBuilder($from, $to); + + return $this->getSingleIntegerResult($queryBuilder); + } + + /** + * @param \DateTime|null $from + * @param \DateTime|null $to + * @return QueryBuilder + */ + private function getVisitCountQueryBuilder(\DateTime $from = null, \DateTime $to = null) + { + $queryBuilder = $this->createUniqueVisitQueryBuilder(); + $this->applyDateLimit($queryBuilder, $from, $to); + + $queryBuilder + ->select($queryBuilder->expr()->countDistinct('t.userIdentifier')) + ->join('t.trackingWebsite', 'tw') + ->join('tw.channel', 'c') + ->andWhere($queryBuilder->expr()->eq('c.channelType', ':channel')) + ->andWhere($queryBuilder->expr()->eq('c.status', ':status')) + ->setParameter('channel', ChannelType::TYPE) + ->setParameter('status', Channel::STATUS_ACTIVE); + + return $queryBuilder; + } +} diff --git a/src/Oro/Bridge/MarketingCRM/Provider/PrecalculatedWebsiteVisitProvider.php b/src/Oro/Bridge/MarketingCRM/Provider/PrecalculatedWebsiteVisitProvider.php new file mode 100644 index 00000000000..f5a4438ccae --- /dev/null +++ b/src/Oro/Bridge/MarketingCRM/Provider/PrecalculatedWebsiteVisitProvider.php @@ -0,0 +1,70 @@ +visitProvider = $visitProvider; + } + + /** + * {@inheritdoc} + */ + public function getSiteVisitsValues($dateRange) + { + if (!$this->isPrecalculatedStatisticEnabled()) { + return $this->visitProvider->getSiteVisitsValues($dateRange); + } + + if (!$this->isFeaturesEnabled()) { + return 0; + } + + $queryBuilder = $this->getVisitsCountQueryBuilder($dateRange['start'], $dateRange['end']); + + return $this->getSingleIntegerResult($queryBuilder); + } + + /** + * @param \DateTime|null $from + * @param \DateTime|null $to + * @return QueryBuilder + */ + private function getVisitsCountQueryBuilder(\DateTime $from = null, \DateTime $to = null) + { + $queryBuilder = $this->createUniqueVisitQueryBuilder(); + + $queryBuilder->select('SUM(t.visitCount)') + ->join('t.trackingWebsite', 'site') + ->leftJoin('site.channel', 'channel') + ->where($queryBuilder->expr()->orX( + $queryBuilder->expr()->isNull('channel.id'), + $queryBuilder->expr()->andX( + $queryBuilder->expr()->eq('channel.channelType', ':channel'), + $queryBuilder->expr()->eq('channel.status', ':status') + ) + )) + ->setParameter('channel', ChannelType::TYPE) + ->setParameter('status', Channel::STATUS_ACTIVE); + + $this->applyDateLimit($queryBuilder, $from, $to); + + return $queryBuilder; + } +} diff --git a/src/Oro/Bridge/MarketingCRM/Resources/config/services.yml b/src/Oro/Bridge/MarketingCRM/Resources/config/services.yml index 6a9f77d44f5..243ad9a4d37 100644 --- a/src/Oro/Bridge/MarketingCRM/Resources/config/services.yml +++ b/src/Oro/Bridge/MarketingCRM/Resources/config/services.yml @@ -22,3 +22,31 @@ services: class: %oro_channel.listener.channel_repository_subscriber.class% tags: - { name: doctrine.event_subscriber } + + oro_magento.provider.tracking_visit_precalculated.abstract: + class: Oro\Bridge\MarketingCRM\Provider\AbstractPrecalculatedVisitProvider + abstract: true + arguments: + - '@doctrine' + - '@oro_config.global' + - '@oro_security.acl_helper' + + oro_magento.provider.tracking_visit.precalculated: + class: Oro\Bridge\MarketingCRM\Provider\PrecalculatedTrackingVisitProvider + parent: oro_magento.provider.tracking_visit_precalculated.abstract + decorates: oro_magento.provider.tracking_visit + public: false + calls: + - ['setVisitProvider', ['@oro_magento.provider.tracking_visit.precalculated.inner']] + tags: + - { name: oro_featuretogle.feature, feature: 'tracking' } + + oro_magento.provider.website_visit.precalculated: + class: Oro\Bridge\MarketingCRM\Provider\PrecalculatedWebsiteVisitProvider + parent: oro_magento.provider.tracking_visit_precalculated.abstract + decorates: oro_magento.provider.website_visit + public: false + calls: + - ['setVisitProvider', ['@oro_magento.provider.website_visit.precalculated.inner']] + tags: + - { name: oro_featuretogle.feature, feature: 'tracking' } diff --git a/src/Oro/Bridge/MarketingCRM/Tests/Functional/DataFixtures/LoadTrackingWebsiteToMagentoChannel.php b/src/Oro/Bridge/MarketingCRM/Tests/Functional/DataFixtures/LoadTrackingWebsiteToMagentoChannel.php new file mode 100644 index 00000000000..001447007fa --- /dev/null +++ b/src/Oro/Bridge/MarketingCRM/Tests/Functional/DataFixtures/LoadTrackingWebsiteToMagentoChannel.php @@ -0,0 +1,41 @@ +getReference(LoadTrackingWebsites::TRACKING_WEBSITE); + + if (method_exists($website, 'setChannel')) { + /** @var Channel $channel */ + $channel = $this->getReference('default_channel'); + $website->setChannel($channel); + $manager->flush($website); + } + } +} diff --git a/src/Oro/Bridge/MarketingCRM/Tests/Functional/Provider/PrecalculatedTrackingVisitProviderTest.php b/src/Oro/Bridge/MarketingCRM/Tests/Functional/Provider/PrecalculatedTrackingVisitProviderTest.php new file mode 100644 index 00000000000..3acf1b747b7 --- /dev/null +++ b/src/Oro/Bridge/MarketingCRM/Tests/Functional/Provider/PrecalculatedTrackingVisitProviderTest.php @@ -0,0 +1,79 @@ +initClient([], $this->generateBasicAuthHeader()); + $this->loadFixtures([ + LoadTrackingVisits::class, + LoadTrackingWebsiteToMagentoChannel::class + ]); + $this->provider = $this->getContainer()->get('oro_magento.provider.tracking_visit'); + $this->originalProvider = new TrackingVisitProvider( + $this->getContainer()->get('doctrine'), + $this->getContainer()->get('oro_security.acl_helper') + ); + } + + public function testDecoration() + { + $this->assertInstanceOf(PrecalculatedTrackingVisitProvider::class, $this->provider); + } + + public function testGetVisitedCount() + { + $timezone = $this->getTimezone(); + $from = new \DateTime('2012-01-11', $timezone); + $to = new \DateTime('2013-01-13', $timezone); + + $original = $this->originalProvider->getVisitedCount($from, $to); + $precalculated = $this->provider->getVisitedCount($from, $to); + + $this->assertEquals($original, $precalculated); + } + + public function testGetDeeplyVisitedCount() + { + $timezone = $this->getTimezone(); + $from = new \DateTime('2012-01-01', $timezone); + $to = new \DateTime('2013-01-01', $timezone); + + $original = $this->originalProvider->getDeeplyVisitedCount($from, $to); + $precalculated = $this->provider->getDeeplyVisitedCount($from, $to); + + $this->assertEquals($original, $precalculated); + } + + /** + * @return \DateTimeZone + */ + private function getTimezone() + { + $configManager = $this->getContainer()->get('oro_config.global'); + + $timezoneName = $configManager->get('oro_locale.timezone'); + if (!$timezoneName) { + $timezoneName = 'UTC'; + } + return new \DateTimeZone($timezoneName); + } +} diff --git a/src/Oro/Bridge/MarketingCRM/Tests/Functional/Provider/PrecalculatedTrackingWebsiteProviderTest.php b/src/Oro/Bridge/MarketingCRM/Tests/Functional/Provider/PrecalculatedTrackingWebsiteProviderTest.php new file mode 100644 index 00000000000..3d157905b50 --- /dev/null +++ b/src/Oro/Bridge/MarketingCRM/Tests/Functional/Provider/PrecalculatedTrackingWebsiteProviderTest.php @@ -0,0 +1,69 @@ +initClient([], $this->generateBasicAuthHeader()); + $this->loadFixtures([ + LoadTrackingVisits::class, + LoadTrackingWebsiteToMagentoChannel::class + ]); + $this->provider = $this->getContainer()->get('oro_magento.provider.website_visit'); + $this->originalProvider = new WebsiteVisitProvider( + $this->getContainer()->get('doctrine'), + $this->getContainer()->get('oro_security.acl_helper'), + $this->getContainer()->get('oro_dashboard.provider.big_number.date_helper') + ); + } + + public function testDecoration() + { + $this->assertInstanceOf(PrecalculatedWebsiteVisitProvider::class, $this->provider); + } + + public function testGetVisitedCount() + { + $timezone = $this->getTimezone(); + $from = new \DateTime('2012-01-11', $timezone); + $to = new \DateTime('2013-01-13', $timezone); + $dateRange = ['start' => $from, 'end' => $to]; + + $original = $this->originalProvider->getSiteVisitsValues($dateRange); + $precalculated = $this->provider->getSiteVisitsValues($dateRange); + + $this->assertEquals($original, $precalculated); + } + + /** + * @return \DateTimeZone + */ + private function getTimezone() + { + $configManager = $this->getContainer()->get('oro_config.global'); + + $timezoneName = $configManager->get('oro_locale.timezone'); + if (!$timezoneName) { + $timezoneName = 'UTC'; + } + return new \DateTimeZone($timezoneName); + } +} diff --git a/src/Oro/Bridge/MarketingCRM/Tests/Unit/Provider/PrecalculatedTrackingVisitProviderTest.php b/src/Oro/Bridge/MarketingCRM/Tests/Unit/Provider/PrecalculatedTrackingVisitProviderTest.php new file mode 100644 index 00000000000..4407670909b --- /dev/null +++ b/src/Oro/Bridge/MarketingCRM/Tests/Unit/Provider/PrecalculatedTrackingVisitProviderTest.php @@ -0,0 +1,142 @@ +registry = $this->createMock(ManagerRegistry::class); + $this->aclHelper = $this->createMock(AclHelper::class); + $this->configManager = $this->createMock(ConfigManager::class); + $this->featureChecker = $this->createMock(FeatureChecker::class); + + $this->provider = new PrecalculatedTrackingVisitProvider( + $this->registry, + $this->configManager, + $this->aclHelper + ); + $this->provider->setFeatureChecker($this->featureChecker); + $this->provider->addFeature('test'); + + $this->visitProvider = $this->createMock(TrackingVisitProviderInterface::class); + $this->provider->setVisitProvider($this->visitProvider); + } + + public function testGetAggregates() + { + $customers = []; + + $this->visitProvider->expects($this->once()) + ->method('getAggregates') + ->with($customers); + + $this->provider->getAggregates($customers); + } + + public function testGetDeeplyVisitedCountPrecalculationDisabled() + { + $from = new \DateTime(); + $to = new \DateTime(); + + $this->isPrecalculationEnabled(false); + $this->visitProvider->expects($this->once()) + ->method('getDeeplyVisitedCount') + ->with($from, $to) + ->willReturn(42); + $this->assertSame(42, $this->provider->getDeeplyVisitedCount($from, $to)); + } + + public function testGetDeeplyVisitedCountFeatureDisabled() + { + $from = new \DateTime(); + $to = new \DateTime(); + + $this->isPrecalculationEnabled(true); + $this->visitProvider->expects($this->never()) + ->method($this->anything()); + + $this->featureChecker->expects($this->once()) + ->method('isFeatureEnabled') + ->with('test') + ->willReturn(false); + $this->assertSame(0, $this->provider->getDeeplyVisitedCount($from, $to)); + } + + public function testGetVisitedCountPrecalculationDisabled() + { + $from = new \DateTime(); + $to = new \DateTime(); + + $this->isPrecalculationEnabled(false); + $this->visitProvider->expects($this->once()) + ->method('getVisitedCount') + ->with($from, $to) + ->willReturn(42); + $this->assertSame(42, $this->provider->getVisitedCount($from, $to)); + } + + public function testGetVisitedCountFeatureDisabled() + { + $from = new \DateTime(); + $to = new \DateTime(); + + $this->isPrecalculationEnabled(true); + $this->visitProvider->expects($this->never()) + ->method($this->anything()); + + $this->featureChecker->expects($this->once()) + ->method('isFeatureEnabled') + ->with('test') + ->willReturn(false); + $this->assertSame(0, $this->provider->getVisitedCount($from, $to)); + } + + /** + * @param bool $enabled + */ + private function isPrecalculationEnabled($enabled) + { + $this->configManager->expects($this->once()) + ->method('get') + ->with('oro_tracking.precalculated_statistic_enabled') + ->willReturn($enabled); + } +} diff --git a/src/Oro/Bridge/MarketingCRM/Tests/Unit/Provider/PrecalculatedWebsiteVisitProviderTest.php b/src/Oro/Bridge/MarketingCRM/Tests/Unit/Provider/PrecalculatedWebsiteVisitProviderTest.php new file mode 100644 index 00000000000..fee3c0d5438 --- /dev/null +++ b/src/Oro/Bridge/MarketingCRM/Tests/Unit/Provider/PrecalculatedWebsiteVisitProviderTest.php @@ -0,0 +1,109 @@ +registry = $this->createMock(ManagerRegistry::class); + $this->aclHelper = $this->createMock(AclHelper::class); + $this->configManager = $this->createMock(ConfigManager::class); + $this->featureChecker = $this->createMock(FeatureChecker::class); + + $this->provider = new PrecalculatedWebsiteVisitProvider( + $this->registry, + $this->configManager, + $this->aclHelper + ); + $this->provider->setFeatureChecker($this->featureChecker); + $this->provider->addFeature('test'); + + $this->visitProvider = $this->createMock(WebsiteVisitProviderInterface::class); + $this->provider->setVisitProvider($this->visitProvider); + } + + public function testDecoration() + { + $this->assertInstanceOf(PrecalculatedWebsiteVisitProvider::class, $this->provider); + } + + public function testGetDeeplyVisitedCountPrecalculationDisabled() + { + $from = new \DateTime(); + $to = new \DateTime(); + $dateRange = ['start' => $from, 'end' => $to]; + + $this->isPrecalculationEnabled(false); + $this->visitProvider->expects($this->once()) + ->method('getSiteVisitsValues') + ->with($dateRange) + ->willReturn(42); + $this->assertSame(42, $this->provider->getSiteVisitsValues($dateRange)); + } + + public function testGetDeeplyVisitedCountFeatureDisabled() + { + $from = new \DateTime(); + $to = new \DateTime(); + $dateRange = ['start' => $from, 'end' => $to]; + + $this->isPrecalculationEnabled(true); + $this->visitProvider->expects($this->never()) + ->method($this->anything()); + + $this->featureChecker->expects($this->once()) + ->method('isFeatureEnabled') + ->with('test') + ->willReturn(false); + $this->assertSame(0, $this->provider->getSiteVisitsValues($dateRange)); + } + + /** + * @param bool $enabled + */ + private function isPrecalculationEnabled($enabled) + { + $this->configManager->expects($this->once()) + ->method('get') + ->with('oro_tracking.precalculated_statistic_enabled') + ->willReturn($enabled); + } +}