Skip to content

Commit

Permalink
Merge pull request #1185 from nexcess/devel
Browse files Browse the repository at this point in the history 
Devel
  • Loading branch information
@miguelbalparda
miguelbalparda committed May 31, 2016
2 parents bf24144 + 000362f commit 1236f29
Show file tree
Hide file tree
Showing 9 changed files with 112 additions and 32 deletions.
18 changes: 18 additions & 0 deletions app/code/community/Nexcessnet/Turpentine/Helper/Cron.php
View file
Expand Up @@ -145,6 +145,24 @@ public function getCrawlerDebugEnabled() {
return Mage::getStoreConfig('turpentine_varnish/general/crawler_debug');
}

/**
* Get number of urls to crawl per batch
*
* @return int
*/
public function getCrawlerBatchSize() {
return Mage::getStoreConfig('turpentine_varnish/general/crawler_batchsize');
}

/**
* Get time in seconds to wait between url batches
*
* @return int
*/
public function getCrawlerWaitPeriod() {
return Mage::getStoreConfig('turpentine_varnish/general/crawler_batchwait');
}

/**
* Get the list of all URLs
*
Expand Down
19 changes: 19 additions & 0 deletions app/code/community/Nexcessnet/Turpentine/Model/Core/Session.php
View file
Expand Up @@ -50,4 +50,23 @@ public function real_getFormKey()
}
return $this->getData('_form_key');
}

/**
* Creates new Form key
*/
public function renewFormKey()
{
$this->setData('_form_key', Mage::helper('core')->getRandomString(16));
}

/**
* Validates Form key
*
* @param string|null $formKey
* @return bool
*/
public function validateFormKey($formKey)
{
return ($formKey === $this->getFormKey());
}
}
58 changes: 30 additions & 28 deletions app/code/community/Nexcessnet/Turpentine/Model/Observer/Ban.php
View file
Expand Up @@ -298,37 +298,39 @@ public function banCacheType($eventObject) {
* @return bool
*/
public function banProductReview($eventObject) {
$patterns = array();
/* @var $review \Mage_Review_Model_Review*/
$review = $eventObject->getObject();

/* @var $productCollection \Mage_Review_Model_Resource_Review_Product_Collection*/
$productCollection = $review->getProductCollection();

$products = $productCollection->addEntityFilter((int) $review->getEntityPkValue())->getItems();
if (Mage::helper('turpentine/varnish')->getVarnishEnabled()) {
$patterns = array();
/* @var $review \Mage_Review_Model_Review*/
$review = $eventObject->getObject();

/* @var $productCollection \Mage_Review_Model_Resource_Review_Product_Collection*/
$productCollection = $review->getProductCollection();

$products = $productCollection->addEntityFilter((int) $review->getEntityPkValue())->getItems();

$productIds = array_unique(array_map(
create_function('$p', 'return $p->getEntityId();'),
$products ));
$patterns[] = sprintf('/review/product/list/id/(?:%s)/category/',
implode('|', array_unique($productIds)));
$patterns[] = sprintf('/review/product/view/id/%d/',
$review->getEntityId());
$productPatterns = array();
foreach ($products as $p) {
$urlKey = $p->getUrlModel()->formatUrlKey($p->getName());
if ($urlKey) {
$productPatterns[] = $urlKey;
$productIds = array_unique(array_map(
create_function('$p', 'return $p->getEntityId();'),
$products ));
$patterns[] = sprintf('/review/product/list/id/(?:%s)/category/',
implode('|', array_unique($productIds)));
$patterns[] = sprintf('/review/product/view/id/%d/',
$review->getEntityId());
$productPatterns = array();
foreach ($products as $p) {
$urlKey = $p->getUrlModel()->formatUrlKey($p->getName());
if ($urlKey) {
$productPatterns[] = $urlKey;
}
}
}
if ( ! empty($productPatterns)) {
$productPatterns = array_unique($productPatterns);
$patterns[] = sprintf('(?:%s)', implode('|', $productPatterns));
}
$urlPattern = implode('|', $patterns);
if ( ! empty($productPatterns)) {
$productPatterns = array_unique($productPatterns);
$patterns[] = sprintf('(?:%s)', implode('|', $productPatterns));
}
$urlPattern = implode('|', $patterns);

$result = $this->_getVarnishAdmin()->flushUrl($urlPattern);
return $this->_checkResult($result);
$result = $this->_getVarnishAdmin()->flushUrl($urlPattern);
return $this->_checkResult($result);
}
}

/**
Expand Down
15 changes: 15 additions & 0 deletions app/code/community/Nexcessnet/Turpentine/Model/Observer/Cron.php
View file
Expand Up @@ -49,6 +49,11 @@ public function crawlUrls($eventObject) {
if ($maxRunTime === 0) {
$maxRunTime = self::MAX_CRAWL_TIME;
}

$batchSize = $helper->getCrawlerBatchSize();
$timeout = $helper->getCrawlerWaitPeriod();
$crawlCount = 0;

// just in case we have a silly short max_execution_time
$maxRunTime = abs($maxRunTime - self::EXEC_TIME_BUFFER);
while (($helper->getRunTime() < $maxRunTime) &&
Expand All @@ -57,6 +62,16 @@ public function crawlUrls($eventObject) {
Mage::helper('turpentine/debug')->logWarn(
'Failed to crawl URL: %s', $url );
}

if ($crawlCount > 0
&& $timeout > 0
&& $batchSize > 0
&& $crawlCount % $batchSize == 0
) {
Mage::helper('turpentine/debug')->logDebug('Crawled '.$crawlCount.' urls, sleeping for '.$timeout.' seconds');
sleep($timeout);
}
$crawlCount++;
}
}
}
Expand Down
4 changes: 3 additions & 1 deletion app/code/community/Nexcessnet/Turpentine/etc/config.xml
View file
Expand Up @@ -35,6 +35,8 @@
<fix_product_toolbar>0</fix_product_toolbar>
<crawler_enable>0</crawler_enable>
<crawler_debug>0</crawler_debug>
<crawler_batchsize>0</crawler_batchsize>
<crawler_batchwait>0</crawler_batchwait>
</general>
<logging>
<use_custom_log_file>0</use_custom_log_file>
Expand All @@ -59,7 +61,7 @@
<frontend_timeout>300</frontend_timeout>
<admin_timeout>21600</admin_timeout>
<crawlers>127.0.0.1</crawlers>
<crawler_user_agents><![CDATA[ApacheBench/.*,.*Googlebot.*,JoeDog/.*Siege.*,magespeedtest\.com,Nexcessnet_Turpentine/.*]]></crawler_user_agents>
<crawler_user_agents><![CDATA[ApacheBench/.*,.*Googlebot.*,JoeDog/.*Siege.*,magespeedtest\.com,Nexcessnet_Turpentine/.*,.*PTST.*]]></crawler_user_agents>
</backend>
<normalization>
<encoding>1</encoding>
Expand Down
24 changes: 24 additions & 0 deletions app/code/community/Nexcessnet/Turpentine/etc/system.xml
View file
Expand Up @@ -132,6 +132,30 @@
<show_in_website>0</show_in_website>
<show_in_store>0</show_in_store>
</crawler_debug>
<crawler_batchsize translate="label" module="turpentine">
<label>Crawler Batch Size</label>
<comment>Number of URLs to crawl per batch, when 0 requests will not be batched</comment>
<frontend_type>text</frontend_type>
<sort_order>90</sort_order>
<show_in_default>1</show_in_default>
<show_in_website>0</show_in_website>
<show_in_store>0</show_in_store>
<depends>
<crawler_enable>1</crawler_enable>
</depends>
</crawler_batchsize>
<crawler_batchwait translate="label" module="turpentine">
<label>Crawler Batch Wait</label>
<comment>Time in seconds to wait between batches</comment>
<frontend_type>text</frontend_type>
<sort_order>100</sort_order>
<show_in_default>1</show_in_default>
<show_in_website>0</show_in_website>
<show_in_store>0</show_in_store>
<depends>
<crawler_enable>1</crawler_enable>
</depends>
</crawler_batchwait>
</fields>
</general>
<logging translate="label" module="turpentine">
Expand Down
2 changes: 1 addition & 1 deletion app/code/community/Nexcessnet/Turpentine/misc/version-4.vcl
View file
Expand Up @@ -116,7 +116,7 @@ sub vcl_recv {
if (!{{enable_caching}} || req.http.Authorization ||
req.method !~ "^(GET|HEAD|OPTIONS)$" ||
req.http.Cookie ~ "varnish_bypass={{secret_handshake}}") {
return (pipe);
return (pass);
}

if({{send_unmodified_url}}) {
Expand Down
2 changes: 1 addition & 1 deletion build/build_package.py
View file
Expand Up @@ -280,7 +280,7 @@ def _build_dependencies_tag(self, dependencies_tag, connect_dom):
min_tag.text = connect_dom.findtext('depends_php_min')
max_tag = ElementTree.SubElement(php_tag, 'max')
max_tag.text = connect_dom.findtext('depends_php_max')
self._logger.debug('Finished adding dependancies')
self._logger.debug('Finished adding dependencies')
return dependencies_tag

def _get_module_dom(self, ext_name):
Expand Down
2 changes: 1 addition & 1 deletion build/mage-package.xml
View file
Expand Up @@ -26,7 +26,7 @@
</email>
</authors>
<depends_php_min>5.2.13</depends_php_min>
<depends_php_max>6.0.0</depends_php_max>
<depends_php_max>7.1.0</depends_php_max>
<depends>
<package>
<name>
Expand Down

0 comments on commit 1236f29

Please sign in to comment.