add CRAWL_META_ONLY option

YGGverse · Apr 3, 2023 · 79663c8 · 79663c8
1 parent dc55dcb
commit 79663c8
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/config/app.php.txt b/config/app.php.txt
@@ -22,3 +22,5 @@ define('CRAWL_PAGE_LIMIT', 10);
 define('CRAWL_PAGE_SECONDS_OFFSET', 3600);
 
 define('CRAWL_URL_REGEXP', '/^.*$/ui');
+
+define('CRAWL_META_ONLY', false);
diff --git a/crontab/crawler.php b/crontab/crawler.php
@@ -67,7 +67,7 @@
                   Filter::pageTitle($title->item(0)->nodeValue),
                   Filter::pageDescription($description),
                   Filter::pageKeywords($keywords),
-                  Filter::pageData($url->getContent()),
+                  CRAWL_META_ONLY ? '' : Filter::pageData($content),
                   time());
 
   // Update images