Remove bing ping and add IndexNow support

For issue and feature #1105.

Geeklog will ping IndexNow server when a page is updated, created, or deleted.

plugins/xmlsitemap/configuration_validation.php

@@ -57,7 +57,10 @@
 
 // Ping target
 $_CONF_VALIDATE['xmlsitemap']['ping_google'] = ['rule' => 'boolean'];
-$_CONF_VALIDATE['xmlsitemap']['ping_bing']   = ['rule' => 'boolean'];
+
+// IndexNow
+$_CONF_VALIDATE['xmlsitemap']['indexnow']   = ['rule' => 'boolean'];
+$_CONF_VALIDATE['xmlsitemap']['indexnow_key_location'] = ['rule' => 'url'];
 
 // News Sitemap
 $_CONF_VALIDATE['xmlsitemap']['news_sitemap_file'] = ['rule' => 'notEmpty'];

plugins/xmlsitemap/functions.inc

@@ -395,10 +395,6 @@ function XMLSMAP_update($type = null, $include = null, $exclude = null)
         $pingTargets[] = 'google';
     }
 
-    if (isset($_XMLSMAP_CONF['ping_bing']) && $_XMLSMAP_CONF['ping_bing']) {
-        $pingTargets[] = 'bing';
-    }
-
     foreach ($sitemap->getFileNames() as $fileName) {
         $sitemap->sendPing($pingTargets, $fileName);
     }
@@ -456,12 +452,12 @@ function plugin_itemsaved_xmlsitemap($id, $type, $old_id, $sub_type)
 
 	if (!empty($old_id) && ($id !== $old_id)) {
 		// Item updated
-		$Url = PLG_idToURL($type, $sub_type, $old_id);
+		$url = PLG_idToURL($type, $sub_type, $old_id);
 	} else {
-		$Url = PLG_idToURL($type, $sub_type, $id); 
+		$url = PLG_idToURL($type, $sub_type, $id); 
     }
 	// Always need to delete as item may exist already in sitemap but needs to be changed
-	$sitemap->deleteItem($type, $Url); 
+	$sitemap->deleteItem($type, $url); 
 
 	if (!empty($newItem)) { 
 		// Item created
@@ -472,11 +468,16 @@ function plugin_itemsaved_xmlsitemap($id, $type, $old_id, $sub_type)
 	}				
 
 	$result = $sitemap->endUpdate();
-
+	
     if (!$result) {
 		// Tried to update sitemap files partially, but failed.  Let's use the old way
 		XMLSMAP_update($type);
     }
+
+	// Ping IndexNow for changed url. For new, updated, and delete
+	if ($_XMLSMAP_CONF['indexnow']) {
+		$sitemap->submitURL($url);
+	}
 }
 
 /**

plugins/xmlsitemap/install_defaults.php

@@ -86,7 +86,12 @@
 
 // Ping targets
 $_XMLSMAP_DEFAULT['ping_google'] = true;
-$_XMLSMAP_DEFAULT['ping_bing']   = true;
+
+// IndexNow
+$_XMLSMAP_DEFAULT['indexnow'] = false;
+$_XMLSMAP_DEFAULT['indexnow_key'] = '';
+$_XMLSMAP_DEFAULT['indexnow_key_location'] = '';
+
 
 // News Sitemap settings
 // Array of article topics for news. If none then all topics.
@@ -142,17 +147,23 @@ function plugin_initconfig_xmlsitemap()
         $c->add('fs_ping', null, 'fieldset', 0, 3, null, 0, true, $me, 3);
         $c->add('ping_google', $_XMLSMAP_DEFAULT['ping_google'], 'select', 0,
             3, 1, 100, true, $me, 3);
-        $c->add('ping_bing', $_XMLSMAP_DEFAULT['ping_bing'], 'select', 0,
+
+		// IndexNow
+        $c->add('indexnow', $_XMLSMAP_DEFAULT['indexnow'], 'select', 0,
             3, 1, 110, true, $me, 3);
-
+		$c->add('indexnow_key', $_XMLSMAP_DEFAULT['indexnow_key'], 'text', 0,
+            3, null, 111, true, $me, 3);			
+		$c->add('indexnow_key_location', $_XMLSMAP_DEFAULT['indexnow_key_location'], 'text', 0,
+            3, null, 112, true, $me, 3);			
+
         // News Sitemap
         $c->add('tab_news', null, 'tab', 0, 4, null, 0, true, $me, 4);
         $c->add('fs_news', null, 'fieldset', 0, 4, null, 0, true, $me, 4);
         $c->add('news_sitemap_file', $_XMLSMAP_DEFAULT['news_sitemap_file'],
             'text', 0, 4, null, 120, false, $me, 4);
         $c->add('news_sitemap_topics', $_XMLSMAP_DEFAULT['news_sitemap_topics'], '%text', 0, 4, null, 130,
             true, $me, 4);
-        $c->add('news_sitemap_age',$_XMLSMAP_DEFAULT['news_sitemap_age'],'text',0,4,NULL,140,TRUE, $me, 4);
+        $c->add('news_sitemap_age',$_XMLSMAP_DEFAULT['news_sitemap_age'], 'text', 0, 4, null, 140, true, $me, 4);
     }
 
     return true;

plugins/xmlsitemap/install_updates.php

@@ -76,5 +76,17 @@ function xmlsitemap_update_ConfValues_2_0_3()
     // Drop $_XMLSMAP_CONF['exclude']
     $c->del('exclude', $me);
 
+	// Drop $_XMLSMAP_CONF['ping_bing']
+	$c->del('ping_bing', $me);
+
+	// IndexNow
+	$c->add('indexnow', $_XMLSMAP_DEFAULT['indexnow'], 'select', 0,
+		3, 1, 110, true, $me, 3);
+	$c->add('indexnow_key', $_XMLSMAP_DEFAULT['indexnow_key'], 'text', 0,
+		3, null, 111, true, $me, 3);			
+	$c->add('indexnow_key_location', $_XMLSMAP_DEFAULT['indexnow_key_location'], 'text', 0,
+		3, null, 112, true, $me, 3);			
+
+
 	return true;
 }

plugins/xmlsitemap/language/english.php

@@ -60,7 +60,9 @@
     'priorities'          => 'Priority',
     'frequencies'         => 'Frequency',
     'ping_google'         => 'Send ping to Google',
-    'ping_bing'           => 'Send ping to Bing',
+	'indexnow'            => 'Enable IndexNow',
+	'indexnow_key'        => 'IndexNow Key',
+	'indexnow_key_location' => 'IndexNow Key Location',
     'news_sitemap_file'   => 'News Sitemap file name',
     'news_sitemap_topics' => 'Include Articles from these Topics',
     'news_sitemap_age'    => 'Max Age of Articles',
@@ -82,7 +84,7 @@
     'fs_main' => 'XMLSitemap Main Settings',
     'fs_pri'  => 'Priority (default = 0.5, lowest = 0.0, highest = 1.0)',
     'fs_freq' => 'Update frequency',
-    'fs_ping' => 'Send ping on updating sitemap',
+    'fs_ping' => 'Send ping on on change',
     'fs_news' => 'News Sitemap Settings',
 );
 

plugins/xmlsitemap/language/english_utf-8.php

@@ -60,7 +60,9 @@
     'priorities'          => 'Priority',
     'frequencies'         => 'Frequency',
     'ping_google'         => 'Send ping to Google',
-    'ping_bing'           => 'Send ping to Bing',
+	'indexnow'            => 'Enable IndexNow',
+	'indexnow_key'        => 'IndexNow Key',
+	'indexnow_key_location' => 'IndexNow Key Location',
     'news_sitemap_file'   => 'News Sitemap file name',
     'news_sitemap_topics' => 'Include Articles from these Topics',
     'news_sitemap_age'    => 'Max Age of Articles',
@@ -82,7 +84,7 @@
     'fs_main' => 'XMLSitemap Main Settings',
     'fs_pri'  => 'Priority (default = 0.5, lowest = 0.0, highest = 1.0)',
     'fs_freq' => 'Update frequency',
-    'fs_ping' => 'Send ping on updating sitemap',
+    'fs_ping' => 'Send ping on change',
     'fs_news' => 'News Sitemap Settings',
 );
 

plugins/xmlsitemap/language/japanese_utf-8.php

@@ -56,7 +56,9 @@
     'priorities' => '優先度',
     'frequencies' => '頻度',
     'ping_google' => 'Googleにpingを送信する',
-    'ping_bing' => 'Bingにpingを送信する',
+	'indexnow' => 'Enable IndexNow',
+	'indexnow_key' => 'IndexNow Key',
+	'indexnow_key_location' => 'IndexNow Key Location',
     'news_sitemap_file' => 'ニュースサイトマップ名',
     'news_sitemap_topics' => 'この話題の記事を含める',
     'news_sitemap_age' => '記事の古さの最大値'

plugins/xmlsitemap/xmlsitemap.class.php

@@ -1032,10 +1032,6 @@ public function endUpdate()
             $pingTargets[] = 'google';
         }
 
-        if (isset($_XMLSMAP_CONF['ping_bing']) && $_XMLSMAP_CONF['ping_bing']) {
-            $pingTargets[] = 'bing';
-        }
-
         // Get file names
         list ($filename, $mobileFilename, $newsFilename) = $this->getFileNames();
 
@@ -1122,12 +1118,13 @@ public function sendPing(array $destinations, $filename)
 
             switch ($dest) {
                 case 'google':
-                    $url = 'http://www.google.com/ping?sitemap=' . $sitemapUrl;
-                    break;
-
-                case 'bing':
-                    $url = 'https://www.bing.com/ping?sitemap=' . $sitemapUrl;
+                    $url = 'https://www.google.com/ping?sitemap=' . $sitemapUrl;
                     break;
+
+				// See: https://github.com/Geeklog-Core/geeklog/issues/1105
+                //case 'bing':
+                //    $url = 'https://www.bing.com/ping?sitemap=' . $sitemapUrl;
+                //    break;
 
                 default:
                     $url = '';
@@ -1174,4 +1171,82 @@ public function sendPing(array $destinations, $filename)
 
         return (count($destinations) === $success);
     }
+
+    /**
+     * Submit a changed URL to IndexNow
+     *
+     * @param  string  $changedURL    the full URL to the changed page
+     * @return int                    if successful or not
+     */
+    public function submitURL($changedURL)
+    {
+        global $_CONF, $_XMLSMAP_CONF;
+
+        if (COM_isDemoMode() || !$_XMLSMAP_CONF['indexnow']) {
+            return 0;
+        }
+
+        if (empty($_XMLSMAP_CONF['indexnow_key'])) {
+            COM_errorLog(__METHOD__ . ': IndexNow key is not set.');
+
+            return 0;
+		}
+
+        if (empty($changedURL)) {
+            COM_errorLog(__METHOD__ . ': url is not specified to ping IndexNow with.');
+
+            return 0;
+        } elseif (preg_match('@\Ahttps?://localhost/@i', $_CONF['site_url'])) {
+            // It seems that 'localhost' is not accepted
+            return 0;
+        }
+
+		$_validator = Validator::getInstance();		
+		$ruleParams[] = $changedURL;
+		if (!$_validator->dispatchMethod("url", $ruleParams)) {
+            COM_errorLog(__METHOD__ . ': url "' . $changedURL . '" is not valid and therefore cannot be used by IndexNow.');
+
+            return 0;
+		}
+
+		if (!empty($_XMLSMAP_CONF['indexnow_key_location'])) {
+			unset($ruleParams);
+			$ruleParams[] = $_XMLSMAP_CONF['indexnow_key_location'];
+			if (!$_validator->dispatchMethod("url", $ruleParams)) {
+				COM_errorLog(__METHOD__ . ': key location is not a valid url and therefore cannot be used with IndexNow.');
+
+				return 0;
+			}
+
+		}
+
+		$success = 0;
+
+		// Build url
+		$url = 'https://api.indexnow.org/indexnow?url=' . urlencode($changedURL) . '&key=' . $_XMLSMAP_CONF['indexnow_key'];
+		if (!empty($_XMLSMAP_CONF['indexnow_key_location'])) {
+			$url .= '&keyLocation=' . urlencode($_XMLSMAP_CONF['indexnow_key_location']);
+		}
+
+		// Sends a ping to the endpoint of IndexNow
+		if (!empty($url)) {
+			$req = new HTTP_Request2($url, HTTP_Request2::METHOD_GET);
+
+			try {
+				$req->setHeader('User-Agent', 'Geeklog/' . VERSION);
+				$response = $req->send();
+				$status = $response->getStatus();
+
+				if ($status == 200 || $status == 202) {
+					$success++;
+				} else {
+					COM_errorLog(sprintf('Failed to send a ping to %s: HTTP status %d', $url, $status));
+				}
+			} catch (HTTP_Request2_Exception $e) {
+				COM_errorLog(__METHOD__ . ': ' . $e->getMessage());
+			}
+		}
+
+        return $success;
+    }	
 }

public_html/docs/english/xmlsitemap.html

@@ -77,7 +77,7 @@ <h3><a name="frequencies">Frequencies</a></h3>
   <!-- Note: Added multiple anchors here as a workaround so tooltip would work in Geeklog configuration for at least the default options. Unfortunately will not work for any additional plugins. -->
   <td><a name="desc_frequencies">frequencies</a><a name="desc_frequencies_article"></a><a name="desc_frequencies_polls"></a><a name="desc_frequencies_staticpages"></a><a name="desc_frequencies_calendar"></a></td>
   <td>'weekly' for staticpages, 'daily' for others</td>
-  <td>The frequency with which some type of contents are likely to be updated.  Valid values include 'always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', and 'never'.  This value will be used by search engine crawlers just to decide how often they should check the contents.  However, specifying 'always' doesn't mean that search engine crawlers will surely visit your site.  So, specifiying 'daily' or 'weekly' would be better in most cases. Specifiying 'hidden' means no frequency is displayed for the content.</td>
+  <td>The frequency with which some type of contents are likely to be updated.  Valid values include 'always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', and 'never'.  This value will be used by search engine crawlers just to decide how often they should check the contents.  However, specifying 'always' doesn't mean that search engine crawlers will surely visit your site.  So, specifying 'daily' or 'weekly' would be better in most cases. Specifying 'hidden' means no frequency is displayed for the content.</td>
 </tr>
 </table>
 
@@ -94,9 +94,19 @@ <h3><a name="ping">Ping</a></h3>
   <td>If you set this to true, the XMLSitemap plugin will automatically inform Google crawlers when you update contents. For main Sitemap file only.</td>
 </tr>
 <tr>
-  <td><a name="desc_ping_bing">ping_bing</a></td>
-  <td>True</td>
-  <td>If you set this to true, the XMLSitemap plugin will automatically inform Bing crawlers when you update contents. For main Sitemap file only.</td>
+  <td><a name="desc_indexnow">IndexNow</a></td>
+  <td>False</td>
+  <td>If you set this to true, the XMLSitemap plugin will automatically inform IndexNow by a ping that a URL and its content has been added, updated, or deleted. IndexNow will then inform search engines like Bing and Yandex of these changes.<br><br>To submit URLs, you must "prove" ownership of the website for which URLs are being submitted by hosting at least one text file within the website. Once you submit your URLs to search engines, search engines will crawl the key file to verify ownership. Only you and the search engines should know the key and your file key location.<br><br>Your key should be random numbers and letters at least 8 characters long. Create a ".txt" file with the name being your key and place it in the root directory of your website. If you choose you can specify that your key file is hosted in another folder on on your website.<br><br>For more documentation see <a href="https://www.indexnow.org/">IndexNow</a> or <a href="https://bing.com/indexnow">Bing IndexNow</a>.</td>
+</tr>
+<tr>
+  <td><a name="desc_indexnew_key">IndexNow Key</a></td>
+  <td></td>
+  <td>A key is needed to match the ownership of the domain along with submitted URLs. It should be random numbers and letters and be a minimum of 8 characters long.</td>
+</tr>
+<tr>
+  <td><a name="desc_indexnew_key">IndexNow Key Location</a></td>
+  <td></td>
+  <td>If blank, the root URL for your site is assumed as the location for the txt file which allows search engines to verify ownership. If needed you can specify a different folder within your website. The location needs to be a valid URL that includes the text file name.</td>
 </tr>
 </table>
 

public_html/docs/japanese/xmlsitemap.html

@@ -94,9 +94,19 @@ <h3><a name="ping">Ping送信</a></h3>
   <td>「はい」に設定すると、コンテンツを更新するたびにXMLSitemapプラグインはGoogleの検索エンジンにサイトマップファイルの更新を通知します。</td>
 </tr>
 <tr>
-  <td><a name="desc_ping_bing">Bingにpingを送信する(ping_bing)</a></td>
-  <td>いいえ</td>
-  <td>「はい」に設定すると、コンテンツを更新するたびにXMLSitemapプラグインはBingの検索エンジンにサイトマップファイルの更新を通知します。</td>
+  <td><a name="desc_indexnow">IndexNow</a></td>
+  <td>False</td>
+  <td>If you set this to true, the XMLSitemap plugin will automatically inform IndexNow by a ping that a URL and its content has been added, updated, or deleted. IndexNow will then inform search engines like Bing and Yandex of these changes.<br><br>To submit URLs, you must "prove" ownership of the website for which URLs are being submitted by hosting at least one text file within the website. Once you submit your URLs to search engines, search engines will crawl the key file to verify ownership. Only you and the search engines should know the key and your file key location.<br><br>Your key should be random numbers and letters at least 8 characters long. Create a ".txt" file with the name being your key and place it in the root directory of your website. If you choose you can specify that your key file is hosted in another folder on on your website.<br><br>For more documentation see <a href="https://www.indexnow.org/">IndexNow</a> or <a href="https://bing.com/indexnow">Bing IndexNow</a>.</td>
+</tr>
+<tr>
+  <td><a name="desc_indexnew_key">IndexNow Key</a></td>
+  <td></td>
+  <td>A key is needed to match the ownership of the domain along with submitted URLs. It should be random numbers and letters and be a minimum of 8 characters long.</td>
+</tr>
+<tr>
+  <td><a name="desc_indexnew_key">IndexNow Key Location</a></td>
+  <td></td>
+  <td>If blank, the root URL for your site is assumed as the location for the txt file which allows search engines to verify ownership. If needed you can specify a different folder within your website. The location needs to be a valid URL that includes the text file name.</td>
 </tr>
 </table>