New user account type "Channel Relay"

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2024.03-dev (Yellow Archangel)
--- DB_UPDATE_VERSION 1545
+-- DB_UPDATE_VERSION 1546
 -- ------------------------------------------
 
 
@@ -505,6 +505,8 @@ CREATE TABLE IF NOT EXISTS `channel` (
 	`full-text-search` varchar(1023) COMMENT 'Full text search pattern, see https://mariadb.com/kb/en/full-text-index-overview/#in-boolean-mode',
 	`media-type` smallint unsigned COMMENT 'Filtered media types',
 	`languages` mediumtext COMMENT 'Desired languages',
+	`publish` boolean COMMENT 'publish channel content',
+	`valid` boolean COMMENT 'Set, when the full-text-search is valid',
 	 PRIMARY KEY(`id`),
 	 INDEX `uid` (`uid`),
 	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
@@ -1343,7 +1345,7 @@ CREATE TABLE IF NOT EXISTS `post-engagement` (
 	`owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner',
 	`contact-type` tinyint NOT NULL DEFAULT 0 COMMENT 'Person, organisation, news, community, relay',
 	`media-type` tinyint NOT NULL DEFAULT 0 COMMENT 'Type of media in a bit array (1 = image, 2 = video, 4 = audio',
-	`language` varbinary(128) COMMENT 'Language information about this post',
+	`language` varchar(128) COMMENT 'Language information about this post',
 	`searchtext` mediumtext COMMENT 'Simplified text for the full text search',
 	`created` datetime COMMENT '',
 	`restricted` boolean NOT NULL DEFAULT '0' COMMENT 'If true, this post is either unlisted or not from a federated network',

doc/database/db_channel.md

@@ -19,6 +19,8 @@ Fields
 | full-text-search | Full text search pattern, see https://mariadb.com/kb/en/full-text-index-overview/#in-boolean-mode | varchar(1023)      | YES  |     | NULL    |                |
 | media-type       | Filtered media types                                                                              | smallint unsigned  | YES  |     | NULL    |                |
 | languages        | Desired languages                                                                                 | mediumtext         | YES  |     | NULL    |                |
+| publish          | publish channel content                                                                           | boolean            | YES  |     | NULL    |                |
+| valid            | Set, when the full-text-search is valid                                                           | boolean            | YES  |     | NULL    |                |
 
 Indexes
 ------------

doc/database/db_post-engagement.md

@@ -12,7 +12,7 @@ Fields
 | owner-id     | Item owner                                                            | int unsigned       | NO   |     | 0       |       |
 | contact-type | Person, organisation, news, community, relay                          | tinyint            | NO   |     | 0       |       |
 | media-type   | Type of media in a bit array (1 = image, 2 = video, 4 = audio         | tinyint            | NO   |     | 0       |       |
-| language     | Language information about this post                                  | varbinary(128)     | YES  |     | NULL    |       |
+| language     | Language information about this post                                  | varchar(128)       | YES  |     | NULL    |       |
 | searchtext   | Simplified text for the full text search                              | mediumtext         | YES  |     | NULL    |       |
 | created      |                                                                       | datetime           | YES  |     | NULL    |       |
 | restricted   | If true, this post is either unlisted or not from a federated network | boolean            | NO   |     | 0       |       |

src/Content/Conversation/Entity/Timeline.php

@@ -34,6 +34,8 @@
  * @property-read int    $mediaType      Media types that are included in the channel
  * @property-read array  $languages      Channel languages
  * @property-read int    $circle         Circle or timeline this channel is based on
+ * @property-read bool   $publish        Publish the channel
+ * @property-read bool   $valid          Indicates that the search conditions are valid
  */
 class Timeline extends \Friendica\BaseEntity
 {
@@ -61,8 +63,12 @@
 	protected $mediaType;
 	/** @var array */
 	protected $languages;
+	/** @var bool */
+	protected $publish;
+	/** @var bool */
+	protected $valid;
 
-	public function __construct(string $code = null, string $label = null, string $description = null, string $accessKey = null, string $path = null, int $uid = null, string $includeTags = null, string $excludeTags = null, string $fullTextSearch = null, int $mediaType = null, int $circle = null, array $languages = null)
+	public function __construct(string $code = null, string $label = null, string $description = null, string $accessKey = null, string $path = null, int $uid = null, string $includeTags = null, string $excludeTags = null, string $fullTextSearch = null, int $mediaType = null, int $circle = null, array $languages = null, bool $publish = null, bool $valid = null)
 	{
 		$this->code           = $code;
 		$this->label          = $label;
@@ -76,5 +82,7 @@
 		$this->mediaType      = $mediaType;
 		$this->circle         = $circle;
 		$this->languages      = $languages;
+		$this->publish        = $publish;
+		$this->valid          = $valid;
 	}
 }

src/Content/Conversation/Factory/UserDefinedChannel.php

@@ -50,6 +50,8 @@
 			$row['media-type'] ?? null,
 			$row['circle'] ?? null,
 			$row['languages'] ?? null,
+			$row['publish'] ?? null,
+			$row['valid'] ?? null,
 		);
 	}
 }

src/Content/Conversation/Repository/UserDefinedChannel.php

@@ -25,24 +25,27 @@
 use Friendica\Content\Conversation\Collection\UserDefinedChannels;
 use Friendica\Content\Conversation\Entity;
 use Friendica\Content\Conversation\Factory;
-use Friendica\Core\PConfig\Capability\IManagePersonalConfigValues;
+use Friendica\Core\Config\Capability\IManageConfigValues;
 use Friendica\Database\Database;
+use Friendica\Database\DBA;
+use Friendica\Model\Contact;
 use Friendica\Model\Post\Engagement;
 use Friendica\Model\User;
+use Friendica\Util\DateTimeFormat;
 use Psr\Log\LoggerInterface;
 
 class UserDefinedChannel extends \Friendica\BaseRepository
 {
 	protected static $table_name = 'channel';
 
-	/** @var IManagePersonalConfigValues */
-	private $pConfig;
+	/** @var IManageConfigValues */
+	private $config;
 
-	public function __construct(Database $database, LoggerInterface $logger, Factory\UserDefinedChannel $factory, IManagePersonalConfigValues $pConfig)
+	public function __construct(Database $database, LoggerInterface $logger, Factory\UserDefinedChannel $factory, IManageConfigValues $config)
 	{
 		parent::__construct($database, $logger, $factory);
 
-		$this->pConfig = $pConfig;
+		$this->config = $config;
 	}
 
 	/**
@@ -63,6 +66,11 @@
 		return $Entities;
 	}
 
+	public function select(array $condition, array $params = []): UserDefinedChannels
+	{
+		return $this->_select($condition, $params);
+	}
+
 	/**
 	 * Fetch a single user channel
 	 *
@@ -125,6 +133,8 @@
 			'full-text-search' => $Channel->fullTextSearch,
 			'media-type'       => $Channel->mediaType,
 			'languages'        => serialize($Channel->languages),
+			'publish'          => $Channel->publish,
+			'valid'            => $this->isValid($Channel->fullTextSearch),
 		];
 
 		if ($Channel->code) {
@@ -140,40 +150,185 @@
 		return $Channel;
 	}
 
+	private function isValid(string $searchtext): bool
+	{
+		if ($searchtext == '') {
+			return true;
+		}
+
+		return $this->db->select('check-full-text-search', [], ["`pid` = ? AND MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE)", getmypid(), $this->escapeKeywords($searchtext)]) !== false;
+	}
+
 	/**
-	 * Checks, if one of the user defined channels matches with the given search text
-	 * @todo To increase the performance, this functionality should be replaced with a single SQL call.
+	 * Checks, if one of the user defined channels matches with the given search text or languages
 	 *
 	 * @param string $searchtext
 	 * @param string $language
 	 * @return boolean
 	 */
 	public function match(string $searchtext, string $language): bool
 	{
+		$users = $this->db->selectToArray('user', ['uid'], $this->getUserCondition());
+		if (empty($users)) {
+			return [];
+		}
+
+		$uids = array_column($users, 'uid');
+
+		$usercondition = ['uid' => $uids];
+		$condition = DBA::mergeConditions($usercondition, ["`languages` != ? AND `include-tags` = ? AND `full-text-search` = ? AND `circle` = ?", '', '', '', 0]);
+		foreach ($this->select($condition) as $channel) {
+			if (!empty($channel->languages) && in_array($language, $channel->languages)) {
+				return true;
+			}
+		}
+
+		$search = '';
+		$condition = DBA::mergeConditions($usercondition, ["`full-text-search` != ? AND `circle` = ? AND `valid`", '', 0]);
+		foreach ($this->select($condition) as $channel) {
+			$search .= '(' . $channel->fullTextSearch . ') ';
+		}
+
+		$this->insertCheckFullTextSearch($searchtext);
+		$result = $this->inFulltext($search);
+		$this->deleteCheckFullTextSearch();
+
+		return $result;
+	}
+
+	/**
+	 * Fetch the channel users that have got matching channels
+	 *
+	 * @param string $searchtext
+	 * @param string $language
+	 * @param array  $tags
+	 * @param int    $media_type
+	 * @param int    $owner_id
+	 * @param int    $reshare_id
+	 * @return array
+	 */
+	public function getMatchingChannelUsers(string $searchtext, string $language, array $tags, int $media_type, int $owner_id, int $reshare_id): array
+	{
+		$condition = $this->getUserCondition();
+		$condition = DBA::mergeConditions($condition, ["`account-type` IN (?, ?) AND `uid` != ?", User::ACCOUNT_TYPE_RELAY, User::ACCOUNT_TYPE_COMMUNITY, 0]);
+		$users = $this->db->selectToArray('user', ['uid'], $condition);
+		if (empty($users)) {
+			return [];
+		}
+
 		if (!in_array($language, User::getLanguages())) {
 			$this->logger->debug('Unwanted language found. No matched channel found.', ['language' => $language, 'searchtext' => $searchtext]);
-			return false;
+			return [];
 		}
 
-		$store = false;
-		$this->db->insert('check-full-text-search', ['pid' => getmypid(), 'searchtext' => $searchtext], Database::INSERT_UPDATE);
-		$channels = $this->db->select(self::$table_name, ['full-text-search', 'uid', 'label'], ["`full-text-search` != ? AND `circle` = ?", '', 0]);
-		while ($channel = $this->db->fetch($channels)) {
-			$channelsearchtext = $channel['full-text-search'];
-			foreach (Engagement::KEYWORDS as $keyword) {
-				$channelsearchtext = preg_replace('~(' . $keyword . ':.[\w@\.-]+)~', '"$1"', $channelsearchtext);
+		$this->insertCheckFullTextSearch($searchtext);
+
+		$uids = [];
+
+		foreach ($this->select(['uid' => array_column($users, 'uid'), 'publish' => true, 'valid' => true]) as $channel) {
+			if (in_array($channel->uid, $uids)) {
+				continue;
 			}
-			if ($this->db->exists('check-full-text-search', ["`pid` = ? AND MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE)", getmypid(), $channelsearchtext])) {
-				if (in_array($language, $this->pConfig->get($channel['uid'], 'channel', 'languages', [User::getLanguageCode($channel['uid'])]))) {
-					$store = true;
-					$this->logger->debug('Matching channel found.', ['uid' => $channel['uid'], 'label' => $channel['label'], 'language' => $language, 'channelsearchtext' => $channelsearchtext, 'searchtext' => $searchtext]);
-					break;
+			if (!empty($channel->circle) && ($channel->circle > 0) && !in_array($channel->uid, $uids)) {
+				if (!$this->inCircle($channel->circle, $channel->uid, $owner_id) && !$this->inCircle($channel->circle, $channel->uid, $reshare_id)) {
+					continue;
 				}
 			}
+			if (!empty($channel->languages) && !in_array($channel->uid, $uids)) {
+				if (!in_array($language, $channel->languages)) {
+					continue;
+				}
+			} elseif (!in_array($language, User::getWantedLanguages($channel->uid))) {
+				continue;
+			}
+			if (!empty($channel->includeTags) && !in_array($channel->uid, $uids)) {
+				if (!$this->inTaglist($channel->includeTags, $tags)) {
+					continue;
+				}
+			}
+			if (!empty($channel->excludeTags) && !in_array($channel->uid, $uids)) {
+				if ($this->inTaglist($channel->excludeTags, $tags)) {
+					continue;
+				}
+			}
+			if (!empty($channel->mediaType) && !in_array($channel->uid, $uids)) {
+				if (!($channel->mediaType & $media_type)) {
+					continue;
+				}
+			}
+			if (!empty($channel->fullTextSearch) && !in_array($channel->uid, $uids)) {
+				if (!$this->inFulltext($channel->fullTextSearch)) {
+					continue;
+				}
+			}
+			$uids[] = $channel->uid;
+			$this->logger->debug('Matching channel found.', ['uid' => $channel->uid, 'label' => $channel->label, 'language' => $language, 'tags' => $tags, 'media_type' => $media_type, 'searchtext' => $searchtext]);
 		}
-		$this->db->close($channels);
 
+		$this->deleteCheckFullTextSearch();
+		return $uids;
+	}
+
+	private function insertCheckFullTextSearch(string $searchtext)
+	{
+		$this->db->insert('check-full-text-search', ['pid' => getmypid(), 'searchtext' => $searchtext], Database::INSERT_UPDATE);
+	}
+
+	private function deleteCheckFullTextSearch()
+	{
 		$this->db->delete('check-full-text-search', ['pid' => getmypid()]);
-		return $store;
+	}
+
+	private function inCircle(int $circleId, int $uid, int $cid): bool
+	{
+		if ($cid == 0) {
+			return false;
+		}
+
+		$account = Contact::selectFirstAccountUser(['id'], ['pid' => $cid, 'uid' => $uid]);
+		if (empty($account['id'])) {
+			return false;
+		}
+		return $this->db->exists('group_member', ['gid' => $circleId, 'contact-id' => $account['id']]);
+	}
+
+	private function inTaglist(string $tagList, array $tags): bool
+	{
+		if (empty($tags)) {
+			return false;
+		}
+		array_walk($tags, function (&$value) {
+			$value = mb_strtolower($value);
+		});
+		foreach (explode(',', $tagList) as $tag) {
+			if (in_array($tag, $tags)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	private function inFulltext(string $fullTextSearch): bool
+	{
+		return $this->db->exists('check-full-text-search', ["`pid` = ? AND MATCH (`searchtext`) AGAINST (? IN BOOLEAN MODE)", getmypid(), $this->escapeKeywords($fullTextSearch)]);
+	}
+
+	private function escapeKeywords(string $fullTextSearch): string
+	{
+		foreach (Engagement::KEYWORDS as $keyword) {
+			$fullTextSearch = preg_replace('~(' . $keyword . ':.[\w@\.-]+)~', '"$1"', $fullTextSearch);
+		}
+		return $fullTextSearch;
+	}
+
+	private function getUserCondition()
+	{
+		$condition = ["`verified` AND NOT `blocked` AND NOT `account_removed` AND NOT `account_expired` AND `user`.`uid` > ?", 0];
+
+		$abandon_days = intval($this->config->get('system', 'account_abandon_days'));
+		if (!empty($abandon_days)) {
+			$condition = DBA::mergeConditions($condition, ["`last-activity` > ?", DateTimeFormat::utc('now - ' . $abandon_days . ' days')]);
+		}
+		return $condition;
 	}
 }

src/Content/Text/BBCode.php

@@ -258,6 +258,10 @@
 		// Add images because of possible alt texts
 		if (!empty($uri_id)) {
 			$text = Post\Media::addAttachmentsToBody($uri_id, $text, [Post\Media::IMAGE]);
+
+			foreach (Post\Media::getByURIId($uri_id, [Post\Media::HTML]) as $media) {
+				$text .= ' ' . $media['name'] . ' ' . $media['description'];
+			}
 		}
 
 		if (empty($text)) {
@@ -279,7 +283,7 @@
 		// Removes mentions, remove links from hashtags
 		$text = preg_replace('/[@!]\[url\=.*?\].*?\[\/url\]/ism', ' ', $text);
 		$text = preg_replace('/[#]\[url\=.*?\](.*?)\[\/url\]/ism', ' #$1 ', $text);
-		$text = preg_replace('/[@!#]?\[url.*?\[\/url\]/ism', ' ', $text);
+		$text = preg_replace('/[@!#]+?\[url.*?\[\/url\]/ism', ' ', $text);
 		$text = preg_replace("/\[url=[^\[\]]*\](.*)\[\/url\]/Usi", ' $1 ', $text);
 
 		// Convert it to plain text

src/Core/L10n.php

@@ -65,6 +65,9 @@
 		'zh-cn' => '简体中文',
 	];
 
+	/** @var string Undetermined language */
+	const UNDETERMINED_LANGUAGE = 'un';
+
 	/**
 	 * A string indicating the current language used for translation:
 	 * - Two-letter ISO 639-1 code.
@@ -436,7 +439,9 @@
 	{
 		$iso639 = new \Matriphe\ISO639\ISO639;
 
-		$languages = [];
+		// In ISO 639-2 undetermined languages have got the code "und".
+		// There is no official code for ISO 639-1, but "un" is not assigned to any language.   
+		$languages = [self::UNDETERMINED_LANGUAGE => $this->t('Undetermined')];
 
 		foreach ($this->getDetectableLanguages() as $code) {
 			$code     = $this->toISO6391($code);

src/Model/Contact.php

@@ -172,6 +172,11 @@
 		return DBA::selectFirst('account-view', $fields, $condition, $params);
 	}
 
+	public static function selectFirstAccountUser(array $fields = [], array $condition = [], array $params = [])
+	{
+		return DBA::selectFirst('account-user-view', $fields, $condition, $params);
+	}
+
 	/**
 	 * Insert a row into the contact table
 	 * Important: You can't use DBA::lastInsertId() after this call since it will be set to 0.