Skip to content

Commit

Permalink
JSON column for feeds (FreshRSS#1838)
Browse files Browse the repository at this point in the history 
* Draft of JSON column for feeds
FreshRSS#1654

* Add some per-feed options
  * Feed cURL timeout
  * Mark updated articles as read FreshRSS#891
  * Mark as read upon reception FreshRSS#1702
  * Ignore SSL (unsafe) FreshRSS#1811

* Try PHPCS workaround
While waiting for a better syntax support
  • Loading branch information
@Alkarex
Alkarex committed May 1, 2018
1 parent 8dfc46c commit 6938aaa
Show file tree
Hide file tree
Showing 27 changed files with 242 additions and 26 deletions.
15 changes: 10 additions & 5 deletions app/Controllers/feedController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ public static function addFeed($url, $title = '', $cat_id = 0, $new_cat_name = '
'description' => $feed->description(),
'lastUpdate' => time(),
'httpAuth' => $feed->httpAuth(),
'attributes' => array(),
);

$id = $feedDAO->addFeed($values);
Expand Down Expand Up @@ -271,7 +272,6 @@ public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush

$updated_feeds = 0;
$nb_new_articles = 0;
$is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0;
foreach ($feeds as $feed) {
$url = $feed->url(); //For detection of HTTP 301

Expand Down Expand Up @@ -353,8 +353,10 @@ public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush
} else { //This entry already exists but has been updated
//Minz_Log::debug('Entry with GUID `' . $entry->guid() . '` updated in feed ' . $feed->id() .
//', old hash ' . $existingHash . ', new hash ' . $entry->hash());
//TODO: Make an updated/is_read policy by feed, in addition to the global one.
$needFeedCacheRefresh = FreshRSS_Context::$user_conf->mark_updated_article_unread;
$mark_updated_article_unread = $feed->attributes('mark_updated_article_unread') !== null ? (
$feed->attributes('mark_updated_article_unread')
) : FreshRSS_Context::$user_conf->mark_updated_article_unread;
$needFeedCacheRefresh = $mark_updated_article_unread;
$entry->_isRead(FreshRSS_Context::$user_conf->mark_updated_article_unread ? false : null); //Change is_read according to policy.
if (!$entryDAO->inTransaction()) {
$entryDAO->beginTransaction();
Expand All @@ -365,15 +367,18 @@ public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush
// This entry should not be added considering configuration and date.
$oldGuids[] = $entry->guid();
} else {
$read_upon_reception = $feed->attributes('read_upon_reception') !== null ? (
$feed->attributes('read_upon_reception')
) : FreshRSS_Context::$user_conf->mark_when['reception'];
if ($isNewFeed) {
$id = min(time(), $entry_date) . uSecString();
$entry->_isRead($is_read);
$entry->_isRead($read_upon_reception);
} elseif ($entry_date < $date_min) {
$id = min(time(), $entry_date) . uSecString();
$entry->_isRead(true); //Old article that was not in database. Probably an error, so mark as read
} else {
$id = uTimeString();
$entry->_isRead($is_read);
$entry->_isRead($read_upon_reception);
}
$entry->_id($id);

Expand Down
19 changes: 14 additions & 5 deletions app/Controllers/subscriptionController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public function firstAction() {
}

$catDAO = new FreshRSS_CategoryDAO();
$feedDAO = new FreshRSS_FeedDAO();
$feedDAO = FreshRSS_Factory::createFeedDao();

$catDAO->checkDefault();
$feedDAO->updateTTL();
Expand Down Expand Up @@ -74,9 +74,10 @@ public function feedAction() {
return;
}

$this->view->feed = $this->view->feeds[$id];
$feed = $this->view->feeds[$id];
$this->view->feed = $feed;

Minz_View::prependTitle(_t('sub.title.feed_management') . ' · ' . $this->view->feed->name() . ' · ');
Minz_View::prependTitle(_t('sub.title.feed_management') . ' · ' . $feed->name() . ' · ');

if (Minz_Request::isPost()) {
$user = trim(Minz_Request::param('http_user_feed' . $id, ''));
Expand All @@ -95,6 +96,13 @@ public function feedAction() {
$ttl = FreshRSS_Context::$user_conf->ttl_default;
}

$feed->_attributes('mark_updated_article_unread', Minz_Request::paramTernary('mark_updated_article_unread'));
$feed->_attributes('read_upon_reception', Minz_Request::paramTernary('read_upon_reception'));
$feed->_attributes('ssl_verify', Minz_Request::paramTernary('ssl_verify'));

$timeout = intval(Minz_Request::param('timeout', 0));
$feed->_attributes('timeout', $timeout > 0 ? $timeout : null);

$values = array(
'name' => Minz_Request::param('name', ''),
'description' => sanitizeHTML(Minz_Request::param('description', '', true)),
Expand All @@ -106,14 +114,15 @@ public function feedAction() {
'httpAuth' => $httpAuth,
'keep_history' => intval(Minz_Request::param('keep_history', FreshRSS_Feed::KEEP_HISTORY_DEFAULT)),
'ttl' => $ttl * ($mute ? -1 : 1),
'attributes' => $feed->attributes()
);

invalidateHttpCache();

$url_redirect = array('c' => 'subscription', 'params' => array('id' => $id));
if ($feedDAO->updateFeed($id, $values) !== false) {
$this->view->feed->_category($cat);
$this->view->feed->faviconPrepare();
$feed->_category($cat);
$feed->faviconPrepare();

Minz_Request::good(_t('feedback.sub.feed.updated'), $url_redirect);
} else {
Expand Down
2 changes: 1 addition & 1 deletion app/Models/DatabaseDAO.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public function categoryIsCorrect() {
public function feedIsCorrect() {
return $this->checkTable('feed', array(
'id', 'url', 'category', 'name', 'website', 'description', 'lastUpdate',
'priority', 'pathEntries', 'httpAuth', 'error', 'keep_history', 'ttl',
'priority', 'pathEntries', 'httpAuth', 'error', 'keep_history', 'ttl', 'attributes',
'cache_nbEntries', 'cache_nbUnreads'
));
}
Expand Down
8 changes: 7 additions & 1 deletion app/Models/Factory.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,13 @@
class FreshRSS_Factory {

public static function createFeedDao($username = null) {
return new FreshRSS_FeedDAO($username);
$conf = Minz_Configuration::get('system');
switch ($conf->db['type']) {
case 'sqlite':
return new FreshRSS_FeedDAOSQLite($username);
default:
return new FreshRSS_FeedDAO($username);
}
}

public static function createEntryDao($username = null) {
Expand Down
26 changes: 25 additions & 1 deletion app/Models/Feed.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ class FreshRSS_Feed extends Minz_Model {
private $error = false;
private $keep_history = self::KEEP_HISTORY_DEFAULT;
private $ttl = self::TTL_DEFAULT;
private $attributes = array();
private $mute = false;
private $hash = null;
private $lockPath = '';
Expand Down Expand Up @@ -114,6 +115,13 @@ public function keepHistory() {
public function ttl() {
return $this->ttl;
}
public function attributes($key = '') {
if ($key == '') {
return $this->attributes;
} else {
return isset($this->attributes[$key]) ? $this->attributes[$key] : null;
}
}
public function mute() {
return $this->mute;
}
Expand Down Expand Up @@ -234,6 +242,22 @@ public function _ttl($value) {
$this->ttl = abs($value);
$this->mute = $value < self::TTL_DEFAULT;
}

public function _attributes($key, $value) {
if ($key == '') {
if (is_string($value)) {
$value = json_decode($value, true);
}
if (is_array($value)) {
$this->attributes = $value;
}
} elseif ($value === null) {
unset($this->attributes[$key]);
} else {
$this->attributes[$key] = $value;
}
}

public function _nbNotRead($value) {
$this->nbNotRead = intval($value);
}
Expand All @@ -253,7 +277,7 @@ public function load($loadDetails = false, $noCache = false) {
if ($this->httpAuth != '') {
$url = preg_replace('#((.+)://)(.+)#', '${1}' . $this->httpAuth . '@${3}', $url);
}
$feed = customSimplePie();
$feed = customSimplePie($this->attributes());
if (substr($url, -11) === '#force_feed') {
$feed->force_feed(true);
$url = substr($url, 0, -11);
Expand Down
73 changes: 65 additions & 8 deletions app/Models/FeedDAO.php
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,33 @@
<?php

class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {

protected function addColumn($name) {
Minz_Log::warning('FreshRSS_FeedDAO::addColumn: ' . $name);
try {
if ($name === 'attributes') { //v1.11.0
$stm = $this->bd->prepare('ALTER TABLE `' . $this->prefix . 'feed` ADD COLUMN attributes TEXT');
return $stm && $stm->execute();
}
} catch (Exception $e) {
Minz_Log::error('FreshRSS_FeedDAO::addColumn error: ' . $e->getMessage());
}
return false;
}

protected function autoUpdateDb($errorInfo) {
if (isset($errorInfo[0])) {
if ($errorInfo[0] === '42S22' || $errorInfo[0] === '42703') { //ER_BAD_FIELD_ERROR (Mysql), undefined_column (PostgreSQL)
foreach (array('attributes') as $column) {
if (stripos($errorInfo[2], $column) !== false) {
return $this->addColumn($column);
}
}
}
}
return false;
}

public function addFeed($valuesTmp) {
$sql = '
INSERT INTO `' . $this->prefix . 'feed`
Expand All @@ -15,10 +42,11 @@ public function addFeed($valuesTmp) {
`httpAuth`,
error,
keep_history,
ttl
ttl,
attributes
)
VALUES
(?, ?, ?, ?, ?, ?, 10, ?, 0, ?, ?)';
(?, ?, ?, ?, ?, ?, 10, ?, 0, ?, ?, ?)';
$stm = $this->bd->prepare($sql);

$valuesTmp['url'] = safe_ascii($valuesTmp['url']);
Expand All @@ -34,12 +62,16 @@ public function addFeed($valuesTmp) {
base64_encode($valuesTmp['httpAuth']),
FreshRSS_Feed::KEEP_HISTORY_DEFAULT,
FreshRSS_Feed::TTL_DEFAULT,
isset($valuesTmp['attributes']) ? json_encode($valuesTmp['attributes']) : '',
);

if ($stm && $stm->execute($values)) {
return $this->bd->lastInsertId('"' . $this->prefix . 'feed_id_seq"');
} else {
$info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo();
if ($this->autoUpdateDb($info)) {
return $this->addFeed($valuesTmp);
}
Minz_Log::error('SQL error addFeed: ' . $info[2]);
return false;
}
Expand All @@ -60,7 +92,8 @@ public function addFeedObject($feed) {
'website' => $feed->website(),
'description' => $feed->description(),
'lastUpdate' => 0,
'httpAuth' => $feed->httpAuth()
'httpAuth' => $feed->httpAuth(),
'attributes' => $feed->attributes(),
);

$id = $this->addFeed($values);
Expand All @@ -87,8 +120,10 @@ public function updateFeed($id, $valuesTmp) {
foreach ($valuesTmp as $key => $v) {
$set .= '`' . $key . '`=?, ';

if ($key == 'httpAuth') {
if ($key === 'httpAuth') {
$valuesTmp[$key] = base64_encode($v);
} elseif ($key === 'attributes') {
$valuesTmp[$key] = json_encode($v);
}
}
$set = substr($set, 0, -2);
Expand All @@ -105,11 +140,25 @@ public function updateFeed($id, $valuesTmp) {
return $stm->rowCount();
} else {
$info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo();
if ($this->autoUpdateDb($info)) {
return $this->updateFeed($id, $valuesTmp);
}
Minz_Log::error('SQL error updateFeed: ' . $info[2] . ' for feed ' . $id);
return false;
}
}

public function updateFeedAttribute($feed, $key, $value) {
if ($feed instanceof FreshRSS_Feed) {
$feed->_attributes($key, $value);
return $this->updateFeed(
$feed->id(),
array('attributes' => $feed->attributes())
);
}
return false;
}

public function updateLastUpdate($id, $inError = false, $mtime = 0) { //See also updateCachedValue()
$sql = 'UPDATE `' . $this->prefix . 'feed` '
. 'SET `lastUpdate`=?, error=? '
Expand Down Expand Up @@ -252,15 +301,22 @@ public function arrayFeedCategoryNames() { //For API
*/
public function listFeedsOrderUpdate($defaultCacheDuration = 3600) {
$this->updateTTL();
$sql = 'SELECT id, url, name, website, `lastUpdate`, `pathEntries`, `httpAuth`, keep_history, ttl '
$sql = 'SELECT id, url, name, website, `lastUpdate`, `pathEntries`, `httpAuth`, keep_history, ttl, attributes '
. 'FROM `' . $this->prefix . 'feed` '
. ($defaultCacheDuration < 0 ? '' : 'WHERE ttl >= ' . FreshRSS_Feed::TTL_DEFAULT
. ' AND `lastUpdate` < (' . (time() + 60) . '-(CASE WHEN ttl=' . FreshRSS_Feed::TTL_DEFAULT . ' THEN ' . intval($defaultCacheDuration) . ' ELSE ttl END)) ')
. 'ORDER BY `lastUpdate`';
$stm = $this->bd->prepare($sql);
$stm->execute();

return self::daoToFeed($stm->fetchAll(PDO::FETCH_ASSOC));
if ($stm && $stm->execute()) {
return self::daoToFeed($stm->fetchAll(PDO::FETCH_ASSOC));
} else {
$info = $stm == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $stm->errorInfo();
if ($this->autoUpdateDb($info)) {
return $this->listFeedsOrderUpdate($defaultCacheDuration);
}
Minz_Log::error('SQL error listFeedsOrderUpdate: ' . $info[2]);
return array();
}
}

public function listByCategory($cat) {
Expand Down Expand Up @@ -385,6 +441,7 @@ public static function daoToFeed($listDAO, $catID = null) {
$myFeed->_error(isset($dao['error']) ? $dao['error'] : 0);
$myFeed->_keepHistory(isset($dao['keep_history']) ? $dao['keep_history'] : FreshRSS_Feed::KEEP_HISTORY_DEFAULT);
$myFeed->_ttl(isset($dao['ttl']) ? $dao['ttl'] : FreshRSS_Feed::TTL_DEFAULT);
$myFeed->_attributes('', isset($dao['attributes']) ? $dao['attributes'] : '');
$myFeed->_nbNotRead(isset($dao['cache_nbUnreads']) ? $dao['cache_nbUnreads'] : 0);
$myFeed->_nbEntries(isset($dao['cache_nbEntries']) ? $dao['cache_nbEntries'] : 0);
if (isset($dao['id'])) {
Expand Down
17 changes: 17 additions & 0 deletions app/Models/FeedDAOSQLite.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
<?php

class FreshRSS_FeedDAOSQLite extends FreshRSS_FeedDAO {

protected function autoUpdateDb($errorInfo) {
if ($tableInfo = $this->bd->query("PRAGMA table_info('feed')")) {
$columns = $tableInfo->fetchAll(PDO::FETCH_COLUMN, 1);
foreach (array('attributes') as $column) {
if (!in_array($column, $columns)) {
return $this->addColumn($column);
}
}
}
return false;
}

}
1 change: 1 addition & 0 deletions app/SQL/install.sql.mysql.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
`error` boolean DEFAULT 0,
`keep_history` MEDIUMINT NOT NULL DEFAULT -2, -- v0.7
`ttl` INT NOT NULL DEFAULT 0, -- v0.7.3
`attributes` TEXT, -- v1.11.0
`cache_nbEntries` int DEFAULT 0, -- v0.7
`cache_nbUnreads` int DEFAULT 0, -- v0.7
PRIMARY KEY (`id`),
Expand Down
1 change: 1 addition & 0 deletions app/SQL/install.sql.pgsql.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
"error" smallint DEFAULT 0,
"keep_history" INT NOT NULL DEFAULT -2,
"ttl" INT NOT NULL DEFAULT 0,
"attributes" TEXT, -- v1.11.0
"cache_nbEntries" INT DEFAULT 0,
"cache_nbUnreads" INT DEFAULT 0,
FOREIGN KEY ("category") REFERENCES "%1$scategory" ("id") ON DELETE SET NULL ON UPDATE CASCADE
Expand Down
1 change: 1 addition & 0 deletions app/SQL/install.sql.sqlite.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
`error` boolean DEFAULT 0,
`keep_history` MEDIUMINT NOT NULL DEFAULT -2,
`ttl` INT NOT NULL DEFAULT 0,
`attributes` TEXT, -- v1.11.0
`cache_nbEntries` int DEFAULT 0,
`cache_nbUnreads` int DEFAULT 0,
FOREIGN KEY (`category`) REFERENCES `category`(`id`) ON DELETE SET NULL ON UPDATE CASCADE,
Expand Down
2 changes: 2 additions & 0 deletions app/i18n/cz/sub.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,10 @@
'main_stream' => 'Zobrazit ve “Všechny kanály”',
'normal' => 'Show in its category', // TODO
),
'ssl_verify' => 'Verify SSL security', //TODO
'stats' => 'Statistika',
'think_to_add' => 'Můžete přidat kanály.',
'timeout' => 'Timeout in seconds', //TODO
'title' => 'Název',
'title_add' => 'Přidat RSS kanál',
'ttl' => 'Neobnovovat častěji než',
Expand Down
2 changes: 2 additions & 0 deletions app/i18n/de/sub.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,10 @@
'main_stream' => 'In Haupt-Feeds zeigen',
'normal' => 'Zeige in eigener Kategorie',
),
'ssl_verify' => 'Verify SSL security', //TODO
'stats' => 'Statistiken',
'think_to_add' => 'Sie können Feeds hinzufügen.',
'timeout' => 'Timeout in seconds', //TODO
'title' => 'Titel',
'title_add' => 'Einen RSS-Feed hinzufügen',
'ttl' => 'Aktualisiere automatisch nicht öfter als',
Expand Down
Loading

0 comments on commit 6938aaa

Please sign in to comment.