MD5-based caching

simplepie#401
FreshRSS/FreshRSS@9aab83a
FreshRSS/FreshRSS@00127f0
FreshRSS/FreshRSS#3416

src/SimplePie.php

@@ -1634,6 +1634,39 @@ public function enable_exceptions(bool $enable = true)
         $this->enable_exceptions = $enable;
     }
 
+    /**
+     * Compute a hash (md5) for caching,
+     * after having removed the noise from the raw feed (e.g. comments) to improve caching.
+     *
+     * @param string $rss The raw feed content
+     * @return string The hash of the cleaned content
+     */
+    private function clean_hash(string $rss): string
+    {
+        //Process by chunks not to use too much memory
+        if (($stream = fopen('php://temp', 'r+'))
+            && fwrite($stream, $rss)
+            && rewind($stream)
+        ) {
+            $ctx = hash_init('md5');
+            while ($stream_data = fread($stream, 1048576)) {
+                hash_update(
+                    $ctx, preg_replace(
+                        [
+                            '#<(lastBuildDate|pubDate|updated|feedDate|dc:date|slash:comments)>[^<]+</\\1>#',
+                            '#<(media:starRating|media:statistics) [^/<>]+/>#',
+                            '#<!--.+?-->#s',
+                        ],
+                        '', $stream_data
+                    )
+                );
+            }
+            fclose($stream);
+            return hash_final($ctx);
+        }
+        return '';
+    }
+
     /**
      * Initialize the feed object
      *
@@ -1735,6 +1768,9 @@ public function init()
             }
 
             [$headers, $sniffed] = $fetched;
+            if (isset($this->data['md5'])) { // FreshRSS
+                $md5 = $this->data['md5'];
+            }
         }
 
         // Empty response check
@@ -1803,6 +1839,8 @@ public function init()
                         $this->data['headers'] = $headers;
                     }
                     $this->data['build'] = Misc::get_build();
+                    $this->data['md5'] = $md5 === '' ? $this->clean_md5($this->raw_data) : $md5; // FreshRSS
+                    $this->data['mtime'] = time(); // FreshRSS
 
                     // Cache the file if caching is enabled
                     $this->data['cache_expiration_time'] = $this->cache_duration + time();