Get favicons for feeds, well, some/most of them

spudooli · Jun 18, 2022 · 1fe7e49 · 1fe7e49
1 parent a7b0649
commit 1fe7e49
Show file tree

Hide file tree

Showing 43 changed files with 55 additions and 6 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,7 @@
 deploy.sh
 flaskrun.sh
+static/icons/
+
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

diff --git a/bin/get-favicons.py b/bin/get-favicons.py
@@ -0,0 +1,43 @@
+import mysql.connector
+import os
+import requests
+import favicon
+
+connection = mysql.connector.connect(
+    host="localhost",
+    user="root",
+    password="bobthefish",
+    database="reader",
+)
+
+def download(domain,  id):
+    useragent_headers = {'User-Agent': 'Spudooli Reader/1.0 - favicon scraper - https://reader.spudooli.com/about'}
+    icons = favicon.get(domain, headers=useragent_headers)
+    icon = icons[0]
+
+    response = requests.get(icon.url, headers=useragent_headers, stream=True)
+    icon_file = f'{str(id)}.{icon.format}'
+    with open('/var/www/reader/reader/static/icons/' + icon_file, 'wb') as image:
+        for chunk in response.iter_content(1024):
+            image.write(chunk)
+
+    try:
+        os.system(f"convert /var/www/reader/reader/static/icons/{icon_file} -thumbnail 64x64 -flatten /var/www/reader/reader/static/icons/{id}.gif")
+    except:
+        print(f"failed to convert {icon_file}") 
+        pass
+
+if __name__=='__main__':
+
+    cursor = connection.cursor(buffered = True)
+    cursor.execute("SELECT id, websiteurl FROM feeds")
+
+    for row in cursor:
+        try:
+            print(row[1])
+            download(row[1], row[0])
+        except Exception as e:
+            print(e)
+            print("Error processing feed: --------------------------------------------" + row[1])
+            pass
+    cursor.close()  
diff --git a/bin/update-feeds.py b/bin/update-feeds.py
@@ -5,7 +5,7 @@
 from datetime import datetime
 from time import strftime
 
-feedparser.USER_AGENT = "Spudooli Reader/1.0 - 1 subscribers - https://www.spudooli.com"
+feedparser.USER_AGENT = "Spudooli Reader/1.0 - 1 subscribers - https://reader.spudooli.com/about"
 
 if hasattr(ssl, '_create_unverified_context'):
     ssl._create_default_https_context = ssl._create_unverified_context

diff --git a/reader/__init__.py b/reader/__init__.py
@@ -1,5 +1,6 @@
 from flask import Flask
 from datetime import datetime
+import os
 
 
 app = Flask(__name__)
@@ -13,6 +14,4 @@
 def inject_now():
     return {'now': datetime.utcnow()}
 
-
-
 import reader.main
diff --git a/reader/static/icons/1.gif b/reader/static/icons/1.gif
diff --git a/reader/static/icons/11.gif b/reader/static/icons/11.gif
diff --git a/reader/static/icons/12.gif b/reader/static/icons/12.gif
diff --git a/reader/static/icons/13.gif b/reader/static/icons/13.gif
diff --git a/reader/static/icons/14.gif b/reader/static/icons/14.gif
diff --git a/reader/static/icons/15.gif b/reader/static/icons/15.gif
diff --git a/reader/static/icons/18.gif b/reader/static/icons/18.gif
diff --git a/reader/static/icons/19.gif b/reader/static/icons/19.gif
diff --git a/reader/static/icons/2.gif b/reader/static/icons/2.gif
diff --git a/reader/static/icons/20.gif b/reader/static/icons/20.gif
diff --git a/reader/static/icons/22.gif b/reader/static/icons/22.gif
diff --git a/reader/static/icons/24.gif b/reader/static/icons/24.gif
diff --git a/reader/static/icons/26.gif b/reader/static/icons/26.gif
diff --git a/reader/static/icons/28.gif b/reader/static/icons/28.gif
diff --git a/reader/static/icons/29.gif b/reader/static/icons/29.gif
diff --git a/reader/static/icons/3.gif b/reader/static/icons/3.gif
diff --git a/reader/static/icons/33.gif b/reader/static/icons/33.gif
diff --git a/reader/static/icons/37.gif b/reader/static/icons/37.gif
diff --git a/reader/static/icons/38.gif b/reader/static/icons/38.gif
diff --git a/reader/static/icons/39.gif b/reader/static/icons/39.gif
diff --git a/reader/static/icons/4.gif b/reader/static/icons/4.gif
diff --git a/reader/static/icons/40.gif b/reader/static/icons/40.gif
diff --git a/reader/static/icons/42.gif b/reader/static/icons/42.gif
diff --git a/reader/static/icons/45.gif b/reader/static/icons/45.gif
diff --git a/reader/static/icons/46.gif b/reader/static/icons/46.gif
diff --git a/reader/static/icons/47.gif b/reader/static/icons/47.gif
diff --git a/reader/static/icons/48.gif b/reader/static/icons/48.gif
diff --git a/reader/static/icons/49.gif b/reader/static/icons/49.gif
diff --git a/reader/static/icons/5.gif b/reader/static/icons/5.gif
diff --git a/reader/static/icons/52.gif b/reader/static/icons/52.gif
diff --git a/reader/static/icons/55.gif b/reader/static/icons/55.gif
diff --git a/reader/static/icons/6.gif b/reader/static/icons/6.gif
diff --git a/reader/static/icons/7.gif b/reader/static/icons/7.gif
diff --git a/reader/static/icons/8.gif b/reader/static/icons/8.gif
diff --git a/reader/static/icons/9.gif b/reader/static/icons/9.gif
diff --git a/reader/static/main.js b/reader/static/main.js
@@ -49,4 +49,4 @@ function setstar(a) {
 };
 
 
-setInterval("location.reload(true);", 900000);
+setInterval("location.reload(true);", 300000);
diff --git a/reader/static/style.css b/reader/static/style.css
@@ -67,6 +67,8 @@ text-decoration: underline;
 .footer {
   margin-top: 40px;
   margin-bottom: 40px;
+  margin-left: 120px;
+  text-align: left;
 }
 
 .mydiv:hover { 

diff --git a/reader/templates/base.html b/reader/templates/base.html
@@ -35,8 +35,10 @@
 {% endfor %}
 {% block content %}{% endblock %}
 
-<footer class="footer">
+<footer class="footer mt-auto py-3">
   <div class="container">
+    <span class="text-muted"><p>© Copyright {{ now.year }} <br><a href="http://www.spudooli.com" target="_blank">Spudooli Investments
+      Ltd</a></p></span>
   </div>
 </footer>
 <script type="module" src="https://unpkg.com/ionicons@5.5.2/dist/ionicons/ionicons.esm.js"></script>

diff --git a/reader/templates/index.html b/reader/templates/index.html
@@ -32,7 +32,8 @@ <h2>Feeds</h2>
                     <small>
                         <div style="float:right;">{{item[6].strftime('%A, %-d %B, %Y %-H:%M')}} </div>
                     </small>
-                    <h6>{{item[5]}} </h6>
+                    <h6><img src="/static/icons/{{ item[7] }}.gif" width="16" height="16">
+                       {{item[5]}} </h6>
                     {% if item[1] == None %}<h3></h3>
                     {% else %}<h4><div class="star" onclick="setstar({{ item[0] }})">{% if item[8] == None %}<ion-icon id="star" name="star-outline"></ion-icon>{% else %} <ion-icon id="star" name="star"></ion-icon>{% endif %} </div> <a href="{{ item[2]}}">{{ item[1]}}</a></h4>