fixed everything /w useragents + opts (deploy wip) [Stachmerge]

deploy.pdy

@@ -0,0 +1,2 @@
+<?xml version='1.0' encoding='utf-8'?>
+<Project usingdefaultlocations="0" version="7"><Python hostinterpreter="" major="3" minor="7" patch="0" platformpython="win macos linux" sourcedir="$SYSROOT/src/Python-$PDY_PY_MAJOR.$PDY_PY_MINOR.$PDY_PY_MICRO" targetincludedir="/include/python$PDY_PY_MAJOR.$PDY_PY_MINOR" targetlibrary="/lib/libpython$PDY_PY_MAJOR.$PDY_PY_MINOR.a" targetstdlibdir="/lib/python$PDY_PY_MAJOR.$PDY_PY_MINOR" /><Application entrypoint="" isbundle="1" isconsole="1" ispyqt5="1" name="RealBandcampMusicDiscovery" script="src/userinterface.py" syspath=""><Package name="."><PackageContent included="0" isdirectory="0" name="deploy.pdy" /><PackageContent included="0" isdirectory="0" name="lel.py" /><PackageContent included="0" isdirectory="0" name="LICENSE" /><PackageContent included="1" isdirectory="1" name="logs"><PackageContent included="1" isdirectory="0" name="error.log" /></PackageContent><PackageContent included="0" isdirectory="0" name="README.md" /><PackageContent included="0" isdirectory="0" name="skeleton.json" /><PackageContent included="1" isdirectory="1" name="src"><PackageContent included="1" isdirectory="0" name="album.py" /><PackageContent included="1" isdirectory="0" name="htmlparser.py" /><PackageContent included="1" isdirectory="0" name="messagehandler.py" /><PackageContent included="1" isdirectory="0" name="messages.py" /><PackageContent included="0" isdirectory="0" name="requs" /><PackageContent included="1" isdirectory="0" name="save.txt" /><PackageContent included="0" isdirectory="0" name="userinterface.py" /><PackageContent included="1" isdirectory="0" name="webconnector.py" /></PackageContent><Exclude name="*.pyc" /><Exclude name="*.pyd" /><Exclude name="*.pyo" /><Exclude name="*.pyx" /><Exclude name="*.pxi" /><Exclude name="__pycache__" /><Exclude name="*-info" /><Exclude name="EGG_INFO" /><Exclude name="*.so" /></Package></Application><PyQtModule name="QtCore" /><PyQtModule name="QtGui" /><PyQtModule name="QtWidgets" /><StdlibModule name="concurrent.futures" /><Package name="/usr/lib64/python3.7/site-packages/requests"><PackageContent included="1" isdirectory="0" name="__init__.py" /><PackageContent included="1" isdirectory="0" name="__version__.py" /><PackageContent included="1" isdirectory="0" name="adapters.py" /><PackageContent included="1" isdirectory="0" name="api.py" /><PackageContent included="1" isdirectory="0" name="auth.py" /><PackageContent included="1" isdirectory="0" name="certs.py" /><PackageContent included="1" isdirectory="0" name="compat.py" /><PackageContent included="1" isdirectory="0" name="cookies.py" /><PackageContent included="1" isdirectory="0" name="exceptions.py" /><PackageContent included="1" isdirectory="0" name="help.py" /><PackageContent included="1" isdirectory="0" name="hooks.py" /><PackageContent included="1" isdirectory="0" name="_internal_utils.py" /><PackageContent included="1" isdirectory="0" name="models.py" /><PackageContent included="1" isdirectory="0" name="packages.py" /><PackageContent included="1" isdirectory="0" name="sessions.py" /><PackageContent included="1" isdirectory="0" name="status_codes.py" /><PackageContent included="1" isdirectory="0" name="structures.py" /><PackageContent included="1" isdirectory="0" name="utils.py" /><Exclude name="*.pyc" /><Exclude name="*.pyd" /><Exclude name="*.pyo" /><Exclude name="*.pyx" /><Exclude name="*.pxi" /><Exclude name="__pycache__" /><Exclude name="*-info" /><Exclude name="EGG_INFO" /><Exclude name="*.so" /></Package></Project>

src/htmlparser.py

@@ -55,13 +55,13 @@ def parse_albums(data):
     cover_url = ""
 
     for line in data:
-        if ">new arrivals</a>" in line and not albool:
+        if "item_list" in line and not albool:
             albool = True
         elif "<a href=" in line and albool:
             url = line.split("<a href=\"")[1].split("\" title")[0]
             name = line.split("title=\"")[1].split("\">")[0]
         elif "<div class=\"tralbum" in line and albool:
-            cover_url = line.split("url(")[1].split(")'")[0]
+            cover_url = line.split("src=\"")[1].split("\">")[0]
         elif "<div class=\"itemsubtext" in line and albool:
             band = line.split("\">")[1].split("</")[0]
             alb = Album(name, url, band, cover_url)

src/messagehandler.py

@@ -1,6 +1,5 @@
-""" Base Msghandler Class to reduce duplication """
+""" Base Msghandler Class to handle IPC """
 import types
-from time import sleep
 from multiprocessing import Queue
 import logging
 import sys
@@ -38,13 +37,15 @@ def uncaught_exceptions(exc_type, exc_val, exc_trace):
 
 
 class MessageHandler():
-    def __init__(self, queue=None, size=5000, interval=0.5):
-        if queue == None:
+    """ the messagehandler class """
+    def __init__(self, queue=None, size=5000):
+        if queue is None:
             self.queue = Queue(size)
         else:
             self.queue = queue
 
     def __init_subclass__(cls):
+        """ subclass init to make sure that subclasses implement the analyze function """
         if "analyze" in cls.__dict__.keys():
             if not isinstance(cls.__dict__["analyze"], types.FunctionType):
                 raise AttributeError("Function \"analyze\" must be a function")
@@ -53,19 +54,22 @@ def __init_subclass__(cls):
         super().__init_subclass__()
 
     def __del__(self):
+        """ delete dis """
         self.queue.cancel_join_thread()
         self.queue.close()
 
     def send(self, msg):
+        """ sets metadata and enqueues message """
         msg.sender = self.__class__.__name__
-        LOGGER.info("%s sent: %s" % (msg.sender, msg))
+        LOGGER.info("%s sent: %s", msg.sender, msg)
         self.queue.put(msg, block=True)
 
     def recieve(self):
+        """ pulls message from queue and returns it from queue it it's originator """
         if not self.queue.empty():
-            msg = self.queue.get(block=True)
+            msg = self.queue.get(block=False)
             if isinstance(msg, Msg):
                 if msg.sender == self.__class__.__name__:
                     self.send(msg)
                     return None
-                return msg
+            return msg #Either all return statements in a function should return an expression, or none of them should. (inconsistent-return-statements)

src/messages.py

@@ -4,18 +4,26 @@
 
 
 class Msg():
-    """Base Class for all Msg-Types"""
+    """ Base Class for all Msg-Types """
 
     def __init__(self, data=None):
-        """simple init"""
+        """ simple init """
         self.sender = None
         self.data = data
 
+    def get_sender(self):
+        """ returns sender """
+        return self.sender
+
+    def get_data(self):
+        """ returns data """
+        return self.data
+
 
 class MsgGetTags(Msg):
     """ class to set signal for getting all tags """
 
-    def __init__(self, data=None):
+    def __init__(self, data):
         super().__init__(data)
 
 
@@ -47,8 +55,8 @@ def __init__(self, data):
         super().__init__(data)
 
 
-class MsgStop(Msg):
-    """ class to signal the stop of album fetching """
+class MsgQuit(Msg):
+    """ class to signal the stop of program execution """
 
     def __init__(self, data):
-        super().__init__(data)
+        super().__init__(data)

src/userinterface.py

@@ -2,18 +2,15 @@
 
 import sys
 from math import ceil
-from time import sleep
 import logging
-import multiprocessing
 
-from PyQt5 import QtCore, QtGui
-from PyQt5.QtWidgets import *
-from PyQt5.QtCore import *
-from PyQt5.QtGui import *
+from PyQt5.QtWidgets import QMainWindow, QGridLayout, QLayout, QWidget, QMessageBox, QAction, QComboBox, QCompleter, QPushButton, QApplication, QScrollArea
+from PyQt5.QtCore import QTimer, QRect, QSortFilterProxyModel, QSize, Qt
+from PyQt5.QtGui import QIcon
 
 from album import Album
 from webconnector import Connector
-from messages import *
+from messages import MsgGetTags, MsgPutFetchTags, MsgPutTags, MsgPutAlbums, MsgQuit
 from messagehandler import MessageHandler
 
 
@@ -75,7 +72,7 @@ def __init__(self):
         self.scroll_area.setWidget(self.widget)
         self.scroll_area.setWidgetResizable(True)
         self.scroll_area.setVerticalScrollBarPolicy(
-            QtCore.Qt.ScrollBarAlwaysOn)
+            Qt.ScrollBarAlwaysOn)
         self.setCentralWidget(self.scroll_area)
 
         self.setWindowTitle("RealBandcampMusicDisc0very")
@@ -87,14 +84,13 @@ def __init__(self):
         self.fetched_albums = {}
         LOGGER.info("Self: %s", self)
 
-        self.close_event = self.close
-        self.show_event = self.show()
+        self.closeEvent = self.close
         self.statusBar()
         self.firstloaded = True
         self.init_menu()
         self.init_toolbar()
         self.show()
-        self.send(MsgGetTags())
+        self.send(MsgGetTags(None))
         self.setStatusTip("Initializing...")
         self.msgbox = QMessageBox(self)
         self.msgbox.setText("Initializing...")
@@ -104,11 +100,10 @@ def __init__(self):
     def __del__(self):
         """ del """
         MessageHandler.__del__(self)
-        self.connector.__del__()
-        self.queue.close()
 
     def close(self, event):
         """ injection func for closing ther window """
+        self.send(MsgQuit(None))
         LOGGER.debug("Close event:\n%s", event)
         self.__del__()
 
@@ -160,17 +155,17 @@ def init_menu(self):
         self.save_action = QAction(" &Save", self)
         self.save_action.setShortcut("Ctrl+S")
         self.save_action.setStatusTip("Save results to file")
-        self.save_action.triggered.connect(self.saveToFile)
+        self.save_action.triggered.connect(self.save_to_file)
 
         self.help_action = QAction(" &Help", self)
         self.help_action.setShortcut("F1")
         self.help_action.setStatusTip("Show the help")
-        self.help_action.triggered.connect(self.showHelp)
+        self.help_action.triggered.connect(self.show_help)
 
         self.quit_action = QAction(" &Quit", self)
         self.quit_action.setShortcut("Crtl+Q")
         self.quit_action.setStatusTip("Quit this Application")
-        self.quit_action.triggered.connect(self.quitApplication)
+        self.quit_action.triggered.connect(self.quit_application)
 
         self.help_menu.addAction(self.save_action)
         self.help_menu.addAction(self.help_action)
@@ -189,11 +184,11 @@ def save_to_file(self):
                            (album.band, album.name, album.url))
         self.setStatusTip("Data saved to file!")
 
-    def quit_application(self):
+    def quit_application(self): # implement this
         """ quit """
         print("####################################Quit!############################################")
 
-    def show_help(self):
+    def show_help(self): # implement this
         """ help """
         print("####################################Help!############################################")
 
@@ -258,13 +253,13 @@ def add_genre_selector(self):
         tempcombobox.setEditable(True)
         tempcombobox.setInsertPolicy(3)
         compfilter = QSortFilterProxyModel(tempcombobox)
-        compfilter.setFilterCaseSensitivity(QtCore.Qt.CaseInsensitive)
+        compfilter.setFilterCaseSensitivity(Qt.CaseInsensitive)
         compfilter.setFilterKeyColumn(1)
         compfilter.setSourceModel(tempcombobox.model())
         comp = QCompleter(compfilter, tempcombobox)
-        comp.setCaseSensitivity(QtCore.Qt.CaseInsensitive)
+        comp.setCaseSensitivity(Qt.CaseInsensitive)
         comp.setCompletionMode(QCompleter.PopupCompletion)
-        comp.setFilterMode(QtCore.Qt.MatchContains)
+        comp.setFilterMode(Qt.MatchContains)
         tempcombobox.setCompleter(comp)
         tempcombobox.addItems(self.genrelist)
         self.selectorlist.append(self.toolbar.addWidget(tempcombobox))
@@ -324,7 +319,7 @@ def analyze(self, msg):
                 self.store_tags_from_msg(msg)
                 self.finalize_init()
             elif isinstance(msg, MsgPutAlbums):
-                self.processAlbums(msg)
+                self.process_albums(msg)
             else:
                 LOGGER.error("Unknown Message:\n%s", msg)
 

src/webconnector.py

@@ -9,7 +9,7 @@
 
 from messagehandler import MessageHandler
 from htmlparser import parse_album_metadata, parse_albums, parse_maxpages, parse_tags
-from messages import MsgGetTags, MsgPutAlbums, MsgPutFetchTags, MsgPutTags, MsgStop
+from messages import MsgGetTags, MsgPutAlbums, MsgPutFetchTags, MsgPutTags, MsgQuit
 
 LOGGER = logging.getLogger('rbmd.webconnector')
 LOG_FORMAT = "%(asctime)-15s | %(levelname)s | %(module)s %(name)s %(process)d %(thread)d | %(funcName)20s() - Line %(lineno)d | %(message)s"
@@ -65,6 +65,7 @@ def __init__(self, queue=None):
         self.taglist = set()
         self.stop = multiprocessing.Event()
         self.session = requests.Session()
+        self.session.headers.update({"User-Agent": "Mozilla/5.0 (Linux; Android 7.0; PLUS Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.98 Mobile Safari/537.36",})
         self.apiurl = "https://bandcamp.com/tag/%s?page=%s"  # tag, pagenum
         self.start()
         LOGGER.debug("Initialized %s", self)
@@ -137,7 +138,7 @@ def analyze(self, msg):
                 self.get_genres_event.set()
             elif isinstance(msg, MsgPutFetchTags):
                 self.get_fetch_tags(msg)
-            elif isinstance(msg, MsgStop):
+            elif isinstance(msg, MsgQuit):
                 self.__del__()
             else:
                 LOGGER.error("Unknown Message:\n%s", msg)