IMP: Changed configuration completely. New config processing, global/…

…config variables changed over entire application. All changes in the configuration GUI take effect immediately after saving - no more need to restart Mylar, IMP: Added provider order sequence header to the top of the Providers tab for improved visibilty, IMP: Added completed download handling for both SABnzbd and NZBGet - will monitor active queue for downloads sent by Mylar and then post-process them accordingly (no more ComicRN), FIX: When recreating a pullist for a week that's not the current week, will now refresh series that are still missing data in case of late population, IMP: Removed loose/explict search options, and search results will now match more accurately to the terms entered, as well as being much quicker, IMP: Changed colour-codes on search results screen, green now indicates that the series is a print series, non-green indicates that it is either a HC/TPB/digital series and will also be indicated as such, IMP: Fixed weeklypull not refreshing some series due to legacy numbering on series deemed as Ended by Mylar, or the pull not refreshing propeprly and thereby were not elgible to be considered as a valid series for matching, IMP: Fixed problem with the scheduler of the weekly pull not running at the designated time, which causes problems when attempting to match series to the pullist for issues, IMP: Changed the autosnatch script so that the get.conf is no longer required to be maintained seperately. The get.conf config items are now located in the main config.ini and have to be repopulated initially, FIX: Fixed problem with the import count not resetting to 0 on subsequent runs of the Import A Directory option, IMP: Changed the sab api key from using the nzbkey to the full api key as is required for completed download handling, FIX: Fixed the retry option for newznab entries so that it should work across the board now for newznabs as opposed to only some providers, IMP: some minor code cleanup

data/interfaces/default/base.html

@@ -104,9 +104,7 @@
                         </div>
 			<div id="version">
 				Version: <em>${mylar.CURRENT_VERSION}</em>
-				%if mylar.CONFIG.GIT_BRANCH != 'master':
-				(${version.MYLAR_VERSION})
-				%endif
+				(${mylar.CONFIG.GIT_BRANCH})
 			</div>			
 		</footer>
 		<a href="#main" id="toTop"><span>Back to top</span></a>

data/interfaces/default/config.html

@@ -362,6 +362,11 @@ <h1 class="clearfix"><img src="interfaces/default/images/icon_gear.png" alt="set
                                     </div>
                                 </div>
 
+                                <div class="row checkbox left clearfix">
+                                    <input type="checkbox" id="sab_client_post_processing" onclick="initConfigCheckbox($this));" name="sab_client_post_processing" value="1" ${config['sab_client_post_processing']} /><label>Enable Completed Download Handling<label>
+                                    <small>The category label above is used to when completed download handling is enabled</small>
+                                </div>
+
                                 <div align="center" class="row">
                                      <input type="button" value="Test SABnzbd" id="test_sab" style="float:center" /></br>
                                      <input type="text" name="sabstatus" style="text-align:center; font-size:11px;" id="sabstatus" size="50" DISABLED />
@@ -409,7 +414,12 @@ <h1 class="clearfix"><img src="interfaces/default/images/icon_gear.png" alt="set
                                                 %endfor
                                                 </select>
                                               </div>
-
+
+                            <div class="row checkbox left clearfix">
+                                <input type="checkbox" id="nzbget_client_post_processing" onclick="initConfigCheckbox($this));" name="nzbget_client_post_processing" value="1" ${config['nzbget_client_post_processing']} /><label>Enable Completed Download Handling<label>
+                                <small>The category label above is used to when completed download handling is enabled</small>
+                            </div>
+
                          </fieldset>
                          <fieldset id="blackhole_options">
                                     <div class="row">
@@ -424,7 +434,6 @@ <h1 class="clearfix"><img src="interfaces/default/images/icon_gear.png" alt="set
                                 <input type="text" name="usenet_retention" value="${config['usenet_retention']}" size="10">
                             </div>
                          </fieldset>
-
                    </td>
                    <td>
                       <legend>Torrents</legend>
@@ -928,7 +937,6 @@ <h1 class="clearfix"><img src="interfaces/default/images/icon_gear.png" alt="set
                                     </div>
                                 </div>
                         </fieldset>
-
                	</td>
                 <td>
                         <fieldset>
@@ -1797,6 +1805,7 @@ <h3><img src="interfaces/default/images/slack.png" style="vertical-align: middle
                                 return;
                             }
                             $('#sabstatus').val(data);
+                            // $('#sab_apikey').val(data);
                             $('#ajaxMsg').html("<div class='msg'><span class='ui-icon ui-icon-check'></span>"+data+"</div>");
                     });
                     $('#ajaxMsg').addClass('success').fadeIn().delay(3000).fadeOut();

data/interfaces/default/searchresults.html

@@ -6,20 +6,7 @@
 %>
 
 <%def name="headerIncludes()">
-        <div id="subhead_container">
-                <div id="subhead_menu">
-                    %if explicit == 'loose':
-                        <a id="menu_link_retry" title="This will search for ALL of the terms given : ${name}" href="searchit?name=${name |u}&explicit=loose">Search ALL terms</a>
-                        <a id="menu_link_retry" title="This will search EXPLICITLY for only the terms given : ${name}" href="searchit?name=${name |u}&explicit=explicit">Explicit Search</a>
-                    %elif explicit == 'explicit':
-                        <a id="menu_link_retry" title="Warning: This will search for ANY of the terms given : ${name} (this could take awhile)" href="searchit?name=${name |u}&explicit=loose">Loose Search</a>
-                        <a id="menu_link_retry" title="This will search for ALL of the terms given : ${name}" href="searchit?name=${name |u}&explicit=all">Search ALL terms</a>
-                    %elif explicit == 'all':
-                        <a id="menu_link_retry" title="This will search EXPLICITLY for only the terms given : ${name}" href="searchit?name=${name |u}&explicit=explicit">Explicit Search</a>
-                        <a id="menu_link_retry" title="Warning: This will search for ANY of the terms given : ${name} (this could take awhile)" href="searchit?name=${name |u}&explicit=loose">Loose Search</a>
-                    %endif
-                </div>
-        </div>
+
 </%def>
 <%def name="body()">
 	<div id="paddingheader">
@@ -28,12 +15,7 @@
                           typesel = " Story Arc"
                       else:
                           typesel = ""
-                      if explicit == 'loose':
-                          searchtext = "Loose Search results for: </br><center> " + name + "</center>"
-                      elif explicit == 'explicit':
-                          searchtext = "Explicit " + typesel.rstrip() + " Search results for: </br><center> " + name + "</center>"
-                      else:
-                          searchtext = "Search results for : </br><center>" + name + "</center>"
+                      searchtext = "Search results for : </br><center>" + name + "</center>"
                  %>
 
 		<h1 class="clearfix"><img src="interfaces/default/images/icon_search.png" alt="Search results"/>${searchtext}</h1>
@@ -66,17 +48,23 @@ <h1 class="clearfix"><img src="interfaces/default/images/icon_search.png" alt="S
 			%if searchresults:
 				%for result in searchresults:
 				<%
-					if result['comicyear'] == mylar.CURRENT_YEAR:
-					    grade = 'A'
-					else:
-					    grade = 'Z'
-					if result['haveit'] != "No":
-					    grade = 'H';
-
+					grade = 'Z'
                                         rtype = None
                                         if type != 'story_arc':
                                             if result['type'] == 'Digital':
                                                 rtype = '[Digital]'
+                                                grade = 'Z'
+                                            elif result['type'] == 'TPB':
+                                                rtype = '[TPB]'
+                                                grade = 'Z'
+                                            elif result['type'] == 'HC':
+                                                rtype = '[HC]'
+                                                grade = 'Z'
+                                            else:
+                                                grade = 'A'
+
+                                        if result['haveit'] != "No":
+                                            grade = 'H';
 				%>
 				<tr class="grade${grade}">
 					<td class="blank"></td>

data/interfaces/default/weeklypull.html

@@ -9,7 +9,7 @@
 	<div id="subhead_container">
 		<div id="subhead_menu">
                         <a href="#" id="menu_link_refresh" onclick="doAjaxCall('pullist?week=${weekinfo['weeknumber']}&year=${weekinfo['year']}',$(this),'table')" data-success="Refresh submitted.">Refresh Pull-list</a>
-                        <a id="menu_link_retry" href="pullrecreate">Recreate Pull-list</a>
+                        <a href="#" id="menu_link_retry" onclick="doAjaxCall('pullrecreate?weeknumber=${weekinfo['weeknumber']}&year=${weekinfo['year']}',$(this),'table')" data-success="Recreating Pullist for week ${weekinfo['weeknumber']}, ${weekinfo['year']}">Recreate Pull-list</a>
 <!--
                     <a href="#" id="menu_link_retry" onclick="doAjaxCall('create_readlist?weeknumber=${weekinfo['weeknumber']}&year=${weekinfo['year']}',$(this),'table')" data-success="Submitted request for reading-list generation for this week">Generate Reading-List</a>
 -->

mylar/PostProcessor.py

@@ -1027,7 +1027,6 @@ def Process(self):
                     comicname = None
                     issuenumber = None
                     if tmpiss is not None:
-                        logger.info('shouldnt be here')
                         ppinfo.append({'comicid':       tmpiss['ComicID'],
                                        'issueid':       issueid,
                                        'comicname':     tmpiss['ComicName'],

mylar/__init__.py

@@ -83,7 +83,6 @@
 WEEKLY_STATUS = 'Waiting'
 VERSION_STATUS = 'Waiting'
 UPDATER_STATUS = 'Waiting'
-SNATCHED_QUEUE = Queue.Queue()
 SCHED_RSS_LAST = None
 SCHED_WEEKLY_LAST = None
 SCHED_MONITOR_LAST = None
@@ -118,6 +117,9 @@
 USE_UTORRENT = False
 USE_WATCHDIR = False
 SNPOOL = None
+NZBPOOL = None
+SNATCHED_QUEUE = Queue.Queue()
+NZB_QUEUE = Queue.Queue()
 COMICSORT = None
 PULLBYFILE = None
 CFG = None
@@ -134,6 +136,7 @@
 CMTAGGER_PATH = None
 STATIC_COMICRN_VERSION = "1.01"
 STATIC_APC_VERSION = "1.0"
+SAB_PARAMS = None
 SCHED = BackgroundScheduler({
                              'apscheduler.executors.default': {
                                  'class':  'apscheduler.executors.pool:ThreadPoolExecutor',
@@ -149,9 +152,9 @@ def initialize(config_file):
     with INIT_LOCK:
 
         global CONFIG, _INITIALIZED, QUIET, CONFIG_FILE, CURRENT_VERSION, LATEST_VERSION, COMMITS_BEHIND, INSTALL_TYPE, IMPORTLOCK, PULLBYFILE, INKDROPS_32P, \
-               DONATEBUTTON, CURRENT_WEEKNUMBER, CURRENT_YEAR, UMASK, USER_AGENT, SNATCHED_QUEUE, PULLNEW, COMICSORT, WANTED_TAB_OFF, CV_HEADERS, \
+               DONATEBUTTON, CURRENT_WEEKNUMBER, CURRENT_YEAR, UMASK, USER_AGENT, SNATCHED_QUEUE, NZB_QUEUE, PULLNEW, COMICSORT, WANTED_TAB_OFF, CV_HEADERS, \
                IMPORTBUTTON, IMPORT_FILES, IMPORT_TOTALFILES, IMPORT_CID_COUNT, IMPORT_PARSED_COUNT, IMPORT_FAILURE_COUNT, CHECKENABLED, CVURL, DEMURL, WWTURL, TPSEURL, \
-               USE_SABNZBD, USE_NZBGET, USE_BLACKHOLE, USE_RTORRENT, USE_UTORRENT, USE_QBITTORRENT, USE_DELUGE, USE_TRANSMISSION, USE_WATCHDIR, \
+               USE_SABNZBD, USE_NZBGET, USE_BLACKHOLE, USE_RTORRENT, USE_UTORRENT, USE_QBITTORRENT, USE_DELUGE, USE_TRANSMISSION, USE_WATCHDIR, SAB_PARAMS, \
                PROG_DIR, DATA_DIR, CMTAGGER_PATH, DOWNLOAD_APIKEY, LOCAL_IP, STATIC_COMICRN_VERSION, STATIC_APC_VERSION, KEYS_32P, AUTHKEY_32P, FEED_32P, FEEDINFO_32P, \
                MONITOR_STATUS, SEARCH_STATUS, RSS_STATUS, WEEKLY_STATUS, VERSION_STATUS, UPDATER_STATUS, DBUPDATE_INTERVAL, \
                SCHED_RSS_LAST, SCHED_WEEKLY_LAST, SCHED_MONITOR_LAST, SCHED_SEARCH_LAST, SCHED_VERSION_LAST, SCHED_DBUPDATE_LAST
@@ -165,7 +168,7 @@ def initialize(config_file):
             return False
 
         #set up the default values here if they're wrong.
-        cc.configure()
+        #cc.configure()
 
         # Start the logger, silence console logging if we need to
         logger.initLogger(console=not QUIET, log_dir=CONFIG.LOG_DIR, verbose=VERBOSE) #logger.mylar_log.initLogger(verbose=VERBOSE)
@@ -381,29 +384,25 @@ def start():
                     SCHED.add_job(func=ss.run, id='search', name='Auto-Search', next_run_time=search_diff, trigger=IntervalTrigger(hours=0, minutes=CONFIG.SEARCH_INTERVAL, timezone='UTC'))
 
             if all([CONFIG.ENABLE_TORRENTS, CONFIG.AUTO_SNATCH, OS_DETECT != 'Windows']) and any([CONFIG.TORRENT_DOWNLOADER == 2, CONFIG.TORRENT_DOWNLOADER == 4]):
-                logger.info('[AUTO-SNATCHER] Auto-Snatch of completed torrents enabled & attempting to backgroun load....')
+                logger.info('[AUTO-SNATCHER] Auto-Snatch of completed torrents enabled & attempting to background load....')
                 SNPOOL = threading.Thread(target=helpers.worker_main, args=(SNATCHED_QUEUE,), name="AUTO-SNATCHER")
                 SNPOOL.start()
                 logger.info('[AUTO-SNATCHER] Succesfully started Auto-Snatch add-on - will now monitor for completed torrents on client....')
 
-            helpers.latestdate_fix()
+            if CONFIG.POST_PROCESSING is True and ( all([CONFIG.NZB_DOWNLOADER == 0, CONFIG.SAB_CLIENT_POST_PROCESSING is True]) or all([CONFIG.NZB_DOWNLOADER == 1, CONFIG.NZBGET_CLIENT_POST_PROCESSING is True]) ):
+                if CONFIG.NZB_DOWNLOADER == 0:
+                    logger.info('[SAB-MONITOR] Completed post-processing handling enabled for SABnzbd. Attempting to background load....')
+                elif CONFIG.NZB_DOWNLOADER == 1:
+                    logger.info('[NZBGET-MONITOR] Completed post-processing handling enabled for NZBGet. Attempting to background load....')
+                NZBPOOL = threading.Thread(target=helpers.nzb_monitor, args=(NZB_QUEUE,), name="AUTO-COMPLETE-NZB")
+                NZBPOOL.start()
+                if CONFIG.NZB_DOWNLOADER == 0:
+                    logger.info('[AUTO-COMPLETE-NZB] Succesfully started Completed post-processing handling for SABnzbd - will now monitor for completed nzbs within sabnzbd and post-process automatically....')
+                elif CONFIG.NZB_DOWNLOADER == 1:
+                    logger.info('[AUTO-COMPLETE-NZB] Succesfully started Completed post-processing handling for NZBGet - will now monitor for completed nzbs within nzbget and post-process automatically....')
 
-            #initiate startup rss feeds for torrents/nzbs here...
-            if CONFIG.ENABLE_RSS:
-                logger.info('[RSS-FEEDS] Initiating startup-RSS feed checks.')
-                if SCHED_RSS_LAST is not None:
-                    rss_timestamp = float(SCHED_RSS_LAST)
-                    logger.info('[RSS-FEEDS] RSS last run @ %s' % datetime.datetime.utcfromtimestamp(rss_timestamp))
-                else:
-                    rss_timestamp = helpers.utctimestamp() + (int(CONFIG.RSS_CHECKINTERVAL) *60)
-                rs = rsscheckit.tehMain()
-                duration_diff = (helpers.utctimestamp() - rss_timestamp)/60
-                if duration_diff >= int(CONFIG.RSS_CHECKINTERVAL):
-                    SCHED.add_job(func=rs.run, id='rss', name='RSS Feeds', args=[True], next_run_time=datetime.datetime.now(), trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.RSS_CHECKINTERVAL), timezone='UTC'))
-                else:
-                    rss_diff = datetime.datetime.utcfromtimestamp(helpers.utctimestamp() + (int(CONFIG.RSS_CHECKINTERVAL) * 60) - (duration_diff * 60))
-                    logger.fdebug('[RSS-FEEDS] Scheduling next run for @ %s every %s minutes' % (rss_diff, CONFIG.RSS_CHECKINTERVAL))
-                    SCHED.add_job(func=rs.run, id='rss', name='RSS Feeds', args=[True], next_run_time=rss_diff, trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.RSS_CHECKINTERVAL), timezone='UTC'))
+
+            helpers.latestdate_fix()
 
             if CONFIG.ALT_PULL == 2:
                 weektimer = 4
@@ -425,12 +424,29 @@ def start():
 
             if duration_diff >= weekly_interval/60:
                 logger.info('[WEEKLY] Weekly Pull-Update initializing immediately as it has been %s hours since the last run' % (duration_diff/60))
-                SCHED.add_job(func=ws.run, id='weekly', name='Weekly Pullist', next_run_time=datetime.datetime.now(), trigger=IntervalTrigger(hours=weektimer, minutes=0, timezone='UTC'))
+                SCHED.add_job(func=ws.run, id='weekly', name='Weekly Pullist', next_run_time=datetime.datetime.utcnow(), trigger=IntervalTrigger(hours=weektimer, minutes=0, timezone='UTC'))
             else:
                 weekly_diff = datetime.datetime.utcfromtimestamp(helpers.utctimestamp() + (weekly_interval - (duration_diff * 60)))
                 logger.fdebug('[WEEKLY] Scheduling next run for @ %s every %s hours' % (weekly_diff, weektimer))
                 SCHED.add_job(func=ws.run, id='weekly', name='Weekly Pullist', next_run_time=weekly_diff, trigger=IntervalTrigger(hours=weektimer, minutes=0, timezone='UTC'))
 
+            #initiate startup rss feeds for torrents/nzbs here...
+            if CONFIG.ENABLE_RSS:
+                logger.info('[RSS-FEEDS] Initiating startup-RSS feed checks.')
+                if SCHED_RSS_LAST is not None:
+                    rss_timestamp = float(SCHED_RSS_LAST)
+                    logger.info('[RSS-FEEDS] RSS last run @ %s' % datetime.datetime.utcfromtimestamp(rss_timestamp))
+                else:
+                    rss_timestamp = helpers.utctimestamp() + (int(CONFIG.RSS_CHECKINTERVAL) *60)
+                rs = rsscheckit.tehMain()
+                duration_diff = (helpers.utctimestamp() - rss_timestamp)/60
+                if duration_diff >= int(CONFIG.RSS_CHECKINTERVAL):
+                    SCHED.add_job(func=rs.run, id='rss', name='RSS Feeds', args=[True], next_run_time=datetime.datetime.utcnow(), trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.RSS_CHECKINTERVAL), timezone='UTC'))
+                else:
+                    rss_diff = datetime.datetime.utcfromtimestamp(helpers.utctimestamp() + (int(CONFIG.RSS_CHECKINTERVAL) * 60) - (duration_diff * 60))
+                    logger.fdebug('[RSS-FEEDS] Scheduling next run for @ %s every %s minutes' % (rss_diff, CONFIG.RSS_CHECKINTERVAL))
+                    SCHED.add_job(func=rs.run, id='rss', name='RSS Feeds', args=[True], next_run_time=rss_diff, trigger=IntervalTrigger(hours=0, minutes=int(CONFIG.RSS_CHECKINTERVAL), timezone='UTC'))
+
             if CONFIG.CHECK_GITHUB:
                 vs = versioncheckit.CheckVersion()
                 SCHED.add_job(func=vs.run, id='version', name='Check Version', trigger=IntervalTrigger(hours=0, minutes=CONFIG.CHECK_GITHUB_INTERVAL, timezone='UTC'))
@@ -1102,6 +1118,17 @@ def halt():
             except:
                 SCHED.shutdown(wait=False)
 
+            if NZBPOOL is not None:
+                logger.info('Terminating the nzb auto-complete thread.')
+                try:
+                    NZBPOOL.join(10)
+                    logger.info('Joined pool for termination -  successful')
+                except KeyboardInterrupt:
+                    NZB_QUEUE.put('exit')
+                    NZBPOOL.join(5)
+                except AssertionError:
+                    os._exit(0)
+
             if SNPOOL is not None:
                 logger.info('Terminating the auto-snatch thread.')
                 try:
@@ -1112,7 +1139,6 @@ def halt():
                     SNPOOL.join(5)
                 except AssertionError:
                     os._exit(0)
-
             _INITIALIZED = False
 
 def shutdown(restart=False, update=False):

mylar/auth32p.py

@@ -164,7 +164,7 @@ def authenticate(self):
             return
         else:
             mylar.FEEDINFO_32P = feedinfo
-            return
+            return feedinfo
 
     def searchit(self):
         #self.searchterm is a tuple containing series name, issue number, volume and publisher.