Merge pull request #7 from infosecmatters2021/main

#6 - Request: please include qualifiers - new, live and premiere

resources/data/json/providers.json

@@ -11,7 +11,6 @@
 
         "main_loader": true,
         "main_threads": 1,
-
         "adv_loader": false,
 
         "menu_visibility": false
@@ -58,4 +57,4 @@
 
     }
 
-}
+}

resources/lib/db.py

@@ -70,7 +70,7 @@ def create_epg_db(self, provider, pre_load):
                        """(channel_id TEXT, broadcast_id TEXT PRIMARY KEY, start INTEGER, end INTEGER,"""
                        """ title TEXT, subtitle TEXT, desc TEXT, image TEXT, date TEXT, country TEXT,"""
                        """ star TEXT, rating TEXT, credits TEXT, season_episode_num TEXT,"""
-                       """ genres TEXT, advanced INTEGER, db_link TEXT)""".format(f"pre_{provider}" if pre_load else provider))
+                       """ genres TEXT, qualifiers TEXT, advanced INTEGER, db_link TEXT)""".format(f"pre_{provider}" if pre_load else provider))
         self.confirm_update()
         return
 
@@ -84,21 +84,21 @@ def remove_epg_db(self, provider, pre_load):
 
     def retrieve_epg_db_items(self, provider, channel):
         self.c.execute("""SELECT channel_id, broadcast_id, start, end, title, subtitle, desc, """
-                        """image, date, country, star, rating, credits, season_episode_num, genres, advanced"""
+                        """image, date, country, star, rating, credits, season_episode_num, genres, qualifiers, advanced"""
                         """ FROM {} WHERE channel_id = ? ORDER BY start ASC""".format(provider),
                        (channel,))
         return self.c.fetchall()
 
     def write_epg_db_items(self, provider, to_be_added, pre_load):
         [self.c.execute("""INSERT OR REPLACE INTO {} """
                         """(channel_id, broadcast_id, start, end, title, subtitle, desc, """
-                        """image, date, country, star, rating, credits, season_episode_num, genres, advanced, db_link)"""
-                        """VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""".format(f"pre_{provider}" if pre_load else provider),
+                        """image, date, country, star, rating, credits, season_episode_num, genres, qualifiers, advanced, db_link)"""
+                        """VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""".format(f"pre_{provider}" if pre_load else provider),
                         (channel_id, broadcast_id, start, end, title, subtitle, desc, 
                          image, date, country, json.dumps(star), json.dumps(rating), json.dumps(credits), 
-                         json.dumps(season_episode_num), json.dumps(genres), 0, None))
+                         json.dumps(season_episode_num), json.dumps(genres), json.dumps(qualifiers), 0, None))
         for channel_id, broadcast_id, start, end, title, subtitle, desc, image, date,
-            country, star, rating, credits, season_episode_num, genres in to_be_added]
+            country, star, rating, credits, season_episode_num, genres, qualifiers in to_be_added]
         return
 
     def remove_epg_db_items(self, provider, to_be_removed):
@@ -112,12 +112,12 @@ def update_epg_db_items(self, provider, to_be_updated, advanced):
                         """desc = coalesce(?, desc), image = coalesce(?, image), """
                         """date = coalesce(?, date), country = coalesce(?, country), star = coalesce(?, star), rating = coalesce(?, rating), """
                         """credits = coalesce(?, credits), season_episode_num = coalesce(?, season_episode_num), """
-                        """genres = coalesce(?, genres), advanced = {} """
+                        """genres = coalesce(?, genres), qualifiers = {}, advanced = {} """
                         """WHERE broadcast_id = ?""".format(provider, 1 if advanced else 0),
                         (start, end, title, subtitle, desc, image, date, country, json.dumps(star), json.dumps(rating), 
-                         json.dumps(credits), json.dumps(season_episode_num), json.dumps(genres), broadcast_id))
+                         json.dumps(credits), json.dumps(season_episode_num), json.dumps(genres), json.dumps(qualifiers), broadcast_id))
         for channel_id, start, end, title, subtitle, desc, image, date, country, star, rating,
-            credits, season_episode_num, genres, broadcast_id in to_be_updated]
+            credits, season_episode_num, genres, qualifiers, broadcast_id in to_be_updated]
         return
 
     def simple_epg_db_update(self, provider):
@@ -164,7 +164,7 @@ def simple_epg_db_update(self, provider):
             del cached_broadcasts, cached_advanced_broadcasts
 
             self.c.execute("""SELECT channel_id, broadcast_id, start, end, title, subtitle, """
-                            """desc, image, date, country, star, rating, credits, season_episode_num, genres """
+                            """desc, image, date, country, star, rating, credits, season_episode_num, genres, qualifiers """
                             """FROM {} WHERE channel_id IN ({})""".format(f"pre_{provider}", channel))
             new_broadcasts_items = [item for item in self.c.fetchall()]
 
@@ -293,11 +293,10 @@ def main_downloader(self, provider_name, data=None):
         url_list_part = []
 
         for i in url_list:
-            if len(url_list_part) >= self.providers[provider_name].get("max_dl_num", 50):
+            url_list_part.append(i)
+            if len(url_list_part) == self.providers[provider_name].get("max_dl_num", 50):
                 final_list.append(url_list_part)
                 url_list_part = []
-            else:
-                url_list_part.append(i)
         if len(url_list_part) > 0:
             final_list.append(url_list_part)
 
@@ -329,7 +328,7 @@ def main_downloader(self, provider_name, data=None):
                         [(i["c_id"], i["b_id"], i["start"], i["end"], i["title"], 
                           i.get("subtitle", ""), i.get("desc", ""), i.get("image", ""), 
                           i.get("date", ""), i.get("country", ""), i.get("star", {}), i.get("rating", {}), i.get("credits", {}),
-                          i.get("season_episode_num", {}), i.get("genres", [])) for i in m], True)
+                          i.get("season_episode_num", {}), i.get("genres", []), i.get("qualifiers", [])) for i in m], True)
             self.epg_cache = {}
             self.fl_pr = self.fl_pr + 1
 

resources/lib/epg.py

@@ -171,7 +171,7 @@ def grabber_process(self):
                     inner_worker = 0
 
                     for (channel_id, broadcast_id, start, end, title, subtitle, desc, image, 
-                         date, country, star, rating, credits, season_episode_num, genres, advanced) \
+                         date, country, star, rating, credits, season_episode_num, genres, qualifiers, advanced) \
                             in results:
 
                             if self.cancellation or self.exit:
@@ -207,11 +207,11 @@ def grabber_process(self):
                             else:
                                 program["title"] = {"@lang": lang, "#text": "No programme title available"}
 
-                            # SUBTITLE
+                           # SUBTITLE
                             if subtitle is not None and subtitle != "":
                                 program["sub-title"] = {"@lang": lang, "#text": subtitle}
 
-                            # DESC
+                           # DESC
                             if desc is not None and desc != "":
 
                                 # RATING MAPPER
@@ -354,6 +354,7 @@ def grabber_process(self):
                                 program["episode-num"] = {"@system": "xmltv_ns", 
                                     "#text": f"{int(season_num) - 1} . {int(episode_num) - 1} . "}
 
+
                             # AGE RATING
                             if rating is not None and rating != "":
                                 if rating_type is not None and rating_type != "":
@@ -368,19 +369,28 @@ def grabber_process(self):
                                 else:
                                     program["star-rating"] = {"value": {"#text": star_value}}
 
+                           # QUALIFIERS
+                            if 'New' in qualifiers:
+                                program["new"]={""}
+                            if 'Live' in qualifiers:
+                                program["live"]={""}
+                            if 'Premiere' in qualifiers:
+                                program["premiere"]={"Premiere"}
+
                             pr["programme"].append(program)
                             pn = pn + 1
 
                             inner_worker = inner_worker + 1
                             self.pr.progress = (inner_worker / inner_value) * self.basic_value + self.worker + 50
-
-                            if pn == self.user_db.main["settings"]["pn_max"]:
+    
+                            if pn == self.user_db.main["settings"]["pn_max"]:                                
                                 file.write(xmltodict.unparse(pr, pretty=True, encoding="UTF-8", full_document=False))
+
                                 pn = 0
                                 pr["programme"] = []
-
+                                
                     self.worker = self.worker + (100 * self.basic_value / 2)
-                
+
                 if len(pr["programme"]) > 0:
                     file.write(xmltodict.unparse(pr, pretty=True, encoding="UTF-8", full_document=False))
 
@@ -389,6 +399,18 @@ def grabber_process(self):
             if os.path.exists(f"{self.file_paths['storage']}xml/epg.xml"):
                 os.remove(f"{self.file_paths['storage']}xml/epg.xml")
             os.rename(f"{self.file_paths['storage']}xml/test.xml", f"{self.file_paths['storage']}xml/epg.xml")
+
+            replacements = {'<new></new>':'<new />', '<live></live>':'<live />'}
+
+            with open(f"{self.file_paths['storage']}xml/epg.xml", encoding="UTF-8") as infile, open(f"{self.file_paths['storage']}xml/epg_new.xml", 'w', encoding="UTF-8") as outfile:
+                for line in infile:
+                   for src, target in replacements.items():
+                       line = line.replace(src, target)
+                   outfile.write(line)
+
+            if os.path.exists(f"{self.file_paths['storage']}xml/epg.xml"):
+                os.remove(f"{self.file_paths['storage']}xml/epg.xml")
+            os.rename(f"{self.file_paths['storage']}xml/epg_new.xml", f"{self.file_paths['storage']}xml/epg.xml")           
 
             self.status = "Creating compressed file..."
             with open(f"{self.file_paths['storage']}xml/epg.xml", 'rb') as f_in, gzip.open(f"{self.file_paths['storage']}xml/epg.xml.gz", 'wb') as f_out:

resources/lib/providers/gntms.py

@@ -68,6 +68,10 @@ def epg_main_converter(data, channels, settings, ch_id=None):
         g["season_episode_num"] = {"season": s_num, "episode": e_num}
 
         g["genres"] = i["program"].get("genres", [])
+        if entity_type == "Sports" or entity_type =="Movie":
+        	g["genres"].append(entity_type)
+        if qualifiers is not None:
+            g["qualifiers"] = i.get("qualifiers", [])
 
         # DEFINE AGE RATING
         rating = None