Fix bug in relation with feeds caching files

dev2lead · Jul 17, 2018 · c022d4c · c022d4c
1 parent 704c04d
commit c022d4c
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 48 deletions.
diff --git a/feeds/alienvault.py b/feeds/alienvault.py
@@ -5,7 +5,7 @@
 ##
 # -*- coding: utf-8 -*-
 
-import requests, string, re, ipaddress
+import os, requests, string, re, ipaddress
 
 class alienvault:
     def __init__(self, log, group, agent, timeout):
@@ -15,6 +15,7 @@ def __init__(self, log, group, agent, timeout):
         self.session = requests.Session()
         self.session.headers.update({"User-Agent": agent})
         self.timeout = timeout
+        self.cache = str("assets/cache/{}.txt").format(self.__class__.__name__)
 
     def refresh(self):
         data = []
@@ -26,12 +27,12 @@ def refresh(self):
                     data = data + response.text.splitlines()
             except:
                 self.log.error(str("Request error in '{}' = '{}'").format(self.__class__.__name__, element))
-        if len(data) == 0:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "r+") as fp:
-                data = fp.read().splitlines()
-        else:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "w+") as fp:
+        if len(data) != 0:
+            with open(self.cache, "w+") as fp:
                 fp.write(str("\n").join(data))
+        if len(data) == 0 and os.path.isfile(self.cache):
+            with open(self.cache, "r+") as fp:
+                data = fp.read().splitlines()
         for element in data:
             try:
                 content, revlookup = self.parse(element)

diff --git a/feeds/blocklist.py b/feeds/blocklist.py
@@ -5,7 +5,7 @@
 ##
 # -*- coding: utf-8 -*-
 
-import requests, string, re, ipaddress
+import os, requests, string, re, ipaddress
 
 class blocklist:
     def __init__(self, log, group, agent, timeout):
@@ -15,6 +15,7 @@ def __init__(self, log, group, agent, timeout):
         self.session = requests.Session()
         self.session.headers.update({"User-Agent": agent})
         self.timeout = timeout
+        self.cache = str("assets/cache/{}.txt").format(self.__class__.__name__)
 
     def refresh(self):
         data = []
@@ -26,12 +27,12 @@ def refresh(self):
                     data = data + response.text.splitlines()
             except:
                 self.log.error(str("Request error in '{}' = '{}'").format(self.__class__.__name__, element))
-        if len(data) == 0:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "r+") as fp:
-                data = fp.read().splitlines()
-        else:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "w+") as fp:
+        if len(data) != 0:
+            with open(self.cache, "w+") as fp:
                 fp.write(str("\n").join(data))
+        if len(data) == 0 and os.path.isfile(self.cache):
+            with open(self.cache, "r+") as fp:
+                data = fp.read().splitlines()
         for element in data:
             try:
                 content, revlookup = self.parse(element)

diff --git a/feeds/ciarmy.py b/feeds/ciarmy.py
@@ -5,7 +5,7 @@
 ##
 # -*- coding: utf-8 -*-
 
-import requests, string, re, ipaddress
+import os, requests, string, re, ipaddress
 
 class ciarmy:
     def __init__(self, log, group, agent, timeout):
@@ -15,6 +15,7 @@ def __init__(self, log, group, agent, timeout):
         self.session = requests.Session()
         self.session.headers.update({"User-Agent": agent})
         self.timeout = timeout
+        self.cache = str("assets/cache/{}.txt").format(self.__class__.__name__)
 
     def refresh(self):
         data = []
@@ -26,12 +27,12 @@ def refresh(self):
                     data = data + response.text.splitlines()
             except:
                 self.log.error(str("Request error in '{}' = '{}'").format(self.__class__.__name__, element))
-        if len(data) == 0:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "r+") as fp:
-                data = fp.read().splitlines()
-        else:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "w+") as fp:
+        if len(data) != 0:
+            with open(self.cache, "w+") as fp:
                 fp.write(str("\n").join(data))
+        if len(data) == 0 and os.path.isfile(self.cache):
+            with open(self.cache, "r+") as fp:
+                data = fp.read().splitlines()
         for element in data:
             try:
                 content, revlookup = self.parse(element)

diff --git a/feeds/greensnow.py b/feeds/greensnow.py
@@ -5,7 +5,7 @@
 ##
 # -*- coding: utf-8 -*-
 
-import requests, string, re, ipaddress
+import os, requests, string, re, ipaddress
 
 class greensnow:
     def __init__(self, log, group, agent, timeout):
@@ -15,6 +15,7 @@ def __init__(self, log, group, agent, timeout):
         self.session = requests.Session()
         self.session.headers.update({"User-Agent": agent})
         self.timeout = timeout
+        self.cache = str("assets/cache/{}.txt").format(self.__class__.__name__)
 
     def refresh(self):
         data = []
@@ -26,12 +27,12 @@ def refresh(self):
                     data = data + response.text.splitlines()
             except:
                 self.log.error(str("Request error in '{}' = '{}'").format(self.__class__.__name__, element))
-        if len(data) == 0:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "r+") as fp:
-                data = fp.read().splitlines()
-        else:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "w+") as fp:
+        if len(data) != 0:
+            with open(self.cache, "w+") as fp:
                 fp.write(str("\n").join(data))
+        if len(data) == 0 and os.path.isfile(self.cache):
+            with open(self.cache, "r+") as fp:
+                data = fp.read().splitlines()
         for element in data:
             try:
                 content, revlookup = self.parse(element)

diff --git a/feeds/openphish.py b/feeds/openphish.py
@@ -5,7 +5,7 @@
 ##
 # -*- coding: utf-8 -*-
 
-import requests, string, re, ipaddress
+import os, requests, string, re, ipaddress
 
 class openphish:
     def __init__(self, log, group, agent, timeout):
@@ -15,6 +15,7 @@ def __init__(self, log, group, agent, timeout):
         self.session = requests.Session()
         self.session.headers.update({"User-Agent": agent})
         self.timeout = timeout
+        self.cache = str("assets/cache/{}.txt").format(self.__class__.__name__)
 
     def refresh(self):
         data = []
@@ -26,12 +27,12 @@ def refresh(self):
                     data = data + response.text.splitlines()
             except:
                 self.log.error(str("Request error in '{}' = '{}'").format(self.__class__.__name__, element))
-        if len(data) == 0:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "r+") as fp:
-                data = fp.read().splitlines()
-        else:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "w+") as fp:
+        if len(data) != 0:
+            with open(self.cache, "w+") as fp:
                 fp.write(str("\n").join(data))
+        if len(data) == 0 and os.path.isfile(self.cache):
+            with open(self.cache, "r+") as fp:
+                data = fp.read().splitlines()
         for element in data:
             try:
                 content, revlookup = self.parse(element)

diff --git a/feeds/phishtank.py b/feeds/phishtank.py
@@ -5,7 +5,7 @@
 ##
 # -*- coding: utf-8 -*-
 
-import requests, string, re, ipaddress
+import os, requests, string, re, ipaddress
 
 class phishtank:
     def __init__(self, log, group, agent, timeout):
@@ -15,6 +15,7 @@ def __init__(self, log, group, agent, timeout):
         self.session = requests.Session()
         self.session.headers.update({"User-Agent": agent})
         self.timeout = timeout
+        self.cache = str("assets/cache/{}.txt").format(self.__class__.__name__)
 
     def refresh(self):
         data = []
@@ -26,12 +27,12 @@ def refresh(self):
                     data = data + response.text.splitlines()
             except:
                 self.log.error(str("Request error in '{}' = '{}'").format(self.__class__.__name__, element))
-        if len(data) == 0:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "r+") as fp:
-                data = fp.read().splitlines()
-        else:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "w+") as fp:
+        if len(data) != 0:
+            with open(self.cache, "w+") as fp:
                 fp.write(str("\n").join(data))
+        if len(data) == 0 and os.path.isfile(self.cache):
+            with open(self.cache, "r+") as fp:
+                data = fp.read().splitlines()
         for element in data:
             try:
                 content, revlookup = self.parse(element)

diff --git a/feeds/ransomware.py b/feeds/ransomware.py
@@ -5,7 +5,7 @@
 ##
 # -*- coding: utf-8 -*-
 
-import requests, string, re, ipaddress
+import os, requests, string, re, ipaddress
 
 class ransomware:
     def __init__(self, log, group, agent, timeout):
@@ -16,6 +16,7 @@ def __init__(self, log, group, agent, timeout):
         self.session = requests.Session()
         self.session.headers.update({"User-Agent": agent})
         self.timeout = timeout
+        self.cache = str("assets/cache/{}.txt").format(self.__class__.__name__)
 
     def refresh(self):
         data = []
@@ -27,12 +28,12 @@ def refresh(self):
                     data = data + response.text.splitlines()
             except:
                 self.log.error(str("Request error in '{}' = '{}'").format(self.__class__.__name__, element))
-        if len(data) == 0:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "r+") as fp:
-                data = fp.read().splitlines()
-        else:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "w+") as fp:
+        if len(data) != 0:
+            with open(self.cache, "w+") as fp:
                 fp.write(str("\n").join(data))
+        if len(data) == 0 and os.path.isfile(self.cache):
+            with open(self.cache, "r+") as fp:
+                data = fp.read().splitlines()
         for element in data:
             try:
                 content, revlookup = self.parse(element)

diff --git a/feeds/spamhaus.py b/feeds/spamhaus.py
@@ -5,7 +5,7 @@
 ##
 # -*- coding: utf-8 -*-
 
-import requests, string, re, ipaddress
+import os, requests, string, re, ipaddress
 
 class spamhaus:
     def __init__(self, log, group, agent, timeout):
@@ -16,6 +16,7 @@ def __init__(self, log, group, agent, timeout):
         self.session = requests.Session()
         self.session.headers.update({"User-Agent": agent})
         self.timeout = timeout
+        self.cache = str("assets/cache/{}.txt").format(self.__class__.__name__)
 
     def refresh(self):
         data = []
@@ -27,12 +28,12 @@ def refresh(self):
                     data = data + response.text.splitlines()
             except:
                 self.log.error(str("Request error in '{}' = '{}'").format(self.__class__.__name__, element))
-        if len(data) == 0:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "r+") as fp:
-                data = fp.read().splitlines()
-        else:
-            with open(str("assets/cache/{}.txt").format(self.__class__.__name__), "w+") as fp:
+        if len(data) != 0:
+            with open(self.cache, "w+") as fp:
                 fp.write(str("\n").join(data))
+        if len(data) == 0 and os.path.isfile(self.cache):
+            with open(self.cache, "r+") as fp:
+                data = fp.read().splitlines()
         for element in data:
             try:
                 content, revlookup = self.parse(element)