Archive events.log and errors.log

Swiftea · Feb 11, 2016 · a2934eb · a2934eb
1 parent 4632505
commit a2934eb
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 5 deletions.
diff --git a/crawler/swiftea_bot/data.py b/crawler/swiftea_bot/data.py
@@ -16,8 +16,8 @@
 DIR_OUTPUT = DIR_DATA + 'output/'
 DIR_INDEX = DIR_DATA + 'inverted_index/'
 DIR_STATS = DIR_DATA + 'stats/'
-FILE_NEWS = DIR_CONFIG + 'events.log'
-FILE_ERROR = DIR_CONFIG + 'errors.log'
+FILE_EVENTS = DIR_CONFIG + 'events.log'
+FILE_ERRORS = DIR_CONFIG + 'errors.log'
 FILE_CONFIG = DIR_CONFIG + 'config.ini'
 FILE_DOC = DIR_CONFIG + 'Readme'
 FILE_BASELINKS = DIR_LINKS + '0'

diff --git a/crawler/swiftea_bot/file_manager.py b/crawler/swiftea_bot/file_manager.py
@@ -8,7 +8,7 @@
 from configparser import ConfigParser
 import json
 
-from swiftea_bot.data import MAX_LINKS, FILE_CONFIG, DIR_LINKS, FILE_INDEX, DIR_INDEX, DIR_DATA
+from swiftea_bot.data import MAX_LINKS, FILE_CONFIG, DIR_LINKS, FILE_INDEX, DIR_INDEX, DIR_DATA, FILE_EVENTS, FILE_ERRORS
 from swiftea_bot.module import tell, remove_duplicates, convert_keys
 
 class FileManager(object):
@@ -104,6 +104,23 @@ def ckeck_size_links(self, links):
 				str(self.writing_file_number)), severity=-1
 			)
 
+	def check_size_files(self):
+		for filelog in [FILE_EVENTS, FILE_ERRORS]:
+			filearchive = filelog[:-3] + 'zip'
+			with open(filelog, 'r') as myfile:
+				content = myfile.readlines()
+			if len(content) > MAX_SIZE:
+				if not path.exists(filearchive):
+					zipfile.ZipFile(file=filearchive, mode='w').close()
+					filename = '0'
+				else:
+					with zipfile.ZipFile(filearchive, 'r') as myzip:
+						filename = str(int(myzip.namelist()[-1])+1)  # The last one +1
+				rename(filelog, filename)
+				with zipfile.ZipFile(filearchive, 'w') as myzip:
+					myzip.write(filename)
+				remove(filename)
+				tell('Archiving ' + filelog + ': ' + filename, severity=-1)
 
 	def get_url(self):
 		"""Get url of next webpage.

diff --git a/crawler/swiftea_bot/module.py b/crawler/swiftea_bot/module.py
@@ -36,7 +36,7 @@ def tell(message, error_code='', severity=1):
 		print(msg_to_print.upper())
 		print(''.center(len(msg_to_print), '='))
 
-	with open(data.FILE_NEWS, 'a') as myfile:
+	with open(data.FILE_EVENTS, 'a') as myfile:
 		myfile.write(strftime('%d/%m/%y %H:%M:%S') + str(error_code) + ' ' + message + '\n')
 
 def errors(message, error_code):
@@ -50,7 +50,7 @@ def errors(message, error_code):
 	:type error_code: int
 
 	"""
-	with open(data.FILE_ERROR, 'a') as myfile:
+	with open(data.FILE_ERRORS, 'a') as myfile:
 		myfile.write(' ' + str(error_code) + ' ' + strftime("%d/%m/%y %H:%M:%S") + ': ' + message + '\n')