Skip to content

Commit

Permalink
Always log to log file but allow console logging muting
Browse files Browse the repository at this point in the history
- always logs program information to log file, but when `log_silently` is set to `True`, program does not display anything to the console
  - when `log_silently` is set to `False` (default), program logs program information to both the log file AND displays the same information in the console
  • Loading branch information
shailshouryya committed Dec 31, 2020
1 parent 6d9d538 commit fb83118
Show file tree
Hide file tree
Showing 7 changed files with 29 additions and 32 deletions.
4 changes: 2 additions & 2 deletions python/dev/__init__.py
Expand Up @@ -178,7 +178,7 @@ def __str__(self):
'''


def create_list_for(self, url=None, file_name=None, channel=None, channel_type=None, log_to_file=False):
def create_list_for(self, url=None, file_name=None, channel=None, channel_type=None, log_silently=False):
'''
The create_list_for() method creates a list using the arguments specified during instantiation of the ListCreator object.
You need to specify just the url to the channel you want to scrape.
Expand All @@ -197,4 +197,4 @@ def create_list_for(self, url=None, file_name=None, channel=None, channel_type=N
raise RuntimeError(Common().missing_url + ModuleMessage().url_argument_usage)
_execution_type = 'module'
instance_attributes = (self.txt, self.csv, self.markdown, self.reverse_chronological, self.headless, self.scroll_pause_time, self.driver)
return execute.logic(channel, channel_type, file_name, log_to_file, *instance_attributes, _execution_type)
return execute.logic(channel, channel_type, file_name, log_silently, *instance_attributes, _execution_type)
3 changes: 2 additions & 1 deletion python/dev/custom_logger.py
Expand Up @@ -11,7 +11,8 @@ def log(message, logging_output_location):
isoformat = datetime.datetime.isoformat
now = datetime.datetime.now
message = f'===>{thread_name:>>14} {isoformat(now())}: {message}\n'
logging_output_location.writelines(message)
for location in logging_output_location:
location.writelines(message)


def log_extraction_information(module, writer_function, args, kwargs):
Expand Down
23 changes: 10 additions & 13 deletions python/dev/execute.py
Expand Up @@ -13,7 +13,7 @@
from .custom_logger import log


def logic(channel, channel_type, file_name, log_to_file, txt, csv, markdown, reverse_chronological, headless, scroll_pause_time, user_driver, execution_type):
def logic(channel, channel_type, file_name, log_silently, txt, csv, markdown, reverse_chronological, headless, scroll_pause_time, user_driver, execution_type):
common_message = Common()
module_message = ModuleMessage()
script_message = ScriptMessage()
Expand Down Expand Up @@ -141,14 +141,11 @@ def show_user_how_to_set_up_selenium():


@contextlib.contextmanager
def yield_file_writer(file_name):
def yield_logger(file_name):
log_file = f'{file_name}.log'
with open (log_file, 'a', encoding='utf-8') as output_location:
yield output_location

@contextlib.contextmanager
def yield_stdout_writer():
yield sys.stdout
if log_silently is True: yield (output_location,)
else: yield (output_location, sys.stdout)


user_os = determine_user_os()
Expand All @@ -173,12 +170,12 @@ def yield_stdout_writer():
driver.set_window_size(780, 800)
driver.set_window_position(0, 0)
file_name = determine_file_name()
with yield_file_writer(file_name) if log_to_file is True else yield_stdout_writer() as logging_output_location:
log( '>' * 50 + 'STARTING PROGRAM' + '<' * 50, logging_output_location)
log(f'Now scraping {url} using the {user_driver}driver:', logging_output_location)
program.determine_action(url, driver, scroll_pause_time, reverse_chronological, file_name, txt, csv, markdown, logging_output_location)
with yield_logger(file_name) as logging_locations:
log( '>' * 50 + 'STARTING PROGRAM' + '<' * 50, logging_locations)
log(f'Now scraping {url} using the {user_driver}driver:', logging_locations)
program.determine_action(url, driver, scroll_pause_time, reverse_chronological, file_name, txt, csv, markdown, logging_locations)
program_end = time.perf_counter()
total_time = program_end - program_start
log(f'This program took {total_time} seconds to complete.', logging_output_location)
log( '>' * 50 + 'PROGRAM COMPLETED' + '<' * 50, logging_output_location)
log(f'This program took {total_time} seconds to complete.', logging_locations)
log( '>' * 50 + 'PROGRAM COMPLETED' + '<' * 50, logging_locations)
return file_name
2 changes: 1 addition & 1 deletion python/tests/test_shared.py
Expand Up @@ -185,7 +185,7 @@ def verify_update(driver, schafer5_url, test_file, full_file, log_file):
if is_reverse_chronological: suffix = 'reverse_chronological_videos_list'
else: suffix = 'chronological_videos_list'
create_file(test_file, suffix, log_file) # the file this function creates should be the SAME as the returned string to the file_name variable in the next line
test_output_file = driver.create_list_for(schafer5_url, log_to_file=True)
test_output_file = driver.create_list_for(schafer5_url, log_silently=True)
# verify calling the create_list_for() method updates the partial file properly
failed = compare_test_files_to_reference_files(full_file, test_output_file, log_file)
if failed == 'Failed!':
Expand Down
4 changes: 2 additions & 2 deletions python/yt_videos_list/__init__.py
Expand Up @@ -178,7 +178,7 @@ def __str__(self):
'''


def create_list_for(self, url=None, file_name=None, channel=None, channel_type=None, log_to_file=False):
def create_list_for(self, url=None, file_name=None, channel=None, channel_type=None, log_silently=False):
'''
The create_list_for() method creates a list using the arguments specified during instantiation of the ListCreator object.
You need to specify just the url to the channel you want to scrape.
Expand All @@ -197,4 +197,4 @@ def create_list_for(self, url=None, file_name=None, channel=None, channel_type=N
raise RuntimeError(Common().missing_url + ModuleMessage().url_argument_usage)
_execution_type = 'module'
instance_attributes = (self.txt, self.csv, self.markdown, self.reverse_chronological, self.headless, self.scroll_pause_time, self.driver)
return execute.logic(channel, channel_type, file_name, log_to_file, *instance_attributes, _execution_type)
return execute.logic(channel, channel_type, file_name, log_silently, *instance_attributes, _execution_type)
3 changes: 2 additions & 1 deletion python/yt_videos_list/custom_logger.py
Expand Up @@ -9,7 +9,8 @@ def log(message, logging_output_location):
isoformat = datetime.datetime.isoformat
now = datetime.datetime.now
message = f'===>{thread_name:>>14} {isoformat(now())}: {message}\n'
logging_output_location.writelines(message)
for location in logging_output_location:
location.writelines(message)
def log_extraction_information(module, writer_function, args, kwargs):
start_time = time.perf_counter()
extension = writer_function.__name__.split('_')[-1]
Expand Down
22 changes: 10 additions & 12 deletions python/yt_videos_list/execute.py
Expand Up @@ -9,7 +9,7 @@
from .download.user_os_info import determine_user_os
from .notifications import Common, ModuleMessage, ScriptMessage
from .custom_logger import log
def logic(channel, channel_type, file_name, log_to_file, txt, csv, markdown, reverse_chronological, headless, scroll_pause_time, user_driver, execution_type):
def logic(channel, channel_type, file_name, log_silently, txt, csv, markdown, reverse_chronological, headless, scroll_pause_time, user_driver, execution_type):
common_message = Common()
module_message = ModuleMessage()
script_message = ScriptMessage()
Expand Down Expand Up @@ -109,13 +109,11 @@ def show_user_how_to_set_up_selenium():
common_message.tell_user_to_download_driver(user_driver)
common_message.display_dependency_setup_instructions(user_driver, user_os)
@contextlib.contextmanager
def yield_file_writer(file_name):
def yield_logger(file_name):
log_file = f'{file_name}.log'
with open (log_file, 'a', encoding='utf-8') as output_location:
yield output_location
@contextlib.contextmanager
def yield_stdout_writer():
yield sys.stdout
if log_silently is True: yield (output_location,)
else: yield (output_location, sys.stdout)
user_os = determine_user_os()
url, seleniumdriver = check_user_input()
program_start = time.perf_counter()
Expand All @@ -136,12 +134,12 @@ def yield_stdout_writer():
driver.set_window_size(780, 800)
driver.set_window_position(0, 0)
file_name = determine_file_name()
with yield_file_writer(file_name) if log_to_file is True else yield_stdout_writer() as logging_output_location:
log( '>' * 50 + 'STARTING PROGRAM' + '<' * 50, logging_output_location)
log(f'Now scraping {url} using the {user_driver}driver:', logging_output_location)
program.determine_action(url, driver, scroll_pause_time, reverse_chronological, file_name, txt, csv, markdown, logging_output_location)
with yield_logger(file_name) as logging_locations:
log( '>' * 50 + 'STARTING PROGRAM' + '<' * 50, logging_locations)
log(f'Now scraping {url} using the {user_driver}driver:', logging_locations)
program.determine_action(url, driver, scroll_pause_time, reverse_chronological, file_name, txt, csv, markdown, logging_locations)
program_end = time.perf_counter()
total_time = program_end - program_start
log(f'This program took {total_time} seconds to complete.', logging_output_location)
log( '>' * 50 + 'PROGRAM COMPLETED' + '<' * 50, logging_output_location)
log(f'This program took {total_time} seconds to complete.', logging_locations)
log( '>' * 50 + 'PROGRAM COMPLETED' + '<' * 50, logging_locations)
return file_name

0 comments on commit fb83118

Please sign in to comment.