Skip to content

Commit

Permalink
formatting with black and isort
Browse files Browse the repository at this point in the history
  • Loading branch information
@stared
stared committed Dec 8, 2022
1 parent df25201 commit 54f91c0
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 20 deletions.
16 changes: 8 additions & 8 deletions bdfr/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
click.option("-t", "--time", type=click.Choice(("all", "hour", "day", "week", "month", "year")), default=None),
click.option("-u", "--user", type=str, multiple=True, default=None),
click.option("-v", "--verbose", default=None, count=True),
click.option('--progress-bar', is_flag=True, default=False),
click.option("--progress-bar", is_flag=True, default=False),
]

_downloader_options = [
Expand Down Expand Up @@ -123,7 +123,7 @@ def cli_archive(context: click.Context, **_):
def cli_clone(context: click.Context, **_):
config = Configuration()
config.process_click_arguments(context)
setup_logging(config.verbose, config.progress_bar),
setup_logging(config.verbose, config.progress_bar),
try:
reddit_scraper = RedditCloner(config)
reddit_scraper.download()
Expand All @@ -142,13 +142,13 @@ def filter(self, record: logging.LogRecord) -> bool:

if progress_bar:
logger.setLevel(logging.CRITICAL)
logging.getLogger('progress_bar').setLevel(logging.INFO)
logging.getLogger("progress_bar").setLevel(logging.INFO)
else:
logger.setLevel(1)
stream = logging.StreamHandler(sys.stdout)
stream.addFilter(StreamExceptionFilter())

formatter = logging.Formatter('[%(asctime)s - %(name)s - %(levelname)s] - %(message)s')
formatter = logging.Formatter("[%(asctime)s - %(name)s - %(levelname)s] - %(message)s")
stream.setFormatter(formatter)

logger.addHandler(stream)
Expand All @@ -159,11 +159,11 @@ def filter(self, record: logging.LogRecord) -> bool:
else:
stream.setLevel(9)

logging.getLogger('progress_bar').setLevel(logging.CRITICAL)
logging.getLogger("progress_bar").setLevel(logging.CRITICAL)

logging.getLogger('praw').setLevel(logging.CRITICAL)
logging.getLogger('prawcore').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger("praw").setLevel(logging.CRITICAL)
logging.getLogger("prawcore").setLevel(logging.CRITICAL)
logging.getLogger("urllib3").setLevel(logging.CRITICAL)


if __name__ == "__main__":
Expand Down
26 changes: 15 additions & 11 deletions bdfr/downloader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,22 @@
from datetime import datetime
from multiprocessing import Pool
from pathlib import Path
from tqdm import tqdm
from tqdm.contrib.logging import logging_redirect_tqdm

import praw
import praw.exceptions
import praw.models
import prawcore
from tqdm import tqdm
from tqdm.contrib.logging import logging_redirect_tqdm

from bdfr import exceptions as errors
from bdfr.configuration import Configuration
from bdfr.connector import RedditConnector
from bdfr.site_downloaders.download_factory import DownloadFactory

logger = logging.getLogger(__name__)
logger_progress_bar = logging.getLogger('progress_bar')
logger_progress_bar = logging.getLogger("progress_bar")


def _calc_hash(existing_file: Path):
chunk_size = 1024 * 1024
Expand All @@ -36,6 +37,7 @@ def _calc_hash(existing_file: Path):
file_hash = md5_hash.hexdigest()
return existing_file, file_hash


class RedditDownloader(RedditConnector):
def __init__(self, args: Configuration):
super(RedditDownloader, self).__init__(args)
Expand All @@ -48,23 +50,23 @@ def download(self):
for generator in tqdm(self.reddit_lists, desc="Subreddits", unit="subreddit", colour="red"):
desc = generator.url if isinstance(generator, praw.models.ListingGenerator) else "Posts"
for submission in tqdm(list(generator), desc=desc, unit="post", colour="green", leave=False):
title_short = submission.title[:60] + (submission.title[60:] and '...')
log_str = f'{submission.score:5d}🔼 {title_short}'
title_short = submission.title[:60] + (submission.title[60:] and "...")
log_str = f"{submission.score:5d}🔼 {title_short}"
try:
if self._download_submission(submission):
logger_progress_bar.info(f'✅ {log_str}')
logger_progress_bar.info(f"✅ {log_str}")
else:
logger_progress_bar.info(f'❌ {log_str}')
logger_progress_bar.info(f"❌ {log_str}")
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to download due to a PRAW exception: {e}')
logger_progress_bar.info(f'❌ {log_str}')
logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
logger_progress_bar.info(f"❌ {log_str}")
else:
for generator in self.reddit_lists:
for submission in generator:
try:
self._download_submission(submission)
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to download due to a PRAW exception: {e}')
logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")

def _download_submission(self, submission: praw.models.Submission):
if submission.id in self.excluded_submission_ids:
Expand Down Expand Up @@ -163,7 +165,9 @@ def _download_submission(self, submission: praw.models.Submission):
self.master_hash_list[resource_hash] = destination
logger.debug(f"Hash added to master list: {resource_hash}")
logger.info(f"Downloaded submission {submission.id} from {submission.subreddit.display_name}")
logger_progress_bar.info(f"✓ {submission.subreddit.display_name} ({submission.score}) {submission.title} - {submission.author}")
logger_progress_bar.info(
f"✓ {submission.subreddit.display_name} ({submission.score}) {submission.title} - {submission.author}"
)
return True

@staticmethod
Expand Down
7 changes: 6 additions & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,9 @@

from setuptools import setup

setup(setup_requires=['pbr', 'appdirs'], pbr=True, data_files=[('config', ['bdfr/default_config.cfg'])], python_requires='>=3.9.0')
setup(
setup_requires=["pbr", "appdirs"],
pbr=True,
data_files=[("config", ["bdfr/default_config.cfg"])],
python_requires=">=3.9.0",
)

0 comments on commit 54f91c0

Please sign in to comment.