Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ classifiers = [
"Operating System :: OS Independent",
"Programming Language :: Python",
]
packages = []
packages = [{ include = "*", from = "src" }]
include = [{ path = "tests", format = "sdist" }]
exclude = []

Expand Down
1 change: 1 addition & 0 deletions src/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Package Source."""
1 change: 0 additions & 1 deletion src/html_tracing/utilities/__init__.py

This file was deleted.

8 changes: 8 additions & 0 deletions src/utilities/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
"""Package Utilities."""


from .logger import Logger

__all__ = ["Logger"]

logger = Logger(tracing=True)
69 changes: 69 additions & 0 deletions src/utilities/logger.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""Module Logger."""


from __future__ import annotations

import logging
import sys


class Logger:
"""Interface representing logger utilities."""

def __init__(
self: Logger,
*,
debugging: bool = False,
tracing: bool = False,
newline: bool = True,
) -> None:
formatter = "%(name)s:%(levelname)s => %(msg)s" + "\n" if newline else ""
handler = logging.StreamHandler()
handler.setFormatter(fmt=logging.Formatter(fmt=formatter))
logger = logging.getLogger(name="LOG")
logger.addHandler(hdlr=handler)
logger.setLevel(level=logging.DEBUG if debugging else logging.INFO)

self.logger = logger
self.debugging = debugging
self.tracing = tracing
self.newline = newline

def debug_(self: Logger, msg: str) -> None:
"""Log a message with severity 'DEBUG'."""
self.logger.debug(msg=msg)

def warn_(self: Logger, msg: str) -> None:
"""Log a message with severity 'WARN'."""
self.logger.warning(msg=msg)

def info_(self: Logger, msg: str) -> None:
"""Log a message with severity 'INFO'."""
self.logger.info(msg=msg)

def error_(self: Logger, msg: str) -> None:
"""Log a message with severity 'ERROR'."""
self.logger.error(msg=msg)

def critical_(self: Logger, msg: str) -> None:
"""Log a message with severity 'CRITICAL'."""
self.logger.critical(msg=msg)

def trace_(self: Logger, msg: str | None = None) -> None:
"""Log a message with severity 'DEBUG' tracing the called function."""
if not self.tracing:
return

frame = sys._getframe(1) # noqa: SLF001

message = "TRACE"

if "self" in frame.f_locals:
message += f" - CLASS {frame.f_locals['self'].__class__.__name__}"

message += f" - FUNCTION {frame.f_code.co_name}"

if msg is not None:
message += f" - {msg}"

self.info_(msg=f"{message}")
9 changes: 9 additions & 0 deletions src/web_scraping/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
"""Package Web Scraping."""


from .agents import UserAgents
from .clone import Clone
from .proxies import Proxies, Query
from .session import Session

__all__ = ["UserAgents", "Clone", "Proxies", "Session", "Query"]
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,11 @@

import json
import random
from logging import INFO, basicConfig, info
from pathlib import Path

import requests
from bs4 import BeautifulSoup, ResultSet, Tag

basicConfig(level=INFO)
from utilities import logger


class UserAgents:
Expand Down Expand Up @@ -58,7 +56,6 @@ def fetch(

for endpoint in self.endpoints:
response = requests.get(url=f"{self.url}{endpoint}", timeout=10)
info(response.ok, response.status_code)
soup = BeautifulSoup(markup=response.content, features="html.parser")
data += str(soup.find("table"))

Expand All @@ -81,6 +78,7 @@ def refresh(
self: UserAgents,
) -> None:
"""Refresh the list of user agents."""
logger.trace_()
self.fetch()
self.convert()

Expand Down
Loading