diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e05bafb..5f7a0b2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -29,6 +29,6 @@ jobs: run: python setup.py sdist bdist_wheel - name: Upload to PyPI env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: twine upload dist/* --skip-existing + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: twine upload dist/* diff --git a/README.md b/README.md index 5391ade..ba2c5c5 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ usage: ytmdl [-h] [-q] [-o OUTPUT_DIR] [--song SONG-METADATA] [--filename NAME] [--pl-start NUMBER] [--pl-end NUMBER] [--pl-items ITEM_SPEC] [--ignore-errors] [--title-as-name] [--level LEVEL] [--disable-file] [--list-level] - [SONG_NAME [SONG_NAME ...]] + [SONG_NAME ...] positional arguments: SONG_NAME Name of the song to download. Can be an URL to a @@ -251,7 +251,8 @@ Metadata: --on-meta-error ON_META_ERROR What to do if adding the metadata fails for some reason like lack of metadata or perhaps a network - issue. Options are ['exit', 'skip', 'manual'] + issue. Options are ['exit', 'skip', 'manual', + 'youtube'] Playlist: --pl-start NUMBER Playlist video to start at (default is 1) diff --git a/setup.py b/setup.py index 99650d5..12e368c 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ 'requests', 'colorama', 'beautifulsoup4', - 'downloader-cli', + 'downloader-cli>=0.3.4', 'pyxdg', 'ffmpeg-python', 'pysocks', diff --git a/ytmdl/__version__.py b/ytmdl/__version__.py index d14eb18..569a07a 100644 --- a/ytmdl/__version__.py +++ b/ytmdl/__version__.py @@ -1,2 +1,2 @@ # Store the version of the package -__version__ = "2023.07.27" +__version__ = "2023.11.26" diff --git a/ytmdl/core.py b/ytmdl/core.py index 88bc057..0d4ea39 100644 --- a/ytmdl/core.py +++ b/ytmdl/core.py @@ -16,6 +16,7 @@ from ytmdl.exceptions import ( DownloadError, ConvertError, NoMetaError, MetadataError ) +from ytmdl.meta.yt import extract_meta_from_yt logger = Logger("core") @@ -200,7 +201,7 @@ def trim(name: str, args) -> None: trim.Trim(name) -def meta(conv_name: str, song_name: str, search_by: str, args): +def meta(conv_name: str, song_name: str, search_by: str, link: str, args): """Handle adding the metadata for the passed song. We will use the passed name to search for metadata, ask @@ -236,14 +237,26 @@ def meta(conv_name: str, song_name: str, search_by: str, args): # If no meta was found raise error if not TRACK_INFO: - # Check if we are supposed to add manual meta - if args.on_meta_error != "manual": + # Check if we are supposed to add manual meta or from youtube + if args.on_meta_error not in ["manual", "youtube"]: raise NoMetaError(search_by) - - TRACK_INFO = manual.get_data(song_name) - return TRACK_INFO + + if args.on_meta_error == "manual": + TRACK_INFO = manual.get_data(song_name) + elif args.on_meta_error == 'youtube': + # Extract meta from youtube + track_info = extract_meta_from_yt(link) + TRACK_INFO = [track_info] + + option = song.setData(TRACK_INFO, IS_QUIET, conv_name, PASSED_FORMAT, 0, skip_showing_choice=True) + if not isinstance(option, int): + raise MetadataError(search_by) + + return TRACK_INFO[option] logger.info('Setting data...') + + option = song.setData(TRACK_INFO, IS_QUIET, conv_name, PASSED_FORMAT, args.choice) @@ -260,6 +273,6 @@ def meta(conv_name: str, song_name: str, search_by: str, args): logger.info( "Amending the search because -2 was entered as the option") search_by = utility.get_new_meta_search_by(search_by) - return meta(conv_name, song_name, search_by, args) + return meta(conv_name, song_name, search_by, link, args) return TRACK_INFO[option] diff --git a/ytmdl/dir.py b/ytmdl/dir.py index ae1f4a7..41af0ea 100644 --- a/ytmdl/dir.py +++ b/ytmdl/dir.py @@ -22,6 +22,17 @@ def __replace_special_characters(passed_name: str) -> str: return sub(r'/', '-', passed_name) +def get_abs_path(path_passed: str) -> str: + """ + Get the absolute path by removing special path directives + that `ytmdl` supports. + """ + if "$" not in path_passed: + return path_passed + + return path_passed.split("$")[0] + + def cleanup(TRACK_INFO, index, datatype, remove_cached=True, filename_passed=None): """Move the song from temp to the song dir.""" try: @@ -42,7 +53,7 @@ def cleanup(TRACK_INFO, index, datatype, remove_cached=True, filename_passed=Non SONG_NAME = filename_passed + ".{}".format(datatype) DIR = defaults.DEFAULT.SONG_DIR - logger.debug(DIR) + logger.debug("directory being used: ", DIR) # Check if DIR has $ in its path # If it does then make those folders accordingly diff --git a/ytmdl/main.py b/ytmdl/main.py index 542f929..28be9ca 100644 --- a/ytmdl/main.py +++ b/ytmdl/main.py @@ -411,7 +411,7 @@ def post_processing( # Else fill the meta by searching try: - track_selected = meta(conv_name, song_name, song_metadata, args) + track_selected = meta(conv_name, song_name, song_metadata, link, args) except NoMetaError as no_meta_error: if args.on_meta_error == 'skip': # Write to the archive file @@ -493,7 +493,7 @@ def pre_checks(args): # Ensure the output directory is legitimate if (args.output_dir is not None): - if path.isdir(path.expanduser(args.output_dir)): + if path.isdir(path.expanduser(dir.get_abs_path(args.output_dir))): defaults.DEFAULT.SONG_DIR = path.expanduser(args.output_dir) else: logger.warning( diff --git a/ytmdl/manual.py b/ytmdl/manual.py index eb2f260..176d343 100644 --- a/ytmdl/manual.py +++ b/ytmdl/manual.py @@ -26,14 +26,24 @@ class Meta: track_number : Number of the track in the album artwork_url_100 : URL of the album cover """ - def __init__(self): - self.release_date = "{}T00:00:00Z".format(datetime.now().date()) - self.track_name = "N/A" - self.artist_name = "N/A" - self.collection_name = "N/A" - self.primary_genre_name = "N/A" - self.track_number = "1" - self.artwork_url_100 = "" + def __init__( + self, + release_date: str = None, + track_name: str = "N/A", + artist_name: str = "N/A", + collection_name: str = "N/A", + primary_genre_name: str = "N/A", + track_number: str = "1", + artwork_url_100: str = "" + ): + self.release_date = "{}T00:00:00Z".format(datetime.now().date()) if \ + release_date is None else release_date + self.track_name = track_name + self.artist_name = artist_name + self.collection_name = collection_name + self.primary_genre_name = primary_genre_name + self.track_number = track_number + self.artwork_url_100 = artwork_url_100 def _read_individual(self, default_value): """ diff --git a/ytmdl/meta/yt.py b/ytmdl/meta/yt.py new file mode 100644 index 0000000..fc5e5e2 --- /dev/null +++ b/ytmdl/meta/yt.py @@ -0,0 +1,54 @@ +""" +Handle metadata extraction from youtube +""" + +from typing import Dict, List +from datetime import datetime + +from yt_dlp import YoutubeDL +from simber import Logger + +from ytmdl.manual import Meta +from ytmdl.utils.ytdl import get_ytdl_opts +from ytmdl.exceptions import ExtractError + +# Create logger +logger = Logger("meta:yt") + + +def __parse_release_date_from_utc(timestamp: int) -> str: + if timestamp is None: + return None + + dt_object = datetime.utcfromtimestamp(timestamp) + return dt_object.strftime('%Y-%m-%dT%H:%M:%SZ') + +def __parse_genre_name_from_categories(categories: List[str]) -> str: + return categories[0] if len(categories) else "N/A" + +def __parse_meta_from_details(details: Dict) -> Meta: + """ + Parse the meta object from the passed details + """ + return Meta( + release_date=__parse_release_date_from_utc(details.get("release_timestamp", None)), + track_name=details.get("title", "N/A"), + artist_name=details.get("channel", "N/A"), + primary_genre_name=__parse_genre_name_from_categories(details.get("categories", [])), + artwork_url_100=details.get("thumbnail", "N/A") + ) + +def extract_meta_from_yt(video_url: str) -> Meta: + """ + Extract the metadata from the passed video ID and return + it accordingly. + """ + ytdl_obj = YoutubeDL(get_ytdl_opts()) + + try: + details = ytdl_obj.extract_info(video_url, download=False) + return __parse_meta_from_details(details) + except Exception as e: + logger.debug("Got exception while extracting details for video: ", video_url) + logger.warning("Failed to extract metadata from yt with exception: ", str(e)) + raise ExtractError(f"error extracting data from yt: {str(e)}") diff --git a/ytmdl/setupConfig.py b/ytmdl/setupConfig.py index 91d571d..daa24d6 100644 --- a/ytmdl/setupConfig.py +++ b/ytmdl/setupConfig.py @@ -139,7 +139,7 @@ def __init__(self): self.DEFAULT_FORMAT = 'mp3' - self.ON_ERROR_OPTIONS = ['exit', 'skip', 'manual'] + self.ON_ERROR_OPTIONS = ['exit', 'skip', 'manual', 'youtube'] self.ON_ERROR_DEFAULT = 'exit' diff --git a/ytmdl/song.py b/ytmdl/song.py index aa5503c..cd975fa 100644 --- a/ytmdl/song.py +++ b/ytmdl/song.py @@ -384,7 +384,7 @@ def _get_option(SONG_INFO, is_quiet, choice): return int(option) -def setData(SONG_INFO, is_quiet, song_path, datatype='mp3', choice=None): +def setData(SONG_INFO, is_quiet, song_path, datatype='mp3', choice=None, skip_showing_choice: bool = False): """Add the metadata to the song.""" # Some providers need extra daa from other endpoints, @@ -392,7 +392,11 @@ def setData(SONG_INFO, is_quiet, song_path, datatype='mp3', choice=None): # it from logger.debug(choice) - option = _get_option(SONG_INFO, is_quiet, choice) + option = choice + + if not skip_showing_choice: + option = _get_option(SONG_INFO, is_quiet, choice) + logger.debug(option) # If -1 or -2 then skip setting the metadata diff --git a/ytmdl/stringutils.py b/ytmdl/stringutils.py index 95243f7..6065beb 100644 --- a/ytmdl/stringutils.py +++ b/ytmdl/stringutils.py @@ -57,7 +57,7 @@ def compute_jaccard(tokens1, tokens2): return len(intersect)/len(union) def remove_unwanted_chars(string): - return re.sub(r" |/", "#", string) + return re.sub(r"\s|\||/", "#", string) def urlencode(text): diff --git a/ytmdl/utils/ytdl.py b/ytmdl/utils/ytdl.py index 7bc566a..31c4f99 100644 --- a/ytmdl/utils/ytdl.py +++ b/ytmdl/utils/ytdl.py @@ -14,6 +14,20 @@ logger = Logger("yt") +def get_ytdl_opts() -> Dict: + is_quiet: bool = utility.determine_logger_level( + ) != logger.level_map["DEBUG"] + no_warnings: bool = utility.determine_logger_level( + ) > logger.level_map["WARNING"] + + return { + "quiet": is_quiet, + 'no_warnings': no_warnings, + 'nocheckcertificate': True, + 'source_address': '0.0.0.0' + } + + def is_ytdl_config_present(path_passed: str) -> bool: """ Check if the passed file is present or not. @@ -40,17 +54,7 @@ def ydl_opts_with_config(ytdl_config: str = None) -> Dict: If the config is not present, return an empty dictionary """ - is_quiet: bool = utility.determine_logger_level( - ) != logger.level_map["DEBUG"] - no_warnings: bool = utility.determine_logger_level( - ) > logger.level_map["WARNING"] - - ydl_opts = { - "quiet": is_quiet, - 'no_warnings': no_warnings, - 'nocheckcertificate': True, - 'source_address': '0.0.0.0' - } + ydl_opts = get_ytdl_opts() # If config is passed, generated opts with config if ytdl_config is not None: diff --git a/ytmdl/yt.py b/ytmdl/yt.py index 3a73772..59ad466 100644 --- a/ytmdl/yt.py +++ b/ytmdl/yt.py @@ -37,7 +37,8 @@ def get_youtube_streams(url): # Function to be called by ytdl progress hook. def progress_handler(d): - d_obj = Download('', '') + d_obj = Download('', '', icon_done="━", icon_left="━", + icon_current=" ", color_done="green", color_left="black", icon_border=" ") if d['status'] == 'downloading': length = d_obj._get_terminal_length()