From f637494d99146b84a141ccae31fe124479243b19 Mon Sep 17 00:00:00 2001 From: MrTeferi Date: Fri, 22 Jul 2022 05:37:09 -0500 Subject: [PATCH] Major improvements for v1.1.6 - Implemented new web scraping methodology by looking at set checklist and pulling by collector number, in the case of judge promo and misc promo cards use card name, artist name, and string "likeness" for set name. Much more accurate than the previous method. - Deprecated "alternate card" system as it is thus far unneeded with the new methodology - Deprecated download_special() and special.json as it is thus far unneeded with the new methodology - Deprecated core.handle() as it was never used, may write a log_exception() in the future - Implemented new print queue system using constants.console - Implemented scryfall commands using best practices from the scryfall API (docs pending) - Added card count - Added unidecode to reqs --- build.py | 1 - cards.txt | 530 +++++++++++++++++++++++----------------------- lib/card.py | 82 +++---- lib/codes.json | 3 +- lib/core.py | 138 +++++++++--- lib/scryfall.json | 8 + lib/settings.py | 4 +- lib/special.json | 6 - main.py | 99 +++------ requirements.txt | 3 +- 10 files changed, 450 insertions(+), 424 deletions(-) create mode 100644 lib/scryfall.json delete mode 100644 lib/special.json diff --git a/build.py b/build.py index 87a933f..5d282f8 100644 --- a/build.py +++ b/build.py @@ -24,7 +24,6 @@ # --- SOURCE DIRECTORY {'src': os.path.join(MTG, 'codes.json'), 'dst': os.path.join(DIST_MTG, 'codes.json')}, {'src': os.path.join(MTG, 'links.json'), 'dst': os.path.join(DIST_MTG, 'links.json')}, - {'src': os.path.join(MTG, 'special.json'), 'dst': os.path.join(DIST_MTG, 'special.json')}, {'src': os.path.join(MTG, 'scryfall.json'), 'dst': os.path.join(DIST_MTG, 'scryfall.json')}, ] diff --git a/cards.txt b/cards.txt index a73d89e..3a4e2af 100644 --- a/cards.txt +++ b/cards.txt @@ -1,266 +1,264 @@ -A Little Chat (SNC) -All-Seeing Arbiter (SNC) -Angelic Observer (SNC) -snc--Angel of Suffering -snc--An Offer You Can't Refuse -snc--Antagonize -snc--Arcane Bombardment -snc--Arc Spitter -snc--Attended Socialite -snc--Aven Heartstabber -snc--Backstreet Bruiser -snc--Backup Agent -snc--Ballroom Brawlers -snc--Big Score -snc--Black Market Tycoon -snc--Body Dropper -snc--Body Launderer -snc--Boon of Safety -snc--Bootleggers' Stash -snc--Botanical Plaza -snc--Bouncer's Beatdown -snc--Brass Knuckles -snc--Brazen Upstart -snc--Broken Wings -snc--Brokers Ascendancy -snc--Brokers Charm -snc--Brokers Hideout -snc--Brokers Initiate -snc--Brokers Veteran -snc--Buy Your Silence -snc--Cabaretti Ascendancy -snc--Cabaretti Charm -snc--Cabaretti Courtyard -snc--Cabaretti Initiate -snc--Caldaia Strongarm -snc--Call In a Professional -snc--Capenna Express -snc--Case the Joint -snc--Celebrity Fencer -snc--Celestial Regulator -snc--Cement Shoes -snc--Cemetery Tampering -snc--Ceremonial Groundbreaker -snc--Chrome Cat -snc--Citizen's Crowbar -snc--Civic Gardener -snc--Civil Servant -snc--Cleanup Crew -snc--Cormela, Glamour Thief -snc--Corpse Appraiser -snc--Corpse Explosion -snc--Corrupt Court Official -snc--Courier's Briefcase -snc--Crew Captain -snc--Crooked Custodian -snc--Cut of the Profits -snc--Cutthroat Contender -snc--Cut Your Losses -snc--Dapper Shieldmate -snc--Daring Escape -snc--Darling of the Masses -snc--Deal Gone Bad -snc--Demon's Due -snc--Depopulate -snc--Devilish Valet -snc--Dig Up the Body -snc--Disciplined Duelist -snc--Disdainful Stroke -snc--Dusk Mangler -snc--Echo Inspector -snc--Elegant Entourage -snc--Elspeth Resplendent -snc--Endless Detour -snc--Errant, Street Artist -snc--Evelyn, the Covetous -snc--Even the Score -snc--Evolving Door -snc--Exhibition Magician -snc--Exotic Pets -snc--Expendable Lackey -snc--Extraction Specialist -snc--Extract the Truth -snc--Faerie Vandal -snc--Fake Your Own Death -snc--Falco Spara, Pactweaver -snc--Fatal Grudge -snc--Fight Rigging -snc--Fleetfoot Dancer -snc--Forest -snc--Forge Boss -snc--For the Family -snc--Freelance Muscle -snc--Gala Greeters -snc--Gathering Throng -snc--Getaway Car -snc--Giada, Font of Hope -snc--Gilded Pinions -snc--Girder Goons -snc--Glamorous Outlaw -snc--Glittering Stockpile -snc--Glittermonger -snc--Goldhound -snc--Graveyard Shift -snc--Grisly Sigil -snc--Halo Fountain -snc--Halo Scarab -snc--High-Rise Sawjack -snc--Hoard Hauler -snc--Hold for Ransom -snc--Hostile Takeover -snc--Hypnotic Grifter -snc--Illicit Shipment -snc--Illuminator Virtuoso -snc--Incandescent Aria -snc--Incriminate -snc--Inspiring Overseer -snc--Involuntary Employment -snc--Island -snc--Jackhammer -snc--Jaxis, the Troublemaker -snc--Jetmir, Nexus of Revels -snc--Jetmir's Fixer -snc--Jetmir's Garden -snc--Jewel Thief -snc--Jinnie Fay, Jetmir's Second -snc--Join the Maestros -snc--Kill Shot -snc--Knockout Blow -snc--Lagrella, the Magpie -snc--Ledger Shredder -snc--Light 'Em Up -snc--Lord Xander, the Collector -snc--Luxior, Giada's Gift -snc--Luxurious Libation -snc--Maestros Ascendancy -snc--Maestros Charm -snc--Maestros Diabolist -snc--Maestros Initiate -snc--Maestros Theater -snc--Mage's Attendant -snc--Majestic Metamorphosis -snc--Make Disappear -snc--Masked Bandits -snc--Mayhem Patrol -snc--Meeting of the Five -snc--Metropolis Angel -snc--Midnight Assassin -snc--Most Wanted -snc--Mountain -snc--Mr. Orfeo, the Boulder -snc--Murder -snc--Mysterious Limousine -snc--Night Clubber -snc--Nimble Larcenist -snc--Ob Nixilis, the Adversary -snc--Obscura Ascendancy -snc--Obscura Charm -snc--Obscura Initiate -snc--Obscura Interceptor -snc--Obscura Storefront -snc--Ognis, the Dragon's Lash -snc--Ominous Parcel -snc--Out of the Way -snc--Paragon of Modernity -snc--Park Heights Pegasus -snc--Patch Up -snc--Plains -snc--Plasma Jockey -snc--Prizefight -snc--Professional Face-Breaker -snc--Psionic Snoop -snc--Psychic Pickpocket -snc--Public Enemy -snc--Pugnacious Pugilist -snc--Pyre-Sledge Arsonist -snc--Queza, Augur of Agonies -snc--Quick-Draw Dagger -snc--Rabble Rousing -snc--Racers' Ring -snc--Raffine, Scheming Seer -snc--Raffine's Guidance -snc--Raffine's Informant -snc--Raffine's Silencer -snc--Raffine's Tower -snc--Rakish Revelers -snc--Ready to Rumble -snc--Refuse to Yield -snc--Reservoir Kraken -snc--Revelation of Power -snc--Revel Ruiner -snc--Rhox Pummeler -snc--Rigo, Streetwise Mentor -snc--Riveteers Ascendancy -snc--Riveteers Charm -snc--Riveteers Decoy -snc--Riveteers Initiate -snc--Riveteers Overlook -snc--Riveteers Requisitioner -snc--Rob the Archives -snc--Rocco, Cabaretti Caterer -snc--Rogues' Gallery -snc--Rooftop Nuisance -snc--Rumor Gatherer -snc--Run Out of Town -snc--Sanctuary Warden -snc--Sanguine Spy -snc--Scheming Fence -snc--Scuttling Butler -snc--Security Bypass -snc--Security Rhox -snc--Sewer Crocodile -snc--Shadow of Mortality -snc--Shakedown Heavy -snc--Shattered Seraph -snc--Sizzling Soloist -snc--Skybridge Towers -snc--Sky Crier -snc--Sleep with the Fishes -snc--Slip Out the Back -snc--Snooping Newsie -snc--Social Climber -snc--Soul of Emancipation -snc--Spara's Adjudicators -snc--Spara's Headquarters -snc--Speakeasy Server -snc--Sticky Fingers -snc--Stimulus Package -snc--Strangle -snc--Structural Assault -snc--Suspicious Bookcase -snc--Swamp -snc--Swooping Protector -snc--Syndicate Infiltrator -snc--Tainted Indulgence -snc--Take to the Streets -snc--Tavern Swindler -snc--Tenacious Underdog -snc--Titan of Industry -snc--Toluz, Clever Conductor -snc--Topiary Stomper -snc--Torch Breath -snc--Tramway Station -snc--Undercover Operative -snc--Unleash the Inferno -snc--Unlicensed Hearse -snc--Unlucky Witness -snc--Urabrask, Heretic Praetor -snc--Vampire Scrivener -snc--Venom Connoisseur -snc--Vivien on the Hunt -snc--Voice of the Vermin -snc--Void Rend -snc--Warm Welcome -snc--Waterfront District -snc--Whack -snc--Widespread Thieving -snc--Wingshield Agent -snc--Wiretapping -snc--Witness Protection -snc--Witty Roastmaster -snc--Workshop Warchief -snc--Wrecking Crew -snc--Xander's Lounge -snc--Ziatora's Envoy -snc--Ziatora's Proving Ground -snc--Ziatora, the Incinerator \ No newline at end of file +Adaptive Shimmerer +Adventurous Impulse +Aegis Turtle +iko--Alert Heedbonder +iko--Almighty Brushwagg +iko--Anticipate +iko--Archipelagore +iko--Auspicious Starrix +iko--Avian Oddity +iko--Back for More +iko--Barrier Breach +iko--Bastion of Remembrance +iko--Blade Banish +iko--Blazing Volley +iko--Blisterspit Gremlin +iko--Blitz Leech +iko--Blitz of the Thunder-Raptor +iko--Blood Curdle +iko--Bloodfell Caves +iko--Blossoming Sands +iko--Bonders' Enclave +iko--Boneyard Lurker +iko--Boon of the Wish-Giver +iko--Boot Nipper +iko--Bristling Boar +iko--Brokkos, Apex of Forever +iko--Bushmeat Poacher +iko--Call of the Death-Dweller +iko--Capture Sphere +iko--Cathartic Reunion +iko--Cavern Whisperer +iko--Channeled Force +iko--Charge of the Forever-Beast +iko--Checkpoint Officer +iko--Chevill, Bane of Monsters +iko--Chittering Harvester +iko--Clash of Titans +iko--Cloudpiercer +iko--Colossification +iko--Convolute +iko--Coordinated Charge +iko--Corpse Churn +iko--Crystacean +iko--Crystalline Giant +iko--Cubwarden +iko--Cunning Nightbonder +iko--Dark Bargain +iko--Daysquad Marshal +iko--Dead Weight +iko--Death's Oasis +iko--Dire Tactics +iko--Dirge Bat +iko--Dismal Backwater +iko--Divine Arrow +iko--Drannith Healer +iko--Drannith Magistrate +iko--Drannith Stinger +iko--Dreamtail Heron +iko--Durable Coilbug +iko--Duskfang Mentor +iko--Easy Prey +iko--Eerie Ultimatum +iko--Emergent Ultimatum +iko--Escape Protocol +iko--Essence Scatter +iko--Essence Symbiote +iko--Everquill Phoenix +iko--Evolving Wilds +iko--Excavation Mole +iko--Extinction Event +iko--Exuberant Wolfbear +iko--Facet Reader +iko--Farfinder +iko--Ferocious Tigorilla +iko--Fertilid +iko--Fiend Artisan +iko--Fight as One +iko--Fire Prophecy +iko--Flame Spill +iko--Flourishing Fox +iko--Flycatcher Giraffid +iko--Footfall Crater +iko--Forbidden Friendship +iko--Forest +iko--Frenzied Raptor +iko--Frillscare Mentor +iko--Frondland Felidar +iko--Frost Lynx +iko--Frostveil Ambush +iko--Fully Grown +iko--Garrison Cat +iko--Gemrazer +iko--General Kudro of Drannith +iko--General's Enforcer +iko--Genesis Ultimatum +iko--Glimmerbell +iko--Gloom Pangolin +iko--Glowstone Recluse +iko--Go for Blood +iko--Greater Sandwurm +iko--Grimdancer +iko--Gust of Wind +iko--Gyruda, Doom of Depths +iko--Hampering Snare +iko--Heartless Act +iko--Heightened Reflexes +iko--Helica Glider +iko--Honey Mammoth +iko--Hornbash Mentor +iko--Humble Naturalist +iko--Hunted Nightmare +iko--Huntmaster Liger +iko--Illuna, Apex of Wishes +iko--Imposing Vantasaur +iko--Indatha Crystal +iko--Indatha Triome +iko--Insatiable Hemophage +iko--Inspired Ultimatum +iko--Island +iko--Ivy Elemental +iko--Jegantha, the Wellspring +iko--Jubilant Skybonder +iko--Jungle Hollow +iko--Kaheera, the Orphanguard +iko--Keensight Mentor +iko--Keep Safe +iko--Keruga, the Macrosage +iko--Ketria Crystal +iko--Ketria Triome +iko--Kinnan, Bonder Prodigy +iko--Kogla, the Titan Ape +iko--Labyrinth Raptor +iko--Lavabrink Venturer +iko--Lava Serpent +iko--Lead the Stampede +iko--Light of Hope +iko--Lore Drakkis +iko--Lukka, Coppercoat Outcast +iko--Luminous Broodmoth +iko--Lurking Deadeye +iko--Lurrus of the Dream-Den +iko--Lutri, the Spellchaser +iko--Majestic Auricorn +iko--Maned Serval +iko--Memory Leak +iko--Migration Path +iko--Migratory Greathorn +iko--Momentum Rumbler +iko--Monstrous Step +iko--Mosscoat Goriak +iko--Mountain +iko--Mutual Destruction +iko--Mysterious Egg +iko--Mystic Subdual +iko--Mythos of Brokkos +iko--Mythos of Illuna +iko--Mythos of Nethroi +iko--Mythos of Snapdax +iko--Mythos of Vadrok +iko--Narset of the Ancient Way +iko--Necropanther +iko--Nethroi, Apex of Death +iko--Neutralize +iko--Nightsquad Commando +iko--Obosh, the Preypiercer +iko--Offspring's Revenge +iko--Of One Mind +iko--Ominous Seas +iko--Pacifism +iko--Parcelbeast +iko--Patagia Tiger +iko--Perimeter Sergeant +iko--Phase Dolphin +iko--Plains +iko--Plummet +iko--Pollywog Symbiote +iko--Porcuparrot +iko--Pouncing Shoreshark +iko--Prickly Marmoset +iko--Primal Empathy +iko--Proud Wildbonder +iko--Pyroceratops +iko--Quartzwood Crasher +iko--Raking Claws +iko--Ram Through +iko--Raugrin Crystal +iko--Raugrin Triome +iko--Reconnaissance Mission +iko--Regal Leosaur +iko--Reptilian Reflection +iko--Rielle, the Everwise +iko--Rooting Moloch +iko--Rugged Highlands +iko--Ruinous Ultimatum +iko--Rumbling Rockslide +iko--Sanctuary Lockdown +iko--Sanctuary Smasher +iko--Savai Crystal +iko--Savai Sabertooth +iko--Savai Thundermane +iko--Savai Triome +iko--Scoured Barrens +iko--Sea-Dasher Octopus +iko--Serrated Scorpion +iko--Shark Typhoon +iko--Shredded Sails +iko--Skull Prophet +iko--Skycat Sovereign +iko--Sleeper Dart +iko--Slitherwisp +iko--Snapdax, Apex of the Hunt +iko--Snare Tactician +iko--Solid Footing +iko--Song of Creation +iko--Sonorous Howlbonder +iko--Spelleater Wolverine +iko--Splendor Mare +iko--Spontaneous Flight +iko--Springjaw Trap +iko--Sprite Dragon +iko--Startling Development +iko--Stormwild Capridor +iko--Sudden Spinnerets +iko--Suffocating Fumes +iko--Survivors' Bond +iko--Swallow Whole +iko--Swamp +iko--Swiftwater Cliffs +iko--Tentative Connection +iko--The Ozolith +iko--Thieving Otter +iko--Thornwood Falls +iko--Thwart the Enemy +iko--Titanoth Rex +iko--Titans' Nest +iko--Tranquil Cove +iko--Trumpeting Gnarr +iko--Umori, the Collector +iko--Unbreakable Bond +iko--Unexpected Fangs +iko--Unlikely Aid +iko--Unpredictable Cyclone +iko--Vadrok, Apex of Thunder +iko--Valiant Rescuer +iko--Vivien, Monsters' Advocate +iko--Void Beckoner +iko--Voracious Greatshark +iko--Vulpikeet +iko--Weaponize the Monsters +iko--Whirlwind of Thought +iko--Whisper Squad +iko--Will of the All-Hunter +iko--Wilt +iko--Wind-Scarred Crag +iko--Wingfold Pteron +iko--Wingspan Mentor +iko--Winota, Joiner of Forces +iko--Yidaro, Wandering Monster +iko--Yorion, Sky Nomad +iko--Zagoth Crystal +iko--Zagoth Mamba +iko--Zagoth Triome +iko--Zenith Flare +iko--Zirda, the Dawnwaker \ No newline at end of file diff --git a/lib/card.py b/lib/card.py index 48f22e3..73f1ee4 100644 --- a/lib/card.py +++ b/lib/card.py @@ -2,12 +2,14 @@ CARD CLASSES """ import os +import requests from pathlib import Path from urllib import request from bs4 import BeautifulSoup from colorama import Style, Fore -import requests +from unidecode import unidecode from lib import settings as cfg +from lib.constants import console from lib import core cwd = os.getcwd() @@ -26,8 +28,9 @@ def __init__(self, c): # Inherited card info self.set = c['set'] - self.artist = c['artist'] + self.artist = unidecode(c['artist']) self.num = c['collector_number'] + self.set_name = c['set_name'] # Scrylink if not hasattr(self, 'scrylink'): @@ -42,45 +45,34 @@ def __init__(self, c): if not hasattr(self, 'name'): self.name = c['name'] - # Make sure card num is 3 digits - if len(self.num) == 1: self.num = f"00{self.num}" - elif len(self.num) == 2: self.num = f"0{self.num}" - - # Alternate version or promo card - self.alt = self.check_for_alternate(c) + # Possible promo card? self.promo = self.check_for_promo(c['set_type']) # Get the MTGP code - self.get_mtgp_code() + self.code = self.get_mtgp_code(self.name) # Make folders, setup path if self.path: self.make_folders() self.make_path() - def check_for_alternate(self, c): - """ - Checks if this is an alternate art card - :param c: Card info - :return: bool - """ - if 'list_order' in c: return c['list_order'] - if (c['border_color'] == "borderless" - and c['set'] not in cfg.special_sets - ): - return True - if self.num[-1] == "s": return True - return False - - def get_mtgp_code(self): + def get_mtgp_code(self, name): """ Get the correct mtgp URL code """ - try: self.code = core.get_mtgp_code(self.mtgp_set, self.name, self.alt) - except Exception: - if self.promo: - try: self.code = core.get_mtgp_code("pmo", self.name, self.alt) - except Exception: self.code = self.set+self.num - else: self.code = self.set+self.num + # Try looking for the card under its collector number + code = core.get_mtgp_code(self.mtgp_set, self.num) + if code: return code + + # Judge promo? + if self.mtgp_set == "dci": + code = core.get_mtgp_code_pmo(name, self.artist, self.set_name, "dci") + if code: return code + + # Possible promo set + if self.promo: + code = core.get_mtgp_code_pmo(name, self.artist, self.set_name) + if code: return code + return self.set+self.num def download(self, log_failed=True): """ @@ -114,15 +106,17 @@ def download_mtgp(self, name, path, mtgp_code, back=False): img_link = core.get_card_face(soup_img, back) # Try to download from MTG Pics - request.urlretrieve(img_link, f"{cfg.mtgp}/{path}.jpg") - print(f"{Fore.GREEN}MTGP:{Style.RESET_ALL} {name} [{self.set.upper()}]") + request.urlretrieve(img_link, f"{cfg.mtgp}/{path}") + console.out.append( + f"{Fore.GREEN}MTGP:{Style.RESET_ALL} {name} [{self.set.upper()}]") def download_scryfall(self, name, path, scrylink): """ Download scryfall art crop """ request.urlretrieve(scrylink, f"{cfg.scry}/{path}.jpg") - print(f"{Fore.YELLOW}SCRYFALL:{Style.RESET_ALL} {name} [{self.set.upper()}]") + console.out.append( + f"{Fore.YELLOW}SCRYFALL:{Style.RESET_ALL} {name} [{self.set.upper()}]") def make_folders(self): """ @@ -238,16 +232,7 @@ def __init__(self, c): def get_mtgp_code(self): # Override this method because flip names are different name = self.name.replace("//", "/") - try: - if self.alt: self.code = core.get_mtgp_code(self.mtgp_set, name, True) - else: self.code = core.get_mtgp_code(self.mtgp_set, name) - except: - if self.promo: - try: - if self.alt: self.code = core.get_mtgp_code("pmo", name, True) - else: self.code = core.get_mtgp_code("pmo", name) - except: self.code = self.set+self.num - else: self.code = self.set+self.num + super().get_mtgp_code(name) def make_path(self): # Override this method because // isn't valid in filenames @@ -353,16 +338,7 @@ def __init__ (self, c): def get_mtgp_code(self): # Override this method because split names are different name = self.fullname.replace("//", "/") - try: - if self.alt: self.code = core.get_mtgp_code(self.mtgp_set, name, True) - else: self.code = core.get_mtgp_code(self.mtgp_set, name) - except: - if self.promo: - try: - if self.alt: self.code = core.get_mtgp_code("pmo", name, True) - else: self.code = core.get_mtgp_code("pmo", name) - except: self.code = self.set + self.num - else: self.code = self.set + self.num + super().get_mtgp_code(name) class Meld (Card): diff --git a/lib/codes.json b/lib/codes.json index aefc186..ae126ae 100644 --- a/lib/codes.json +++ b/lib/codes.json @@ -88,5 +88,6 @@ "lea": "alp", "cm1": "cma", "j20": "dci", - "g08": "dci" + "g08": "dci", + "jgp": "dci" } \ No newline at end of file diff --git a/lib/core.py b/lib/core.py index 4a5394f..7956656 100644 --- a/lib/core.py +++ b/lib/core.py @@ -1,13 +1,15 @@ """ CORE FUNCTIONS """ +import json import os -import re -import sys +from urllib.parse import quote_plus +import requests +from difflib import SequenceMatcher from pathlib import Path from colorama import Style, Fore -import requests from bs4 import BeautifulSoup +from unidecode import unidecode from lib import settings as cfg cwd = os.getcwd() @@ -30,7 +32,7 @@ def get_command(com): return None -def get_list(com): +def get_list_from_link(com): """ Webscrape to create list of cards to download from a given list. :param com: Command array including name, and url @@ -52,24 +54,114 @@ def get_list(com): return os.path.join(cwd, f"lists/{com['name']}.txt") -def get_mtgp_code(set_code, name, alternate=False): +def get_list_from_scryfall(com): + """ + User scryfall query to return a list. + :return: Return path to the list file + """ + command = {} + query = "https://api.scryfall.com/cards/search?page=0&q=" + + # Split command by argument + com = com.split(",") + for c in com: + # Obtain key and value for argument + arg = c.split(":") + + # Get the correct separator + ops = ["!", "<", ">"] + param = "".join(["" if c in ops else c for c in arg[0]]) + if param in cfg.scry_args: + sep = cfg.scry_args[param] + else: sep = "=" + + # Add to commands + try: + if arg[0][0] == " ": arg[0] = arg[0][1:] + if arg[1][0] == " ": arg[1] = arg[1][1:] + except: pass + command.update({arg[0]+sep: arg[1]}) + if "set:" in command and "is:" not in command: + command.update({"is:": "booster"}) + + # Add each argument to scryfall search + for k, v in command.items(): + query += quote_plus(f" {k}{v}") + query += "&unique=print" + + # Query scryfall + res = requests.get(query).json() + + # Add additional pages if any exist + cards = [] + while True: + cards.extend(res['data']) + if res['has_more']: + res = requests.get(res['next_page']).json() + else: break + + # Write the list + with open(os.path.join(cwd, f"lists/scry_search.txt"), "w", encoding="utf-8") as f: + # Clear out the txt file if used before + f.truncate(0) + + # Loop through cards adding them to the txt list + for card in cards: + f.write(f"{card['set']}--{card['name']}\n") + return os.path.join(cwd, f"lists/scry_search.txt") + + +def get_mtgp_code(set_code, num): + """ + Webscrape to find the correct MTG Pics code for the card. + """ + try: + # Crawl the mtgpics site to find correct set code + r = requests.get("https://www.mtgpics.com/card?ref="+set_code+"001") + soup = BeautifulSoup(r.content, "html.parser") + soup_td = soup.find("td", {"width": "170", "align": "center"}) + replaced = soup_td.find('a')['href'].replace("set?", "set_checklist?") + mtgp_link = f"https://mtgpics.com/{replaced}" + + # Crawl the set page to find the correct link + r = requests.get(mtgp_link) + soup = BeautifulSoup(r.content, "html.parser") + rows = soup.find_all("div", {"style": "display:block;margin:0px 2px 0px 2px;border-top:1px #cccccc dotted;"}) + for row in rows: + cols = row.find_all('td') + if cols[0].text == num: + return cols[2].find("a")['href'].replace("card?ref=", "") + return None + except: return None + + +def get_mtgp_code_pmo(name, artist, set_name, promo="pmo"): """ Webscrape to find the correct MTG Pics code for the card. """ - # Crawl the mtgpics site to find correct set code - r = requests.get("https://www.mtgpics.com/card?ref="+set_code+"001") - soup = BeautifulSoup(r.content, "html.parser") - soup_td = soup.find("td", {"width": "170", "align": "center"}) - mtgp_link = f"https://mtgpics.com/{soup_td.find('a')['href']}" - - # Crawl the set page to find the correct link - r = requests.get(mtgp_link) - soup = BeautifulSoup(r.content, "html.parser") - soup_i = soup.find_all("img", attrs={"alt": re.compile(f"^({name})(.)*")}) - if isinstance(alternate, int): soup_src = soup_i[alternate]['src'] - elif alternate: soup_src = soup_i[1]['src'] - else: soup_src = soup_i[0]['src'] - return soup_src.replace("../pics/reg/","").replace("/","").replace(".jpg","") + try: + # Track matches + matches = [] + + # Which promo set? + if promo == "dci": req = "https://mtgpics.com/set_checklist?set=72" + else: req = "https://mtgpics.com/set_checklist?set=72" + + # Crawl the set page to find the correct link + r = requests.get(req) + soup = BeautifulSoup(r.content, "html.parser") + rows = soup.find_all("div", {"style": "display:block;margin:0px 2px 0px 2px;border-top:1px #cccccc dotted;"}) + for row in rows: + cols = row.find_all('td') + if artist in unidecode(cols[6].text) and name.lower() in cols[2].text.lower(): + matches.append( + { + "code": cols[2].find("a")['href'].replace("card?ref=", ""), + "match": SequenceMatcher(a=cols[2].text.replace(name, ""), b=set_name).ratio() + } + ) + return sorted(matches, key=lambda i: i['match'])[0]["code"] + except: return None def log(name, set_code=None, txt="failed"): @@ -83,14 +175,6 @@ def log(name, set_code=None, txt="failed"): print(f"{Fore.RED}FAILED: {Style.RESET_ALL}{name} [{set_code.upper()}]") -def handle(error): - """ - Handle error messages - """ - print(f"{error}\nPress enter to exit...") - sys.exit() - - def get_card_face(entries, back): """ Determine which images are back and front diff --git a/lib/scryfall.json b/lib/scryfall.json new file mode 100644 index 0000000..be8903e --- /dev/null +++ b/lib/scryfall.json @@ -0,0 +1,8 @@ +{ + "set": ":", + "rarity": ":", + "cmc": "=", + "color": "=", + "type": ":", + "order": "=" +} \ No newline at end of file diff --git a/lib/settings.py b/lib/settings.py index 3eae595..a395df0 100644 --- a/lib/settings.py +++ b/lib/settings.py @@ -13,12 +13,12 @@ CONSTANTS """ basic_lands = ["Plains", "Island", "Swamp", "Mountain", "Forest"] -with open(os.path.join(cwd, "lib/special.json")) as js: - special_sets = json.load(js) with open(os.path.join(cwd, "lib/codes.json")) as js: replace_sets = json.load(js) with open(os.path.join(cwd, "lib/links.json")) as js: links = json.load(js) +with open(os.path.join(cwd, "lib/scryfall.json")) as js: + scry_args = json.load(js) """ FILES AND FOLDERS diff --git a/lib/special.json b/lib/special.json deleted file mode 100644 index 386db06..0000000 --- a/lib/special.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "secret lair": ["sld"], - "mystical archive": ["sta"], - "judge promo": ["j20", "g08"], - "misc promo": ["pnat", "prm"] -} \ No newline at end of file diff --git a/main.py b/main.py index b3a1a2e..98879c7 100644 --- a/main.py +++ b/main.py @@ -8,11 +8,13 @@ import threading from time import perf_counter from urllib import parse -from colorama import Style, Fore import requests as req from lib import card as dl from lib import settings as cfg from lib import core +from lib.constants import console +from colorama import Style, Fore + os.system("") @@ -28,7 +30,14 @@ def start_command(self): """ Initiate download procedure based on the command. """ - self.list = core.get_list(self.command) + # Valid command received? + if ":" in self.command: + self.list = core.get_list_from_scryfall(self.command) + elif self.command: + self.command = core.get_command(self.command) + if self.command: + self.list = core.get_list_from_link(self.command) + else: self.command = None self.start() def start(self): @@ -36,6 +45,14 @@ def start(self): Open card list, for each card initiate a download """ with open(self.list, 'r', encoding="utf-8") as cards: + # Remove blank lines, print total cards + cards = cards.readlines() + try: cards.remove("") + except ValueError: pass + try: cards.remove(" ") + except ValueError: pass + print(f"{Fore.GREEN}---- Downloading {len(cards)} cards! ----{Style.RESET_ALL}") + # For each card create new thread for i, card in enumerate(cards): # Detailed card including set? @@ -88,19 +105,10 @@ def download_normal(self, card): # Remove full art entries # Add our numbered sets prepared = [] - special = {} - for kind in cfg.special_sets: - special[kind] = [] for t in r['data']: # No fullart to exclude? if not cfg.exclude_fullart or t['full_art'] is False: - # No numbered set to separate? - t['accounted'] = False - for kind in special: - if t['set'] in cfg.special_sets[kind]: - special[kind].append(t) - t['accounted'] = True - if not t['accounted']: prepared.append(t) + prepared.append(t) # Loop through prints of this card for c in prepared: @@ -110,17 +118,13 @@ def download_normal(self, card): if not cfg.download_all and result: return None - # Loop through numbered sets - self.download_special(special) - except Exception: # Try named lookup try: c = req.get(f"https://api.scryfall.com/cards/named?fuzzy={parse.quote(card)}").json() card_class = dl.get_card_class(c) card_class(c).download() - except Exception: - print(f"{card} not found!") + except Exception: console.out.append(f"{card} not found!") @staticmethod def download_detailed(item): @@ -149,9 +153,7 @@ def download_detailed(item): ).json() card_class = dl.get_card_class(c) card_class(c).download() - except Exception as e: - print(e) - print(f"{name} not found!") + except Exception: console.out.append(f"{name} not found!") @staticmethod def download_basic(card): @@ -170,45 +172,8 @@ def download_basic(card): f"&set={parse.quote(land_set)}").json() dl.Land(c).download() break - except Exception: print("Scryfall couldn't find this set. Try again!") - else: print("Error! Illegitimate set. Try again!") - - @staticmethod - def download_special(special): - """ - Download cards from sets with special requirements - :param special: {Set code : [list of cards]} - """ - for s, cards in special.items(): - # Promo sets with numbered items - if s in ('secret lair', "mystical archive"): - num = 1 - if len(cards) == 1: - # One card - for c in cards: - c['list_order'] = 0 - card_class = dl.get_card_class(c) - result = card_class(c).download() - if not cfg.download_all and result: - return None - else: - # A list of cards - for c in sorted(cards, key=lambda i: i['collector_number']): - c['list_order'] = 0 - c['name'] = f"{c['name']} {str(num)}" - card_class = dl.get_card_class(c) - result = card_class(c).download() - if not cfg.download_all and result: - return None - num += 1 - # Judge and misc promos - if s in ('judge promo', 'misc promo'): - for i, c in enumerate(cards): - c['list_order'] = i - card_class = dl.get_card_class(c) - result = card_class(c).download() - if not cfg.download_all and result: - return None + except Exception: console.out.append("Scryfall couldn't find this set. Try again!") + else: console.out.append("Error! Illegitimate set. Try again!") @staticmethod def complete(elapsed): @@ -216,10 +181,13 @@ def complete(elapsed): Tell the user the download process is complete. :param elapsed: Time to complete downloads (seconds) """ - print(f"Downloads finished in {elapsed} seconds!") - input("\nAll available files downloaded.\n" + time.sleep(1) + console.out.append(f"Downloads finished in {elapsed} seconds!") + time.sleep(.02) + console.out.append("\nAll available files downloaded.\n" "See failed.txt for images that couldn't be located.\n" "Press enter to exit :)") + input() sys.exit() @@ -238,7 +206,7 @@ def complete(elapsed): print(" ██╔══██║██╔══██╗ ██║ ██║╚██╗██║██║ ██║██║███╗██║ ") print(" ██║ ██║██║ ██║ ██║ ██║ ╚████║╚██████╔╝╚███╔███╔╝ ") print(" ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═══╝ ╚═════╝ ╚══╝╚══╝ ") - print(f"{Fore.CYAN}{Style.BRIGHT}MTG Art Downloader by Mr Teferi {__VER__}") + print(f"{Fore.CYAN}{Style.BRIGHT}MTG Art Downloader by Mr Teferi v{__VER__}") print("Additional thanks to Trix are for Scoot + Gikkman") print(f"http://mpcfill.com --- Support great MTG Proxies!{Style.RESET_ALL}\n") @@ -247,9 +215,6 @@ def complete(elapsed): "Cards in cards.txt can either be listed as 'Name' or 'SET--Name'\n" "Full Github and README available at: mprox.link/art-downloader\n") - # See if the command matches any known links - com = core.get_command(choice) - # If the command is valid, download based on that, otherwise cards.txt - if com: Download(com).start_command() - else: Download().start() \ No newline at end of file + if choice != "": print() # Add newline gap + Download(choice).start_command() diff --git a/requirements.txt b/requirements.txt index ee9c914..3d06e99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ urllib3 contextlib3 bs4 requests -pyclean \ No newline at end of file +pyclean +unidecode \ No newline at end of file