From 10aac9504c46080144a34bfcb840cc16ac5f3576 Mon Sep 17 00:00:00 2001 From: Akashdeep Dhar Date: Tue, 2 Apr 2024 12:37:52 +0530 Subject: [PATCH] Make corrections to address issues pointed out by Ruff --- mdapi/confdata/standard.py | 2 +- mdapi/database/__init__.py | 8 --- mdapi/database/base.py | 23 ++++----- mdapi/database/main.py | 102 ++++++++++++++++--------------------- mdapi/main.py | 6 +-- mdapi/services/__init__.py | 20 ++++---- mdapi/services/appviews.py | 16 +++--- tests/__init__.py | 20 +++++--- tests/test_clim.py | 23 +++++---- tests/test_data.py | 37 ++++++-------- tests/test_main.py | 60 ++++++++++++---------- tests/test_nodb.py | 42 ++++++++------- 12 files changed, 177 insertions(+), 182 deletions(-) diff --git a/mdapi/confdata/standard.py b/mdapi/confdata/standard.py index 7ac629d..9da7f64 100644 --- a/mdapi/confdata/standard.py +++ b/mdapi/confdata/standard.py @@ -26,7 +26,7 @@ """ # url to the database server: -DB_FOLDER = "/var/tmp" +DB_FOLDER = "/var/tmp" # noqa : S108 LOGGING = { "version": 1, diff --git a/mdapi/database/__init__.py b/mdapi/database/__init__.py index 08697eb..28d1af8 100644 --- a/mdapi/database/__init__.py +++ b/mdapi/database/__init__.py @@ -20,11 +20,3 @@ License and may only be used or replicated with the express permission of Red Hat, Inc. """ - - -def check_archive_validity(): - """ - This is a workaround and not a fix. - See https://github.com/fedora-infra/mdapi/issues/161 - """ - return True diff --git a/mdapi/database/base.py b/mdapi/database/base.py index b58ceee..848d941 100644 --- a/mdapi/database/base.py +++ b/mdapi/database/base.py @@ -28,7 +28,7 @@ def index_database(name, tempdtbs): - servlogr.logrobjc.info("[%s] Indexing database %s" % (name, tempdtbs)) + servlogr.logrobjc.info(f"[{name}] Indexing database {tempdtbs}") if tempdtbs.endswith("primary.sqlite"): # TODO: Try the "with sqlite3.connect(tempdtbs) as connobjc" statement here connobjc = sqlite3.connect(tempdtbs) @@ -80,16 +80,13 @@ def change_row_to_package(self, rowe): def obtain_all_rows(self, location, tableobj, cacheobj): connobjc = sqlite3.connect(location) sqlquery = queries.get(tableobj, DEFAULT_QUERY).format(table=tableobj) - for indx, rowe in enumerate(connobjc.execute(sqlquery)): + for indx, rowe in enumerate(connobjc.execute(sqlquery)): # noqa : B007 if tableobj in self.cache_dependent_tables: if rowe[0] in cacheobj: yield (cacheobj[rowe[0]], *rowe[1:]) else: - servlogr.logrobjc.debug( - "[%s] %s does not appear in the %s cache for %s" - % (self.name, rowe[0], tableobj, location) - ) - servlogr.logrobjc.debug("[%s] Dropping from comparison" % self.name) + servlogr.logrobjc.debug(f"[{self.name}] {rowe[0]} does not appear in the {tableobj} cache for {location}") # noqa : E501 + servlogr.logrobjc.debug(f"[{self.name}] Dropping from comparison") else: yield rowe connobjc.close() @@ -97,7 +94,7 @@ def obtain_all_rows(self, location, tableobj, cacheobj): def build_cache(self, location, tableobj, cacheobjc): connobjc = sqlite3.connect(location) sqlquery = queries.get(tableobj, DEFAULT_QUERY).format(table=tableobj) - for pkgId, pkgname, *args in connobjc.execute(sqlquery): + for pkgId, pkgname, *args in connobjc.execute(sqlquery): # noqa : B007 cacheobjc[pkgId] = pkgname connobjc.close() @@ -108,7 +105,7 @@ def should_compare(self, tableobj): return True def main(self): - servlogr.logrobjc.info("[%s] Comparing %s against %s" % (self.name, self.dbsA, self.dbsB)) + servlogr.logrobjc.info(f"[{self.name}] Comparing {self.dbsA} against {self.dbsB}") tablistA = list(self.obtain_table_names(self.dbsA)) tablistB = list(self.obtain_table_names(self.dbsB)) @@ -121,12 +118,12 @@ def main(self): # We have never downloaded this before... # so we have nothing to compare it against. Just return and say there # are "no differences". - servlogr.logrobjc.warning( - "[%s] Database empty - %s cannot compare" % (self.name, self.dbsB) - ) + servlogr.logrobjc.warning(f"[{self.name}] Database empty - {self.dbsB} cannot compare") return set() - assert len(tablistA) == len(tablistB), "Cannot compare disparate databases" + if len(tablistA) != len(tablistB): + raise ValueError("Cannot compare disparate databases") + # These should be the same tblelist = tablistA = tablistB diff --git a/mdapi/database/main.py b/mdapi/database/main.py index fe3a4f8..30e53ac 100644 --- a/mdapi/database/main.py +++ b/mdapi/database/main.py @@ -32,19 +32,19 @@ import time from xml.etree import ElementTree +import pyzstd import requests from fedora_messaging.api import publish from fedora_messaging.exceptions import ConnectionException, PublishReturned from mdapi_messages.messages import RepoUpdateV1 from mdapi.confdata import servlogr, standard -from mdapi.database import check_archive_validity from mdapi.database.base import compare_databases, index_database def list_branches(status="Active"): - urlx = "%s/api/collections?clt_status=%s" % (standard.PKGDB2_URL, status) - resp = requests.get(urlx, verify=standard.PKGDB2_VERIFY) + urlx = f"{standard.PKGDB2_URL}/api/collections?clt_status={status}" + resp = requests.get(urlx, verify=standard.PKGDB2_VERIFY) # noqa : S113 resp.raise_for_status() data = resp.json() servlogr.logrobjc.info("Branches metadata acquired.") @@ -52,21 +52,21 @@ def list_branches(status="Active"): def fetch_database(name, repmdurl, location): - servlogr.logrobjc.info("[%s] Downloading file %s to %s" % (name, repmdurl, location)) - resp = requests.get(repmdurl, verify=standard.DL_VERIFY) + servlogr.logrobjc.info(f"[{name}] Downloading file {repmdurl} to {location}") + resp = requests.get(repmdurl, verify=standard.DL_VERIFY) # noqa : S113 resp.raise_for_status() with open(location, "wb") as filestrm: filestrm.write(resp.content) def extract_database(name, arcvname, location): - servlogr.logrobjc.info("[%s] Extracting %s to %s" % (name, arcvname, location)) + servlogr.logrobjc.info(f"[{name}] Extracting {arcvname} to {location}") if arcvname.endswith(".xz"): with lzma.open(arcvname) as inp, open(location, "wb") as otptfile: otptfile.write(inp.read()) elif arcvname.endswith(".tar.gz"): with tarfile.open(arcvname) as tararchv: - tararchv.extractall(path=location, members=check_archive_validity()) + tararchv.extractall(path=location, filter="data") elif arcvname.endswith(".gz"): with gzip.open(arcvname, "rb") as inp, open(location, "wb") as otptfile: otptfile.write(inp.read()) @@ -74,23 +74,19 @@ def extract_database(name, arcvname, location): with bz2.open(arcvname) as inp, open(location, "wb") as otptfile: otptfile.write(inp.read()) elif arcvname.endswith(".zst"): - import pyzstd - with open(arcvname, "rb") as inp, open(location, "wb") as otptfile: pyzstd.decompress_stream(inp, otptfile) else: - servlogr.logrobjc.error("Could not extract %s" % (arcvname)) + servlogr.logrobjc.error(f"Could not extract {arcvname}") raise NotImplementedError(arcvname) def publish_changes(name, packages, repmdurl): - servlogr.logrobjc.info("[%s] Publishing differences to Fedora Messaging bus" % (name)) + servlogr.logrobjc.info(f"[{name}] Publishing differences to Fedora Messaging bus") modified = bool(packages) if not modified: - servlogr.logrobjc.warning( - "[%s] No real changes detected - Skipping publishing on Fedora Messaging bus" % (name) - ) + servlogr.logrobjc.warning(f"[{name}] No real changes detected - Skipping publishing on Fedora Messaging bus") # noqa : E501 return # Just publish the suffix of the URL. The prefix is dl.fedoraproject.org @@ -100,7 +96,7 @@ def publish_changes(name, packages, repmdurl): # talking about. urlx = "/".join(repmdurl.split("/")[4:]) - servlogr.logrobjc.info("[%s] URL %s" % (name, urlx)) + servlogr.logrobjc.info(f"[{name}] URL {urlx}") mesgobjc = RepoUpdateV1( body=dict( @@ -113,17 +109,13 @@ def publish_changes(name, packages, repmdurl): try: publish(mesgobjc) except PublishReturned as excp: - servlogr.logrobjc.error( - "Fedora Messaging broker rejected message %s - %s" % (mesgobjc.id, excp) - ) + servlogr.logrobjc.error(f"Fedora Messaging broker rejected message {mesgobjc.id} - {excp}") except ConnectionException as excp: - servlogr.logrobjc.error( - "Error occurred while sending message %s - %s" % (mesgobjc.id, excp) - ) + servlogr.logrobjc.error(f"Error occurred while sending message {mesgobjc.id} - {excp}") def install_database(name, srce, dest): - servlogr.logrobjc.info("[%s] Installing %s to %s" % (name, srce, dest)) + servlogr.logrobjc.info(f"[{name}] Installing {srce} to {dest}") shutil.move(srce, dest) @@ -155,10 +147,10 @@ def process_repo(repo): Retrieve the repo metadata at the given URL and store them using the provided name. """ urlx, name = repo - repmdurl = "%s/repomd.xml" % urlx - response = requests.get(repmdurl, verify=standard.DL_VERIFY) + repmdurl = f"{urlx}/repomd.xml" + response = requests.get(repmdurl, verify=standard.DL_VERIFY) # noqa : S113 if not response: - servlogr.logrobjc.error("[%s] Failed to obtain %s - %s" % (name, repmdurl, response)) + servlogr.logrobjc.error(f"[{name}] Failed to obtain {repmdurl} - {response}") return # Parse the XML document and get a list of locations and their SHAsum @@ -167,7 +159,7 @@ def process_repo(repo): node.find("repo:location", standard.repomd_xml_namespace), node.find("repo:open-checksum", standard.repomd_xml_namespace), ) - for node in ElementTree.fromstring(response.text) + for node in ElementTree.fromstring(response.text) # noqa : S314 ) # Extract out the attributes that we are really interested in @@ -189,25 +181,25 @@ def process_repo(repo): cacA, cacB = {}, {} if not filelist: - servlogr.logrobjc.warning("No SQLite database could be found in %s" % (urlx)) + servlogr.logrobjc.warning(f"No SQLite database could be found in {urlx}") for filename, hashdata, hashtype in filelist: - repmdurl = "%s/%s" % (urlx, filename) + repmdurl = f"{urlx}/{filename}" # First, determine if the file has changed by comparing hash database = None if "primary.sqlite" in filename: - database = "mdapi-%s-primary.sqlite" % name + database = f"mdapi-{name}-primary.sqlite" elif "filelists.sqlite" in filename: - database = "mdapi-%s-filelists.sqlite" % name + database = f"mdapi-{name}-filelists.sqlite" elif "other.sqlite" in filename: - database = "mdapi-%s-other.sqlite" % name + database = f"mdapi-{name}-other.sqlite" # Have we downloaded this before? # Did it change? destfile = os.path.join(standard.DB_FOLDER, database) if not needs_update(destfile, hashdata, hashtype): - servlogr.logrobjc.info("[%s] No change detected from %s" % (name, repmdurl)) + servlogr.logrobjc.info(f"[{name}] No change detected from {repmdurl}") continue # Creating temporary directories with formatted names to remove them later easily, if needed @@ -224,9 +216,7 @@ def process_repo(repo): packages = compare_databases(name, tempdtbs, destfile, cacA, cacB).main() publish_changes(name, packages, repmdurl) else: - servlogr.logrobjc.warning( - "[%s] Not publishing to Fedora Messaging bus - Not comparing databases" % (name) - ) + servlogr.logrobjc.warning(f"[{name}] Not publishing to Fedora Messaging bus - Not comparing databases") # noqa : E501 install_database(name, tempdtbs, destfile) @@ -241,32 +231,30 @@ def index_repositories(): versdata = rels["version"] if versdata == "devel": versdata = "rawhide" - urlx = "%s/pub/fedora/linux/development/%s/Everything/x86_64/os/repodata" % ( - standard.DL_SERVER, - versdata, - ) - servlogr.logrobjc.info( - "Acquired repo for %s/%s of '%s' branch at %s" - % (rels["koji_name"], versdata, rels["status"], urlx) - ) + urlx = f"{standard.DL_SERVER}/pub/fedora/linux/development/{versdata}/Everything/x86_64/os/repodata" # noqa : E501 + servlogr.logrobjc.info(f"Acquired repo for {rels['koji_name']}/{versdata} of '{rels['status']}' branch at {urlx}") # noqa : E501 repolist.append((urlx, rels["koji_name"])) urlx = urlx.replace("/x86_64/os/", "/source/tree/") - repolist.append((urlx, "src_%s" % rels["koji_name"])) + repolist.append((urlx, f"src_{rels['koji_name']}")) urls = { "Fedora Linux": [ - "%s/pub/fedora/linux/releases/%s/Everything/x86_64/os/repodata", - "%s/pub/fedora/linux/updates/%s/Everything/x86_64/repodata", - "%s/pub/fedora/linux/updates/testing/%s/Everything/x86_64/repodata", + "{dlserver}/pub/fedora/linux/releases/{versname}/Everything/x86_64/os/repodata", + "{dlserver}/pub/fedora/linux/updates/{versname}/Everything/x86_64/repodata", + "{dlserver}/pub/fedora/linux/updates/testing/{versname}/Everything/x86_64/repodata", ], "Fedora EPEL": [ - "%s/pub/epel/%s/x86_64/repodata/", - "%s/pub/epel/testing/%s/x86_64/repodata", + "{dlserver}/pub/epel/{versname}/x86_64/repodata/", + "{dlserver}/pub/epel/testing/{versname}/x86_64/repodata", ], } urls["Fedora"] = urls["Fedora Linux"] - repodict = {"fedora": ["%s", "%s-updates", "%s-updates-testing"], "epel": ["%s", "%s-testing"]} + + repodict = { + "fedora": ["{rlid}", "{rlid}-updates", "{rlid}-updates-testing"], + "epel": ["{rlid}", "{rlid}-testing"] + } # Obtain the stable repos stable_releases = list_branches(status="Active") @@ -276,22 +264,22 @@ def index_repositories(): versdata = rels["version"] for jndx, urli in enumerate(urls[rels["name"]]): if rels["name"] in ("Fedora Linux", "Fedora"): - name = repodict["fedora"][jndx] % rels["koji_name"] + name = repodict["fedora"][jndx].format(ridx = rels["koji_name"]) elif rels["name"] == "Fedora EPEL" and versdata == "8": - name = repodict["epel"][jndx] % rels["koji_name"] + name = repodict["epel"][jndx].format(rlid = rels["koji_name"]) urli = urli.replace("/x86_64/", "/Everything/x86_64/") elif rels["name"] == "Fedora EPEL" and versdata == "9": - name = repodict["epel"][jndx] % rels["koji_name"] + name = repodict["epel"][jndx].format(rlid = rels["koji_name"]) urli = urli.replace("/x86_64/", "/Everything/x86_64/") else: - name = repodict["epel"][jndx] % rels["koji_name"] - rurl = urli % (standard.DL_SERVER, versdata) + name = repodict["epel"][jndx].format(rlid = rels["koji_name"]) + rurl = urli.format(dlserver = standard.DL_SERVER, versname = versdata) repolist.append((rurl, name)) rurl = rurl.replace("/x86_64/os", "/source/tree") - repolist.append((rurl, "src_%s" % name)) + repolist.append((rurl, f"src_{name}")) # Finish with the koji repo - repolist.append(("%s/rawhide/latest/x86_64/repodata" % standard.KOJI_REPO, "koji")) + repolist.append((f"{standard.KOJI_REPO}/rawhide/latest/x86_64/repodata", "koji")) # In serial for repo in repolist: diff --git a/mdapi/main.py b/mdapi/main.py index a3472d3..93ff272 100644 --- a/mdapi/main.py +++ b/mdapi/main.py @@ -47,8 +47,8 @@ def main(conffile=None): if conffile is not None: # Load the configuration file to use CONFIG = {} - with open(conffile, "r") as confobjc: - exec(compile(confobjc.read(), conffile, "exec"), CONFIG) + with open(conffile) as confobjc: + exec(compile(confobjc.read(), conffile, "exec"), CONFIG) # noqa : S102 compile_configuration(CONFIG) @@ -70,7 +70,7 @@ def serveapp(): APPSERVE["logging"]["level"], ) ) - subprocess.run(startcmd.split()) + subprocess.run(startcmd.split()) # noqa : S603 except KeyError: print("Invalid configuration detected") return 1 diff --git a/mdapi/services/__init__.py b/mdapi/services/__init__.py index 8ce6f7f..12b2ad4 100644 --- a/mdapi/services/__init__.py +++ b/mdapi/services/__init__.py @@ -53,9 +53,9 @@ async def _get_package(brch, name=None, actn=None, srcn=None): for repotype in ["updates-testing", "updates", "testing", None]: if repotype: - dtbsfile = "%s/mdapi-%s-%s-primary.sqlite" % (standard.DB_FOLDER, brch, repotype) + dtbsfile = f"{standard.DB_FOLDER}/mdapi-{brch}-{repotype}-primary.sqlite" else: - dtbsfile = "%s/mdapi-%s-primary.sqlite" % (standard.DB_FOLDER, brch) + dtbsfile = f"{standard.DB_FOLDER}/mdapi-{brch}-primary.sqlite" if not os.path.exists(dtbsfile): wrongdbs = True @@ -74,7 +74,7 @@ async def _get_package(brch, name=None, actn=None, srcn=None): pckg = [Packages(*item) for item in pkgc] break elif srcn: - async with dtbsobjc.execute(GET_PACKAGE_BY_SRC, ("%s-%%" % srcn,)) as dbcursor: + async with dtbsobjc.execute(GET_PACKAGE_BY_SRC, (f"{srcn}-%",)) as dbcursor: pkgc = await dbcursor.fetchall() if pkgc: for pkgx in pkgc: @@ -87,7 +87,7 @@ async def _get_package(brch, name=None, actn=None, srcn=None): break srcn = re.escape(srcn) - ptrn = re.compile("%s-[0-9]" % srcn) + ptrn = re.compile(f"{srcn}-[0-9]") for pkgx in pkgc: if ptrn.match(pkgx[3]): pckg = Packages(*pkgx) @@ -124,9 +124,9 @@ async def _expand_package_info(pkgs, brch, repotype): for pkgx in pkgs: otpt = pkgx.to_json() if repotype: - dtbsfile = "%s/mdapi-%s-%s-primary.sqlite" % (standard.DB_FOLDER, brch, repotype) + dtbsfile = f"{standard.DB_FOLDER}/mdapi-{brch}-{repotype}-primary.sqlite" else: - dtbsfile = "%s/mdapi-%s-primary.sqlite" % (standard.DB_FOLDER, brch) + dtbsfile = f"{standard.DB_FOLDER}/mdapi-{brch}-primary.sqlite" async with aiosqlite.connect(dtbsfile) as dtbsobjc: """ @@ -177,9 +177,9 @@ async def _get_files(pkid, brch, repotype): Return the list of files for the given package in the specified branch. """ if repotype: - dtbsfile = "%s/mdapi-%s-%s-filelists.sqlite" % (standard.DB_FOLDER, brch, repotype) + dtbsfile = f"{standard.DB_FOLDER}/mdapi-{brch}-{repotype}-filelists.sqlite" else: - dtbsfile = "%s/mdapi-%s-filelists.sqlite" % (standard.DB_FOLDER, brch) + dtbsfile = f"{standard.DB_FOLDER}/mdapi-{brch}-filelists.sqlite" if not os.path.exists(dtbsfile): raise HTTPBadRequest() @@ -203,9 +203,9 @@ async def _get_changelog(pkid, brch, repotype): Return the changelog for the given packages in the specified branch. """ if repotype: - dtbsfile = "%s/mdapi-%s-%s-other.sqlite" % (standard.DB_FOLDER, brch, repotype) + dtbsfile = f"{standard.DB_FOLDER}/mdapi-{brch}-{repotype}-other.sqlite" else: - dtbsfile = "%s/mdapi-%s-other.sqlite" % (standard.DB_FOLDER, brch) + dtbsfile = f"{standard.DB_FOLDER}/mdapi-{brch}-other.sqlite" if not os.path.exists(dtbsfile): raise HTTPBadRequest() diff --git a/mdapi/services/appviews.py b/mdapi/services/appviews.py index a2f4ad5..6322708 100644 --- a/mdapi/services/appviews.py +++ b/mdapi/services/appviews.py @@ -32,12 +32,12 @@ async def index(rqst): - servlogr.logrobjc.info("index %s" % rqst) + servlogr.logrobjc.info(f"index {rqst}") return FileResponse(homepage) async def get_pkg(rqst): - servlogr.logrobjc.info("get_pkg %s" % rqst) + servlogr.logrobjc.info(f"get_pkg {rqst}") brch = rqst.match_info.get("brch") name = rqst.match_info.get("name") pckg, repotype = await _get_package(brch, name) @@ -46,7 +46,7 @@ async def get_pkg(rqst): async def get_src_pkg(rqst): - servlogr.logrobjc.info("get_src_pkg %s" % rqst) + servlogr.logrobjc.info(f"get_src_pkg {rqst}") brch = rqst.match_info.get("brch") name = rqst.match_info.get("name") pckg, repotype = await _get_package(brch, srcn=name) @@ -58,7 +58,7 @@ async def list_branches(rqst): """ Return the list of all branches currently supported by mdapi """ - servlogr.logrobjc.info("list_branches %s" % rqst) + servlogr.logrobjc.info(f"list_branches {rqst}") rslt = sorted( { # Remove the front part `mdapi-` and the end part `-.sqlite` from the filenames @@ -75,14 +75,14 @@ async def _process_dep(rqst, actn): Return the information about the packages having the specified action (as in provides, requires, obsoletes etc.) """ - servlogr.logrobjc.info("process_dep %s %s" % (actn, rqst)) + servlogr.logrobjc.info(f"process_dep {actn} {rqst}") brch = rqst.match_info.get("brch") name = rqst.match_info.get("name") try: pckg, repotype = await _get_package(brch, name, actn=actn) except: # noqa - raise HTTPBadRequest + raise HTTPBadRequest # noqa : B904 rslt = await _expand_package_info(pckg, brch, repotype) return json_response(rslt) @@ -121,7 +121,7 @@ async def get_supplements(rqst): async def get_pkg_files(rqst): - servlogr.logrobjc.info("get_pkg_files %s" % rqst) + servlogr.logrobjc.info(f"get_pkg_files {rqst}") brch = rqst.match_info.get("brch") name = rqst.match_info.get("name") pckg, repotype = await _get_package(brch, name) @@ -130,7 +130,7 @@ async def get_pkg_files(rqst): async def get_pkg_changelog(rqst): - servlogr.logrobjc.info("get_pkg_changelog %s" % rqst) + servlogr.logrobjc.info(f"get_pkg_changelog {rqst}") brch = rqst.match_info.get("brch") name = rqst.match_info.get("name") pckg, repotype = await _get_package(brch, name) diff --git a/tests/__init__.py b/tests/__init__.py index 729cdbc..b56eb26 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -22,20 +22,24 @@ """ import os.path +from tempfile import TemporaryDirectory import requests from bs4 import BeautifulSoup +from mdapi.confdata import standard + """ Standard set of databases to run tests against """ -LOCATION = "/var/tmp/mdapi-tests/" +tempargs = dict(prefix="mdapi-tests-", dir=standard.DB_FOLDER) +LOCATION = f"{TemporaryDirectory(**tempargs).name}/" + BRCHNAME = "rawhide" PROBEURL = { - "koji": "https://kojipkgs.fedoraproject.org/repos/%s/latest/x86_64/repodata/" % BRCHNAME, - "rawhide": "https://dl.fedoraproject.org/pub/fedora/linux/development/%s/Everything/x86_64/os/repodata/" # noqa - % BRCHNAME, + "koji": f"https://kojipkgs.fedoraproject.org/repos/{BRCHNAME}/latest/x86_64/repodata/", + "rawhide": f"https://dl.fedoraproject.org/pub/fedora/linux/development/{BRCHNAME}/Everything/x86_64/os/repodata/", } KEYWORDS = [ "-filelists.sqlite", @@ -50,23 +54,23 @@ def databases_presence(brchname): Return true if the databases are present of the given branch, else false """ for indx in KEYWORDS: - if not os.path.exists("%smdapi-%s%s" % (LOCATION, brchname, indx)): + if not os.path.exists(f"{LOCATION}mdapi-{brchname}{indx}"): return False return True def populate_permalinks(): """ - Get a list of permalinks oft the SQLite database archives from the specified branches + Get a list of permalinks off the SQLite database archives from the specified branches """ linkdict = {} for indx in PROBEURL.keys(): linklist = [] - htmlcont = requests.get(PROBEURL[indx]).text + htmlcont = requests.get(PROBEURL[indx]).text # noqa : S113 soupobjc = BeautifulSoup(htmlcont, "html.parser") for jndx in soupobjc.find_all("a"): for kndx in KEYWORDS: if kndx in jndx.get("href"): - linklist.append("%s%s" % (PROBEURL[indx], jndx.get("href"))) + linklist.append(f"{PROBEURL[indx]}{jndx.get('href')}") linkdict[indx] = linklist return linkdict diff --git a/tests/test_clim.py b/tests/test_clim.py index da29825..5870866 100644 --- a/tests/test_clim.py +++ b/tests/test_clim.py @@ -23,39 +23,42 @@ import os.path +import pytest from click.testing import CliRunner from mdapi import __version__ from mdapi.main import main +@pytest.mark.download_needless def test_cli_application_help_option(): rnnrobjc = CliRunner() rsltobjc = rnnrobjc.invoke(main, ["--help"]) - assert "A simple API for serving the metadata from the RPM repositories" in rsltobjc.output - assert rsltobjc.exit_code == 0 + assert "A simple API for serving the metadata from the RPM repositories" in rsltobjc.output # noqa : S101 + assert rsltobjc.exit_code == 0 # noqa : S101 +@pytest.mark.download_needless def test_cli_application_version_option(): rnnrobjc = CliRunner() rsltobjc = rnnrobjc.invoke(main, ["--version"]) - assert rsltobjc.output == "mdapi, version %s\n" % __version__ - assert rsltobjc.exit_code == 0 + assert rsltobjc.output == "mdapi, version %s\n" % __version__ # noqa : S101 + assert rsltobjc.exit_code == 0 # noqa : S101 +@pytest.mark.download_needless def test_cli_application_with_wrong_configpath_and_no_command(): rnnrobjc = CliRunner() confpath = "/etc/mdapi/myconfig.py" rsltobjc = rnnrobjc.invoke(main, ["--conffile", confpath]) - assert ( - "Error: Invalid value for '-c' / '--conffile': Path '%s' does not exist." % confpath - ) in rsltobjc.output - assert rsltobjc.exit_code == 2 + assert "Error: Invalid value for '-c' / '--conffile': Path '%s' does not exist." % confpath in rsltobjc.output # noqa : S101 + assert rsltobjc.exit_code == 2 # noqa : S101 +@pytest.mark.download_needless def test_cli_application_with_right_configpath_but_no_command(): rnnrobjc = CliRunner() confpath = os.path.dirname(__file__).replace("tests", "mdapi/confdata/standard.py") rsltobjc = rnnrobjc.invoke(main, ["--conffile", confpath]) - assert "Error: Missing command." in rsltobjc.output - assert rsltobjc.exit_code == 2 + assert "Error: Missing command." in rsltobjc.output # noqa : S101 + assert rsltobjc.exit_code == 2 # noqa : S101 diff --git a/tests/test_data.py b/tests/test_data.py index 05f6886..2ad1954 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -27,10 +27,10 @@ import pytest from requests.exceptions import HTTPError +import tests from mdapi.confdata import servlogr from mdapi.database import main from mdapi.database.main import extract_database, fetch_database -from tests import LOCATION, PROBEURL, databases_presence, populate_permalinks @pytest.fixture @@ -39,38 +39,33 @@ def setup_environment(): Collect the SQLite databases from the mirror to have some data to test against """ - if databases_presence("rawhide") and databases_presence("koji"): + if tests.databases_presence("rawhide") and tests.databases_presence("koji"): pass else: - if not os.path.exists(LOCATION): - os.mkdir(LOCATION) - linkdict = populate_permalinks() + if not os.path.exists(tests.LOCATION): + os.mkdir(tests.LOCATION) + linkdict = tests.populate_permalinks() for indx in linkdict.keys(): for filelink in linkdict[indx]: - arcvloca = "%s%s" % (LOCATION, PurePath(filelink).name) - fileloca = "%s%s" % ( - LOCATION, - PurePath(filelink).name.replace( - PurePath(filelink).name.split("-")[0], "mdapi-%s" % indx - ), - ) - fileloca = fileloca.replace(".%s" % fileloca.split(".")[-1], "") + arcvloca = f"{tests.LOCATION}{PurePath(filelink).name}" + fileloca = f"{tests.LOCATION}{PurePath(filelink).name.replace(PurePath(filelink).name.split('-')[0], f'mdapi-{indx}')}" # noqa: E501 + fileloca = fileloca.replace(f".{fileloca.split('.')[-1]}", "") try: fetch_database(indx, filelink, arcvloca) extract_database(indx, arcvloca, fileloca) os.remove(arcvloca) except HTTPError as excp: - servlogr.logrobjc.warning( - "[%s] Archive could not be fetched : %s" % (indx, excp) - ) + servlogr.logrobjc.warning(f"[{indx}] Archive could not be fetched : {excp}") +@pytest.mark.download_required def test_fetch_and_extract_database(setup_environment): - for indx in PROBEURL.keys(): - assert databases_presence(indx) - assert len(os.listdir(LOCATION)) == 6 + for indx in tests.PROBEURL.keys(): + assert tests.databases_presence(indx) # noqa : S101 + assert len(os.listdir(tests.LOCATION)) == 6 # noqa : S101 +@pytest.mark.download_required @pytest.mark.parametrize( "dvlpstat", [ @@ -80,5 +75,5 @@ def test_fetch_and_extract_database(setup_environment): ) def test_list_branches(dvlpstat): rsltobjc = main.list_branches(dvlpstat) - assert isinstance(rsltobjc, list) - assert len(rsltobjc) > 0 + assert isinstance(rsltobjc, list) # noqa : S101 + assert len(rsltobjc) > 0 # noqa : S101 diff --git a/tests/test_main.py b/tests/test_main.py index e358f14..febef50 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -25,89 +25,97 @@ import pytest +import tests from mdapi.confdata import standard from mdapi.services.main import buildapp -from tests import LOCATION, PROBEURL, databases_presence @pytest.fixture @pytest.mark.usefixtures("setup_environment") async def testing_application(setup_environment, event_loop, aiohttp_client): - standard.DB_FOLDER = LOCATION + standard.DB_FOLDER = tests.LOCATION applobjc = await buildapp() return await aiohttp_client(applobjc) +@pytest.mark.download_required async def test_view_index_page(testing_application): respobjc = await testing_application.get("/") - assert respobjc.status == 200 + assert respobjc.status == 200 # noqa : S101 botmtext = "2015-2022 - Red Hat, Inc. - GPLv3+ - Sources:" otptrslt = await respobjc.text() - assert botmtext in otptrslt + assert botmtext in otptrslt # noqa : S101 +@pytest.mark.download_required async def test_view_branches(testing_application): - if False in [databases_presence(indx) for indx in PROBEURL.keys()]: + if False in [tests.databases_presence(indx) for indx in tests.PROBEURL.keys()]: pytest.xfail(reason="Databases are not available locally") else: respobjc = await testing_application.get("/branches") - assert respobjc.status == 200 + assert respobjc.status == 200 # noqa : S101 otptobjc = await respobjc.text() - assert "rawhide" in otptobjc - assert "koji" in otptobjc + assert "rawhide" in otptobjc # noqa : S101 + assert "koji" in otptobjc # noqa : S101 +@pytest.mark.download_required async def test_view_pkg_rawhide(testing_application): - if not databases_presence("rawhide"): + if not tests.databases_presence("rawhide"): pytest.xfail(reason="Databases for 'rawhide' repositories are not available locally") else: respobjc = await testing_application.get("/rawhide/pkg/kernel") - assert respobjc.status == 200 + assert respobjc.status == 200 # noqa : S101 json.loads(await respobjc.text()) +@pytest.mark.download_required async def test_view_pkg_rawhide_invalid(testing_application): - if not databases_presence("rawhide"): + if not tests.databases_presence("rawhide"): pytest.xfail(reason="Databases for 'rawhide' repositories are not available locally") else: respobjc = await testing_application.get("/rawhide/pkg/invalidpackagename") - assert respobjc.status == 404 - assert "404: Not Found" == await respobjc.text() + assert respobjc.status == 404 # noqa : S101 + assert "404: Not Found" == await respobjc.text() # noqa : S101 +@pytest.mark.download_required async def test_view_pkg_srcpkg_rawhide(testing_application): - if not databases_presence("rawhide"): + if not tests.databases_presence("rawhide"): pytest.xfail(reason="Databases for 'rawhide' repositories are not available locally") else: respobjc = await testing_application.get("/rawhide/srcpkg/python-natsort") - assert respobjc.status == 200 - json.loads(await respobjc.text()) + assert respobjc.status == 200 # noqa : S101 + json.loads(await respobjc.text()) # noqa : S101 +@pytest.mark.download_required async def test_view_pkg_srcpkg_rawhide_subpackage_version(testing_application): - if not databases_presence("rawhide"): + if not tests.databases_presence("rawhide"): pytest.xfail(reason="Databases for 'rawhide' repositories are not available locally") else: respobjc = await testing_application.get("/rawhide/pkg/ruby") - assert respobjc.status == 200 + assert respobjc.status == 200 # noqa : S101 pkgversion = json.loads(await respobjc.text())["version"] respobjc = await testing_application.get("/rawhide/srcpkg/ruby") - assert respobjc.status == 200 + assert respobjc.status == 200 # noqa : S101 srcversion = json.loads(await respobjc.text())["version"] - assert pkgversion == srcversion + assert pkgversion == srcversion # noqa : S101 +@pytest.mark.download_required async def test_view_changelog_rawhide(testing_application): - if not databases_presence("rawhide"): + if not tests.databases_presence("rawhide"): pytest.xfail(reason="Databases for 'rawhide' repositories are not available locally") else: respobjc = await testing_application.get("/rawhide/changelog/kernel") - assert respobjc.status == 200 + assert respobjc.status == 200 # noqa : S101 json.loads(await respobjc.text()) +@pytest.mark.download_required @pytest.mark.parametrize( "action, package, status_code", [ @@ -123,10 +131,10 @@ async def test_view_changelog_rawhide(testing_application): ], ) async def test_view_property_koji(testing_application, action, package, status_code): - if not databases_presence("koji"): + if not tests.databases_presence("koji"): pytest.xfail(reason="Databases for 'koji' repositories are not available locally") else: - respobjc = await testing_application.get("/koji/%s/%s" % (action, package)) - assert respobjc.status == status_code + respobjc = await testing_application.get(f"/koji/{action}/{package}") + assert respobjc.status == status_code # noqa : S101 if status_code == 200: - json.loads(await respobjc.text()) + json.loads(await respobjc.text()) # noqa : S101 diff --git a/tests/test_nodb.py b/tests/test_nodb.py index cd58b5f..e01f2c6 100644 --- a/tests/test_nodb.py +++ b/tests/test_nodb.py @@ -31,53 +31,61 @@ async def testing_application_nodb(event_loop, aiohttp_client): standard.DB_FOLDER = "." applobjc = await buildapp() - return await aiohttp_client(applobjc) + return await aiohttp_client(applobjc) # noqa : S101 +@pytest.mark.download_needless async def test_view_index_page(testing_application_nodb): respobjc = await testing_application_nodb.get("/") - assert respobjc.status == 200 + assert respobjc.status == 200 # noqa : S101 botmtext = "2015-2022 - Red Hat, Inc. - GPLv3+ - Sources:" otptrslt = await respobjc.text() - assert botmtext in otptrslt + assert botmtext in otptrslt # noqa : S101 +@pytest.mark.download_needless async def test_view_branches(testing_application_nodb): respobjc = await testing_application_nodb.get("/branches") - assert respobjc.status == 200 - assert "[]" == await respobjc.text() + assert respobjc.status == 200 # noqa : S101 + assert "[]" == await respobjc.text() # noqa : S101 +@pytest.mark.download_needless async def test_view_pkg_rawhide(testing_application_nodb): respobjc = await testing_application_nodb.get("/rawhide/pkg/kernel") - assert respobjc.status == 400 - assert "400: Bad Request" == await respobjc.text() + assert respobjc.status == 400 # noqa : S101 + assert "400: Bad Request" == await respobjc.text() # noqa : S101 +@pytest.mark.download_needless async def test_view_pkg_rawhide_invalid(testing_application_nodb): respobjc = await testing_application_nodb.get("/rawhide/pkg/invalidpackagename") - assert respobjc.status == 400 - assert "400: Bad Request" == await respobjc.text() + assert respobjc.status == 400 # noqa : S101 + assert "400: Bad Request" == await respobjc.text() # noqa : S101 +@pytest.mark.download_needless async def test_view_srcpkg_rawhide(testing_application_nodb): respobjc = await testing_application_nodb.get("/rawhide/srcpkg/python-natsort") - assert respobjc.status == 400 - assert "400: Bad Request" == await respobjc.text() + assert respobjc.status == 400 # noqa : S101 + assert "400: Bad Request" == await respobjc.text() # noqa : S101 +@pytest.mark.download_needless async def test_view_filelist_rawhide(testing_application_nodb): respobjc = await testing_application_nodb.get("/rawhide/files/kernel-core") - assert respobjc.status == 400 - assert "400: Bad Request" == await respobjc.text() + assert respobjc.status == 400 # noqa : S101 + assert "400: Bad Request" == await respobjc.text() # noqa : S101 +@pytest.mark.download_needless async def test_view_changelog_rawhide(testing_application_nodb): respobjc = await testing_application_nodb.get("/rawhide/changelog/kernel") - assert respobjc.status == 400 - assert "400: Bad Request" == await respobjc.text() + assert respobjc.status == 400 # noqa : S101 + assert "400: Bad Request" == await respobjc.text() # noqa : S101 +@pytest.mark.download_needless @pytest.mark.parametrize( "action", [ @@ -93,5 +101,5 @@ async def test_view_changelog_rawhide(testing_application_nodb): ) async def test_view_property_koji(testing_application_nodb, action): respobjc = await testing_application_nodb.get("/koji/%s/R" % (action)) - assert respobjc.status == 400 - assert "400: Bad Request" == await respobjc.text() + assert respobjc.status == 400 # noqa : S101 + assert "400: Bad Request" == await respobjc.text() # noqa : S101