Provide information whether a project is packaged as an RPM in distributions

[fridex]

Is your feature request related to a problem? Please describe.

As a Thoth user, I would like to know whether a package that I am using is trusted by RPM packagers (e.g. by Fedora community). In such a case, I can be more sure about the package.

Describe the solution you'd like

For each distribution, we can obtain a list of Python packages packaged as RPM using dnf repoquery --whatrequires "python(abi)"
For each RPM package, we can obtain information about Python package name on PyPI using dnf repoquery --provides <rpm-package>, PyPI package name can be seen in python3dist(<pypi-projectname>)

Acceptance Criteria

• [ ]

Thanks to torsava for pointers.

[fridex]

The following code lists all the Python rpm modules and shows their PyPI name (if any):

for pkg in `dnf repoquery --whatrequires "python(abi)" 2>/dev/null`; do
        pypi_name=`dnf repoquery --provides $pkg 2>/dev/null | grep 'python3.*dist(' | grep  -o '(\w*)' | tr -d '()'`
        echo "$pkg $pypi_name"
done

[fridex]

Used this code to aggregate the data:

import os
from packaging.utils import canonicalize_name


def parse_nvra(nvra):
    """Split N-V-R.A[.rpm] into a dictionary.

    @param nvra: N-V-R:E.A[.rpm], E:N-V-R.A[.rpm], N-V-R.A[.rpm]:E or N-E:V-R.A[.rpm] string
    @type nvra: str
    @return: {name, version, release, epoch, arch}
    @rtype: dict
    """
    if "/" in nvra:
        nvra = nvra.split("/")[-1]

    epoch = ""
    for i in range(2):
        # run this twice to parse N-V-R.A.rpm:E and N-V-R.A:E.rpm
        if nvra.endswith(".rpm"):
            # strip .rpm suffix
            nvra = nvra[:-4]
        else:
            # split epoch (if exists)
            nvra, epoch = split_nvr_epoch(nvra)

    nvra_parts = nvra.rsplit(".", 1)
    if len(nvra_parts) != 2:
        raise ValueError("Invalid NVRA: %s" % nvra)

    nvr, arch = nvra_parts
    if "-" in arch:
        raise ValueError("Invalid arch '%s' in '%s'" % (arch, nvra))

    result = parse_nvr(nvr)
    result["arch"] = arch
    result["src"] = arch == "src"
    if epoch != "":
        result["epoch"] = epoch
    return result


def parse_nvr(nvre):
    """Split N-V-R into a dictionary.

    @param nvre: N-V-R:E, E:N-V-R or N-E:V-R string
    @type nvre: str
    @return: {name, version, release, epoch}
    @rtype: dict
    """
    if "/" in nvre:
        nvre = nvre.split("/")[-1]

    nvr, epoch = split_nvr_epoch(nvre)

    nvr_parts = nvr.rsplit("-", 2)
    if len(nvr_parts) != 3:
        raise ValueError("Invalid NVR: %s" % nvr)

    # parse E:V
    if epoch == "" and ":" in nvr_parts[1]:
        epoch, nvr_parts[1] = nvr_parts[1].split(":", 1)

    # check if epoch is empty or numeric
    if epoch != "":
        try:
            int(epoch)
        except ValueError:
            raise ValueError("Invalid epoch '%s' in '%s'" % (epoch, nvr))

    result = dict(zip(["name", "version", "release"], nvr_parts))
    result["epoch"] = epoch
    return result


def split_nvr_epoch(nvre):
    """Split nvre to N-V-R and E.

    @param nvre: E:N-V-R or N-V-R:E string
    @type nvre: str
    @return: (N-V-R, E)
    @rtype: (str, str)
    """
    if ":" in nvre:
        if nvre.count(":") != 1:
            raise ValueError("Invalid NVRE: %s" % nvre)

        nvr, epoch = nvre.rsplit(":", 1)
        if "-" in epoch:
            if "-" not in nvr:
                # switch nvr with epoch
                nvr, epoch = epoch, nvr
            else:
                # it's probably N-E:V-R format, handle it after the split
                nvr, epoch = nvre, ""
    else:
        nvr, epoch = nvre, ""

    return (nvr, epoch)


with open("f34_packages.txt") as f:
    content = f.read()


for line in content.splitlines():
    parts = line.split(" ", maxsplit=2)

    if len(parts) != 2:
        # Skip packages reported without their PyPI counterpart.
        continue

    rpm_name, pypi_name = parts

    rpm_name = parse_nvra(rpm_name)["name"]
    pypi_name = canonicalize_name(pypi_name)

    os.makedirs(f"prescriptions/{pypi_name}", exist_ok=True)

    data = f"""\
units:
  wraps:
    - name: {pypi_name.capitalize()}Fedora34RPMWrap
      type: wrap
      should_include:
        adviser_pipeline: true
      match:
        state:
          resolved_dependencies:
            - name: {pypi_name}
              index_url: 'https://pypi.org/simple'
      run:
        stack_info:
          - type: INFO
            message: "Package {pypi_name!r} is packaged as an RPM in Fedora 34 as {rpm_name!r}"
            link: https://src.fedoraproject.org/rpms/{rpm_name}
"""
    print(data)

    with open(f"prescriptions/{pypi_name}/f34_rpm.yaml", "w") as f:
        f.write(data)

Related PR: thoth-station/prescriptions#44

Closing this, we might revisit this approach following thoth-station/thoth-application#1798

/close

[sesheta]

@fridex: Closing this issue.

Details

In response to this:

Used this code to aggregate the data:

import os
from packaging.utils import canonicalize_name


def parse_nvra(nvra):
   """Split N-V-R.A[.rpm] into a dictionary.

   @param nvra: N-V-R:E.A[.rpm], E:N-V-R.A[.rpm], N-V-R.A[.rpm]:E or N-E:V-R.A[.rpm] string
   @type nvra: str
   @return: {name, version, release, epoch, arch}
   @rtype: dict
   """
   if "/" in nvra:
       nvra = nvra.split("/")[-1]

   epoch = ""
   for i in range(2):
       # run this twice to parse N-V-R.A.rpm:E and N-V-R.A:E.rpm
       if nvra.endswith(".rpm"):
           # strip .rpm suffix
           nvra = nvra[:-4]
       else:
           # split epoch (if exists)
           nvra, epoch = split_nvr_epoch(nvra)

   nvra_parts = nvra.rsplit(".", 1)
   if len(nvra_parts) != 2:
       raise ValueError("Invalid NVRA: %s" % nvra)

   nvr, arch = nvra_parts
   if "-" in arch:
       raise ValueError("Invalid arch '%s' in '%s'" % (arch, nvra))

   result = parse_nvr(nvr)
   result["arch"] = arch
   result["src"] = arch == "src"
   if epoch != "":
       result["epoch"] = epoch
   return result


def parse_nvr(nvre):
   """Split N-V-R into a dictionary.

   @param nvre: N-V-R:E, E:N-V-R or N-E:V-R string
   @type nvre: str
   @return: {name, version, release, epoch}
   @rtype: dict
   """
   if "/" in nvre:
       nvre = nvre.split("/")[-1]

   nvr, epoch = split_nvr_epoch(nvre)

   nvr_parts = nvr.rsplit("-", 2)
   if len(nvr_parts) != 3:
       raise ValueError("Invalid NVR: %s" % nvr)

   # parse E:V
   if epoch == "" and ":" in nvr_parts[1]:
       epoch, nvr_parts[1] = nvr_parts[1].split(":", 1)

   # check if epoch is empty or numeric
   if epoch != "":
       try:
           int(epoch)
       except ValueError:
           raise ValueError("Invalid epoch '%s' in '%s'" % (epoch, nvr))

   result = dict(zip(["name", "version", "release"], nvr_parts))
   result["epoch"] = epoch
   return result


def split_nvr_epoch(nvre):
   """Split nvre to N-V-R and E.

   @param nvre: E:N-V-R or N-V-R:E string
   @type nvre: str
   @return: (N-V-R, E)
   @rtype: (str, str)
   """
   if ":" in nvre:
       if nvre.count(":") != 1:
           raise ValueError("Invalid NVRE: %s" % nvre)

       nvr, epoch = nvre.rsplit(":", 1)
       if "-" in epoch:
           if "-" not in nvr:
               # switch nvr with epoch
               nvr, epoch = epoch, nvr
           else:
               # it's probably N-E:V-R format, handle it after the split
               nvr, epoch = nvre, ""
   else:
       nvr, epoch = nvre, ""

   return (nvr, epoch)


with open("f34_packages.txt") as f:
   content = f.read()


for line in content.splitlines():
   parts = line.split(" ", maxsplit=2)

   if len(parts) != 2:
       # Skip packages reported without their PyPI counterpart.
       continue

   rpm_name, pypi_name = parts

   rpm_name = parse_nvra(rpm_name)["name"]
   pypi_name = canonicalize_name(pypi_name)

   os.makedirs(f"prescriptions/{pypi_name}", exist_ok=True)

   data = f"""\
units:
 wraps:
   - name: {pypi_name.capitalize()}Fedora34RPMWrap
     type: wrap
     should_include:
       adviser_pipeline: true
     match:
       state:
         resolved_dependencies:
           - name: {pypi_name}
             index_url: 'https://pypi.org/simple'
     run:
       stack_info:
         - type: INFO
           message: "Package {pypi_name!r} is packaged as an RPM in Fedora 34 as {rpm_name!r}"
           link: https://src.fedoraproject.org/rpms/{rpm_name}
"""
   print(data)

   with open(f"prescriptions/{pypi_name}/f34_rpm.yaml", "w") as f:
       f.write(data)

Related PR: thoth-station/prescriptions#44

Closing this, we might revisit this approach following thoth-station/thoth-application#1798

/close

Instructions for interacting with me using PR comments are available here. If you have questions or suggestions related to my behavior, please file an issue against the kubernetes/test-infra repository.