Merge pull request #1058 from TG1999/migrate/istio

Migrate istio importer #1059

CHANGELOG.rst

@@ -7,6 +7,7 @@ Next release
 
 - We re-enabled support for the mozilla vulnerabilities advisories importer.
 - We re-enabled support for the gentoo vulnerabilities advisories importer.
+- We re-enabled support for the istio vulnerabilities advisories importer.
 
 
 Version v31.1.1

vulnerabilities/importers/__init__.py

@@ -15,6 +15,7 @@
 from vulnerabilities.importers import gentoo
 from vulnerabilities.importers import github
 from vulnerabilities.importers import gitlab
+from vulnerabilities.importers import istio
 from vulnerabilities.importers import mozilla
 from vulnerabilities.importers import nginx
 from vulnerabilities.importers import npm
@@ -47,6 +48,7 @@
     apache_httpd.ApacheHTTPDImporter,
     mozilla.MozillaImporter,
     gentoo.GentooImporter,
+    istio.IstioImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/istio.py

@@ -6,165 +6,129 @@
 # See https://github.com/nexB/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-import asyncio
 import re
+from pathlib import Path
 from typing import Set
 
 import pytz
 import saneyaml
 from dateutil import parser
 from packageurl import PackageURL
-from univers.version_range import VersionRange
+from univers.version_constraint import VersionConstraint
+from univers.version_range import GitHubVersionRange
+from univers.version_range import GolangVersionRange
 from univers.versions import SemverVersion
 
 from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import GitImporter
+from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
-from vulnerabilities.package_managers import GitHubTagsAPI
-from vulnerabilities.utils import nearest_patched_package
 from vulnerabilities.utils import split_markdown_front_matter
 
 is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match
 
 
-class IstioImporter(GitImporter):
-    def __enter__(self):
-        super(IstioImporter, self).__enter__()
+class IstioImporter(Importer):
+    spdx_license_expression = "Apache-2.0"
+    license_url = "https://github.com/istio/istio.io/blob/master/LICENSE"
+    repo_url = "git+https://github.com/istio/istio.io/"
 
-        if not getattr(self, "_added_files", None):
-            self._added_files, self._updated_files = self.file_changes(
-                recursive=True, file_ext="md", subdir="./content/en/news/security"
-            )
-        self.version_api = GitHubTagsAPI()
-        self.set_api()
-
-    def set_api(self):
-        asyncio.run(self.version_api.load_api(["istio/istio"]))
-
-    def updated_advisories(self) -> Set[AdvisoryData]:
-        files = self._added_files.union(self._updated_files)
-        advisories = []
-        for f in files:
+    def advisory_data(self) -> Set[AdvisoryData]:
+        self.clone(self.repo_url)
+        path = Path(self.vcs_response.dest_dir)
+        vuln = path / "content/en/news/security/"
+        for file in vuln.glob("**/*.md"):
             # Istio website has files with name starting with underscore, these contain metadata
             # required for rendering the website. We're not interested in these.
             # See also https://github.com/nexB/vulnerablecode/issues/563
-            if f.endswith("_index.md"):
+            file = str(file)
+            if file.endswith("_index.md"):
                 continue
-            processed_data = self.process_file(f)
-            if processed_data:
-                advisories.extend(processed_data)
-        return self.batch_advisories(advisories)
-
-    def get_pkg_versions_from_ranges(self, version_range_list, release_date):
-        """Takes a list of version ranges(affected) of a package
-        as parameter and returns a tuple of safe package versions and
-        vulnerable package versions"""
-        all_version = self.version_api.get("istio/istio", release_date).valid_versions
-        safe_pkg_versions = []
-        vuln_pkg_versions = []
-        version_ranges = [
-            VersionRange.from_scheme_version_spec_string("semver", r) for r in version_range_list
-        ]
-        for version in all_version:
-            version_obj = SemverVersion(version)
-            if any([version_obj in v for v in version_ranges]):
-                vuln_pkg_versions.append(version)
-
-        safe_pkg_versions = set(all_version) - set(vuln_pkg_versions)
-        return safe_pkg_versions, vuln_pkg_versions
+            yield from self.process_file(file)
 
     def process_file(self, path):
 
-        advisories = []
-
         data = self.get_data_from_md(path)
-        release_date = parser.parse(data["publishdate"]).replace(tzinfo=pytz.UTC)
-
-        releases = []
-        if data.get("releases"):
-            for release in data["releases"]:
-                # If it is of form "All releases prior to x"
-                if "All releases prior" in release:
-                    release = release.strip()
-                    release = release.split(" ")
-                    releases.append("<" + release[4])
-
-                # Eg. 'All releases 1.5 and later'
-                elif "All releases" in release and "and later" in release:
-                    release = release.split()[2].strip()
-                    releases.append(f">={release}")
-
-                elif "to" in release:
-                    release = release.strip()
-                    release = release.split(" ")
-                    lbound = ">=" + release[0]
-                    ubound = "<=" + release[2]
-                    releases.append(lbound + "," + ubound)
-                # If it is a single release
-                elif is_release(release):
-                    releases.append(release)
-
-        data["release_ranges"] = releases
-
-        if not data.get("cves"):
-            data["cves"] = [""]
-
-        for cve_id in data["cves"]:
+        published_date = data.get("publishdate")
+        release_date = None
+        if published_date:
+            release_date = parser.parse(published_date).replace(tzinfo=pytz.UTC)
+
+        constraints = []
+
+        for release in data.get("releases") or []:
+            # If it is of form "All releases prior to x"
+            if "All releases prior" in release:
+                _, _, release = release.strip().rpartition(" ")
+                constraints.append(
+                    VersionConstraint(version=SemverVersion(release), comparator="<")
+                )
 
-            if not cve_id.startswith("CVE"):
-                cve_id = ""
+            # Eg. 'All releases 1.5 and later'
+            elif "All releases" in release and "and later" in release:
+                # remove All releases from string
+                release = release.replace("All releases", "").strip()
+                # remove and later from string
+                release = release.replace("and later", "").strip()
+                if not is_release(release):
+                    continue
+                constraints.append(
+                    VersionConstraint(version=SemverVersion(release), comparator=">=")
+                )
 
-            safe_pkg_versions = []
-            vuln_pkg_versions = []
+            # Eg. 1.5 to 2.0
+            elif "to" in release:
+                lower, _, upper = release.strip().partition("to")
+                constraints.append(VersionConstraint(version=SemverVersion(lower), comparator=">="))
+                constraints.append(VersionConstraint(version=SemverVersion(upper), comparator="<="))
 
-            if not data.get("release_ranges"):
-                data["release_ranges"] = []
+            # If it is a single release
+            elif is_release(release):
+                constraints.append(
+                    VersionConstraint(version=SemverVersion(release), comparator="=")
+                )
 
-            safe_pkg_versions, vuln_pkg_versions = self.get_pkg_versions_from_ranges(
-                data["release_ranges"], release_date
-            )
+        for cve_id in data.get("cves") or []:
+
+            if not cve_id.startswith("CVE"):
+                continue
 
             affected_packages = []
 
-            safe_purls_golang = [
-                PackageURL(type="golang", name="istio", version=version)
-                for version in safe_pkg_versions
-            ]
-
-            vuln_purls_golang = [
-                PackageURL(type="golang", name="istio", version=version)
-                for version in vuln_pkg_versions
-            ]
-
-            affected_packages.extend(nearest_patched_package(vuln_purls_golang, safe_purls_golang))
-
-            safe_purls_github = [
-                PackageURL(type="github", name="istio", version=version)
-                for version in safe_pkg_versions
-            ]
-
-            vuln_purls_github = [
-                PackageURL(type="github", name="istio", version=version)
-                for version in vuln_pkg_versions
-            ]
-
-            affected_packages.extend(nearest_patched_package(vuln_purls_github, safe_purls_github))
-
-            advisories.append(
-                AdvisoryData(
-                    vulnerability_id=cve_id,
-                    summary=data["description"],
-                    affected_packages=affected_packages,
-                    references=[
-                        Reference(
-                            reference_id=data["title"],
-                            url=f"https://istio.io/latest/news/security/{data['title']}/",
-                        )
-                    ],
+            if constraints:
+                affected_packages.append(
+                    AffectedPackage(
+                        package=PackageURL(type="golang", namespace="istio.io", name="istio"),
+                        affected_version_range=GolangVersionRange(constraints=constraints),
+                    )
+                )
+
+                affected_packages.append(
+                    AffectedPackage(
+                        package=PackageURL(type="github", namespace="istio", name="istio"),
+                        affected_version_range=GitHubVersionRange(constraints=constraints),
+                    )
                 )
-            )
 
-        return advisories
+            title = data.get("title") or ""
+            references = []
+            if title:
+                references.append(
+                    Reference(
+                        reference_id=title,
+                        url=f"https://istio.io/latest/news/security/{title}/",
+                    )
+                )
+
+            summary = data.get("description") or ""
+
+            yield AdvisoryData(
+                aliases=[cve_id],
+                summary=summary,
+                affected_packages=affected_packages,
+                references=references,
+                date_published=release_date,
+            )
 
     def get_data_from_md(self, path):
         """Return a mapping of vulnerability data extracted from an advisory."""

vulnerabilities/improvers/default.py

@@ -46,15 +46,25 @@ def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
             for affected_package in advisory_data.affected_packages:
                 # To deal with multiple fixed versions in a single affected package
                 affected_purls, fixed_purls = get_exact_purls(affected_package)
-                for fixed_purl in fixed_purls:
+                if not fixed_purls:
                     yield Inference(
                         aliases=advisory_data.aliases,
                         confidence=MAX_CONFIDENCE,
                         summary=advisory_data.summary,
                         affected_purls=affected_purls,
-                        fixed_purl=fixed_purl,
+                        fixed_purl=None,
                         references=advisory_data.references,
                     )
+                else:
+                    for fixed_purl in fixed_purls or []:
+                        yield Inference(
+                            aliases=advisory_data.aliases,
+                            confidence=MAX_CONFIDENCE,
+                            summary=advisory_data.summary,
+                            affected_purls=affected_purls,
+                            fixed_purl=fixed_purl,
+                            references=advisory_data.references,
+                        )
 
         else:
             yield Inference.from_advisory_data(

vulnerabilities/tests/conftest.py

@@ -29,7 +29,6 @@ def no_rmtree(monkeypatch):
     "test_apache_tomcat.py",
     "test_api.py",
     "test_elixir_security.py",
-    "test_istio.py",
     "test_models.py",
     "test_msr2019.py",
     "test_package_managers.py",

vulnerabilities/tests/test_data/istio/istio-expected.json

@@ -0,0 +1,42 @@
+[
+  {
+    "aliases": [
+      "CVE-2019-12243"
+    ],
+    "summary": "Incorrect access control.",
+    "affected_packages": [
+      {
+        "package": {
+          "type": "golang",
+          "namespace": "istio.io",
+          "name": "istio",
+          "version": null,
+          "qualifiers": null,
+          "subpath": null
+        },
+        "affected_version_range": "vers:golang/<0.0.9|>=1.1.0|<=1.1.15|>=1.3.0|<=1.3.1|>=1.5.0",
+        "fixed_version": null
+      },
+      {
+        "package": {
+          "type": "github",
+          "namespace": "istio",
+          "name": "istio",
+          "version": null,
+          "qualifiers": null,
+          "subpath": null
+        },
+        "affected_version_range": "vers:github/<0.0.9|>=1.1.0|<=1.1.15|>=1.3.0|<=1.3.1|>=1.5.0",
+        "fixed_version": null
+      }
+    ],
+    "references": [
+      {
+        "reference_id": "ISTIO-SECURITY-2019-001",
+        "url": "https://istio.io/latest/news/security/ISTIO-SECURITY-2019-001/",
+        "severities": []
+      }
+    ],
+    "date_published": "2019-05-28T00:00:00+00:00"
+  }
+]

vulnerabilities/tests/test_data/istio/test_file.md

@@ -5,7 +5,7 @@ description: Incorrect access control.
 cves: [CVE-2019-12243]
 cvss: "8.9"
 vector: "CVSS:3.0/AV:A/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:N/E:H/RL:O/RC:C"
-releases: ["1.1 to 1.1.15", "1.2 to 1.2.6", "1.3 to 1.3.1"]
+releases: ["All releases prior to 0.0.9","1.1 to 1.1.15","1.3 to 1.3.1", "All releases 1.5.0 and later"]
 publishdate: 2019-05-28
 
 ---