diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ba0d0b198..294004e08 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,10 +13,16 @@ Version v37.0.0 - We have added new models AdvisoryV2, AdvisoryAlias, AdvisoryReference, AdvisorySeverity, AdvisoryWeakness, PackageV2 and CodeFixV2. - We are using ``avid`` as an internal advisory ID for uniquely identifying advisories. - We have a new route ``/v2`` which only support package search which has information on packages that are reported to be affected or fixing by advisories. -- This version introduces ``/api/v2/advisories-packages`` which has information on packages that are reported to be affected or fixing by advisories. +- This version introduces ``/api/v3/packages`` which has information on packages that are reported to be affected or fixing by advisories. - Pipeline Dashboard improvements #1920. - Throttle API requests based on user permissions #1909. - Add pipeline to compute Advisory ToDos #1764 +- Use related advisory severity to calculate exploitibility, weighted severity and risk scores +- Migrate all importers to use the new advisory models. All new advisories have a unique AVID and all importers will use this AVID as the unique identifier for advisories instead of CVE ID or other identifiers used by the data sources #1881. +- Handle advisories with same and related data https://github.com/aboutcode-org/vulnerablecode/issues/2099. +- Add a pipeline for exporting VulnerableCode data to FederatedCode #2110. +- Plan storing of exploits and EPSS based advisories #2069. + Version v36.1.3 --------------------- diff --git a/PIPELINES-AVID.rst b/PIPELINES-AVID.rst new file mode 100644 index 000000000..43de21e19 --- /dev/null +++ b/PIPELINES-AVID.rst @@ -0,0 +1,74 @@ +.. list-table:: Pipeline AVID Mapping + :header-rows: 1 + :widths: 35 65 + + * - pipeline name + - AVID + * - alpine_linux_importer_v2 + - {package_name}/{distroversion}/{version}/{vulnerability_id} + * - aosp_dataset_fix_commits + - CVE ID of the record + * - apache_httpd_importer_v2 + - CVE ID of the record + * - apache_kafka_importer_v2 + - CVE ID of the record + * - apache_tomcat_importer_v2 + - {page_id}/{cve_id} + * - archlinux_importer_v2 + - AVG ID of the record + * - curl_importer_v2 + - CURL-CVE ID of the record + * - debian_importer_v2 + - {package_name}/{debian_record_id} + * - elixir_security_importer_v2 + - {package_name}/{file_id} + * - epss_importer_v2 + - CVE ID of the record + * - fireeye_importer_v2 + - {file_id} + * - gentoo_importer_v2 + - GLSA ID of the record + * - github_osv_importer_v2 + - ID of the OSV record + * - gitlab_importer_v2 + - Identifier of the GitLab community advisory record + * - istio_importer_v2 + - ISTIO-SECURITY- + * - mattermost_importer_v2 + - MMSA- + * - mozilla_importer_v2 + - MFSA- + * - nginx_importer_v2 + - First alias of the record + * - nodejs_security_wg + - NPM- + * - nvd_importer_v2 + - CVE ID of the record + * - openssl_importer_v2 + - CVE ID of the record + * - oss_fuzz_importer_v2 + - ID of the OSV record + * - postgresql_importer_v2 + - CVE ID of the record + * - project-kb-msr-2019_v2 + - Vulnerability ID of the record + * - project-kb-statements_v2 + - Vulnerability ID of the record + * - pypa_importer_v2 + - ID of the OSV record + * - pysec_importer_v2 + - ID of the OSV record + * - redhat_importer_v2 + - RHSA ID of the record + * - retiredotnet_importer_v2 + - retiredotnet-{file_id} + * - ruby_importer_v2 + - {file_id} + * - suse_importer_v2 + - CVE ID of the record + * - ubuntu_osv_importer_v2 + - ID of the OSV record + * - vulnrichment_importer_v2 + - CVE ID of the record + * - xen_importer_v2 + - XSA- \ No newline at end of file diff --git a/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py b/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py index 642a37435..336afa298 100644 --- a/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py +++ b/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json import logging from pathlib import Path from typing import Any @@ -244,4 +245,5 @@ def load_advisories( references=references, affected_packages=affected_packages, url=url, + original_advisory_text=json.dumps(pkg_infos, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py b/vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py index f55665c56..f7dd12f57 100644 --- a/vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py +++ b/vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py @@ -111,6 +111,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: summary=advisory_list[0].summary, affected_packages=affected_packages, url=page_url, + original_advisory_text=str(content), ) except Exception as e: diff --git a/vulnerabilities/pipelines/v2_importers/archlinux_importer.py b/vulnerabilities/pipelines/v2_importers/archlinux_importer.py index 4f9096f09..b0f005592 100644 --- a/vulnerabilities/pipelines/v2_importers/archlinux_importer.py +++ b/vulnerabilities/pipelines/v2_importers/archlinux_importer.py @@ -14,10 +14,13 @@ from packageurl import PackageURL from univers.version_range import ArchLinuxVersionRange +from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import fetch_response @@ -53,7 +56,9 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: def parse_advisory(self, record) -> AdvisoryDataV2: affected_packages = [] references = [] + severities = [] avg_name = record.get("name") + severity = record.get("severity") aliases = record.get("issues", []) aliases.extend(record.get("advisories", [])) summary = record.get("type", "") @@ -92,13 +97,26 @@ def parse_advisory(self, record) -> AdvisoryDataV2: ) ) + if severity not in severity_systems.ARCHLINUX.choices: + self.log(f"Unknown severity {severity} for {avg_name}") + severity = None + if severity: + severities = [ + VulnerabilitySeverity( + system=severity_systems.ARCHLINUX, + value=severity, + url="https://security.archlinux.org/{avg_name}.json", + ) + ] + return AdvisoryDataV2( advisory_id=avg_name, aliases=aliases, summary=summary, references=references, affected_packages=affected_packages, + severities=severities, weaknesses=[], url=f"https://security.archlinux.org/{avg_name}.json", - original_advisory_text=json.dumps(record), + original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/debian_importer.py b/vulnerabilities/pipelines/v2_importers/debian_importer.py index bdc0770ca..67cba2b11 100644 --- a/vulnerabilities/pipelines/v2_importers/debian_importer.py +++ b/vulnerabilities/pipelines/v2_importers/debian_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json import re from typing import Any from typing import Iterable @@ -171,6 +172,7 @@ def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryD references=references, weaknesses=weaknesses, url=f"https://security-tracker.debian.org/tracker/{record_identifier}", + original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py b/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py index e1d6f6eaf..6f8adc6d1 100644 --- a/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py +++ b/vulnerabilities/pipelines/v2_importers/epss_importer_v2.py @@ -83,4 +83,5 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: severities=[severity], references=[references], url=self.advisory_url, + original_advisory_text=",".join(epss_row), ) diff --git a/vulnerabilities/pipelines/v2_importers/istio_importer.py b/vulnerabilities/pipelines/v2_importers/istio_importer.py index a9a3de881..ce7b7d342 100644 --- a/vulnerabilities/pipelines/v2_importers/istio_importer.py +++ b/vulnerabilities/pipelines/v2_importers/istio_importer.py @@ -42,6 +42,7 @@ class IstioImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "Apache-2.0" license_url = "https://github.com/istio/istio.io/blob/master/LICENSE" repo_url = "git+https://github.com/istio/istio.io" + run_once = True precedence = 200 diff --git a/vulnerabilities/pipelines/v2_importers/mattermost_importer.py b/vulnerabilities/pipelines/v2_importers/mattermost_importer.py index 538bb1d4d..852939fd8 100644 --- a/vulnerabilities/pipelines/v2_importers/mattermost_importer.py +++ b/vulnerabilities/pipelines/v2_importers/mattermost_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json from typing import Iterable from packageurl import PackageURL @@ -122,5 +123,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: summary=details, references=[reference], affected_packages=affected_packages, + severities=severities, url=self.url, + original_advisory_text=json.dumps(advisory, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/nginx_importer.py b/vulnerabilities/pipelines/v2_importers/nginx_importer.py index 81448166b..f5f3e3c67 100644 --- a/vulnerabilities/pipelines/v2_importers/nginx_importer.py +++ b/vulnerabilities/pipelines/v2_importers/nginx_importer.py @@ -62,7 +62,7 @@ def collect_advisories(self): vulnerability_list = soup.select("li p") for vulnerability_info in vulnerability_list: ngnix_advisory = parse_advisory_data_from_paragraph(vulnerability_info) - yield to_advisory_data(ngnix_advisory) + yield to_advisory_data(ngnix_advisory, vulnerability_info) class NginxAdvisory(NamedTuple): @@ -79,7 +79,7 @@ def to_dict(self): return self._asdict() -def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2: +def to_advisory_data(nginx_adv: NginxAdvisory, vulnerability_info) -> AdvisoryDataV2: """ Return AdvisoryDataV2 from an NginxAdvisory tuple. """ @@ -150,6 +150,7 @@ def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2: references=nginx_adv.references, patches=nginx_adv.patches, url="https://nginx.org/en/security_advisories.html", + original_advisory_text=str(vulnerability_info), ) diff --git a/vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py b/vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py index 7dd01233e..1a4411acd 100644 --- a/vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py +++ b/vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py @@ -90,6 +90,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: patches=patches, references=references, url="https://github.com/SAP/project-kb/blob/main/MSR2019/dataset/vulas_db_msr2019_release.csv", + original_advisory_text=",".join(row), ) def clean_downloads(self): diff --git a/vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py b/vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py index b67540bce..0c1c8e05d 100644 --- a/vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py +++ b/vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py @@ -168,6 +168,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: references=references, patches=patches, url=advisory_url, + original_advisory_text=saneyaml.dump(yaml_data, indent=2), ) def clean_downloads(self): diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py index 478d31323..cb87183e3 100644 --- a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -118,6 +118,7 @@ def collect_advisories(self): affected_packages=affected_packages, references=vuln_reference, url=advisory_url, + original_advisory_text=json.dumps(json_doc, indent=2, ensure_ascii=False), ) @staticmethod diff --git a/vulnerabilities/pipelines/v2_importers/ruby_importer.py b/vulnerabilities/pipelines/v2_importers/ruby_importer.py index db02f4823..455a0a491 100644 --- a/vulnerabilities/pipelines/v2_importers/ruby_importer.py +++ b/vulnerabilities/pipelines/v2_importers/ruby_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json import logging from pathlib import Path from typing import Iterable @@ -129,6 +130,7 @@ def parse_ruby_advisory(advisory_id, record, schema_type, advisory_url): severities=get_severities(record), date_published=get_publish_time(record), url=advisory_url, + original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False), ) elif schema_type == "rubies": @@ -147,6 +149,7 @@ def parse_ruby_advisory(advisory_id, record, schema_type, advisory_url): references=get_references(record), date_published=get_publish_time(record), url=advisory_url, + original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/suse_score_importer.py b/vulnerabilities/pipelines/v2_importers/suse_score_importer.py index b39abcd11..92a534ddc 100644 --- a/vulnerabilities/pipelines/v2_importers/suse_score_importer.py +++ b/vulnerabilities/pipelines/v2_importers/suse_score_importer.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import json from typing import Iterable from vulnerabilities import severity_systems @@ -67,4 +68,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: severities=severities, references=[], url=self.url, + original_advisory_text=json.dumps( + self.score_data[cve_id], indent=2, ensure_ascii=False + ), ) diff --git a/vulnerabilities/severity_systems.py b/vulnerabilities/severity_systems.py index 27f9d7d1a..6624a656e 100644 --- a/vulnerabilities/severity_systems.py +++ b/vulnerabilities/severity_systems.py @@ -169,6 +169,14 @@ def get(self, scoring_elements: str) -> dict: "Low", ] +ARCHLINUX.choices = [ + "Critical", + "High", + "Medium", + "Low", + "Very Low", +] + # This is essentially identical to apache_http except for the addition of the "High" score, # which seems to be used interchangeably for "Important". APACHE_TOMCAT = ScoringSystem( diff --git a/vulnerabilities/templates/index.html b/vulnerabilities/templates/index.html index 78effa82d..77ad8d1c0 100644 --- a/vulnerabilities/templates/index.html +++ b/vulnerabilities/templates/index.html @@ -33,6 +33,9 @@

+
+ ATTENTION: We will be deprecating V1 and V2 API by 30th June 2026. V3 endpoint is live now. Please migrate to V3 API before the deprecation date. For more details, please refer to this blog. +
{% endblock %} \ No newline at end of file diff --git a/vulnerabilities/templates/navbar.html b/vulnerabilities/templates/navbar.html index de6dc1a94..3d3fa0e91 100644 --- a/vulnerabilities/templates/navbar.html +++ b/vulnerabilities/templates/navbar.html @@ -26,6 +26,9 @@ Vulnerabilities + + V2 + Documentation diff --git a/vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json b/vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json index 4902884f5..06cee73f3 100644 --- a/vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json +++ b/vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json @@ -107,7 +107,13 @@ } ], "patches": [], - "severities": [], + "severities": [ + { + "system": "archlinux", + "value": "Low", + "scoring_elements": "" + } + ], "date_published": null, "weaknesses": [], "url": "https://security.archlinux.org/AVG-4.json"