From 03fc65fb8919e30eb230113eebdcb748779387fc Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 4 Jul 2025 19:14:09 +0530 Subject: [PATCH 1/2] Fix gitlab and elixir security importer Signed-off-by: Tushar Goel --- vulnerabilities/pipelines/__init__.py | 3 +++ .../v2_importers/elixir_security_importer.py | 14 +++++++++++--- .../pipelines/v2_importers/gitlab_importer.py | 4 ++-- vulnerabilities/templates/advisory_detail.html | 2 +- vulnerabilities/views.py | 4 ++-- vulnerablecode/urls.py | 10 +++++----- 6 files changed, 24 insertions(+), 13 deletions(-) diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index 3d1316cce..897e3b9cf 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -359,6 +359,9 @@ def get_published_package_versions( """ versions_before_until = [] try: + if until and until.tzinfo is None: + breakpoint() + until = until.replace(tzinfo=timezone.utc) versions = package_versions.versions(str(package_url)) for version in versions or []: if until and version.release_date and version.release_date > until: diff --git a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py index d12f7d947..384a2dafb 100644 --- a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py +++ b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py @@ -39,7 +39,12 @@ class ElixirSecurityImporterPipeline(VulnerableCodeBaseImporterPipelineV2): @classmethod def steps(cls): - return (cls.collect_and_store_advisories,) + return (cls.clone, cls.collect_and_store_advisories, cls.clean_downloads) + + def clean_downloads(self): + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() def clone(self): self.log(f"Cloning `{self.repo_url}`") @@ -62,6 +67,9 @@ def collect_advisories(self) -> Iterable[AdvisoryData]: def process_file(self, file, base_path) -> Iterable[AdvisoryData]: relative_path = str(file.relative_to(base_path)).strip("/") + path_segments = str(file).split("/") + # use the last two segments as the advisory ID + advisory_id = "/".join(path_segments[-2:]).replace(".yml", "") advisory_url = ( f"https://github.com/dependabot/elixir-security-advisories/blob/master/{relative_path}" ) @@ -114,8 +122,8 @@ def process_file(self, file, base_path) -> Iterable[AdvisoryData]: date_published = dateparser.parse(yaml_file.get("disclosure_date")) yield AdvisoryData( - advisory_id=cve_id, - aliases=[], + advisory_id=advisory_id, + aliases=[cve_id], summary=summary, references_v2=references, affected_packages=affected_packages, diff --git a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py index a51875ddf..13f61bd75 100644 --- a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py @@ -233,6 +233,8 @@ def parse_gitlab_advisory( # refer to schema here https://gitlab.com/gitlab-org/advisories-community/-/blob/main/ci/schema/schema.json aliases = gitlab_advisory.get("identifiers") advisory_id = gitlab_advisory.get("identifier") + package_slug = gitlab_advisory.get("package_slug") + advisory_id = f"{package_slug}/{advisory_id}" if package_slug else advisory_id if advisory_id in aliases: aliases.remove(advisory_id) summary = build_description(gitlab_advisory.get("title"), gitlab_advisory.get("description")) @@ -244,8 +246,6 @@ def parse_gitlab_advisory( date_published = dateparser.parse(gitlab_advisory.get("pubdate")) date_published = date_published.replace(tzinfo=pytz.UTC) - package_slug = gitlab_advisory.get("package_slug") - advisory_id = f"{package_slug}/{advisory_id}" if package_slug else advisory_id advisory_url = get_advisory_url( file=file, base_path=base_path, diff --git a/vulnerabilities/templates/advisory_detail.html b/vulnerabilities/templates/advisory_detail.html index 8a386d4ec..c3d93619a 100644 --- a/vulnerabilities/templates/advisory_detail.html +++ b/vulnerabilities/templates/advisory_detail.html @@ -156,7 +156,7 @@ " >Affected and Fixed Packages - + Package Details diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 6d9b7ea87..07eafbdf0 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -570,8 +570,8 @@ class AdvisoryPackagesDetails(DetailView): model = models.AdvisoryV2 template_name = "advisory_package_details.html" - slug_url_kwarg = "id" - slug_field = "id" + slug_url_kwarg = "avid" + slug_field = "avid" def get_queryset(self): """ diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index cb9e318eb..8d170678a 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -103,6 +103,11 @@ def __init__(self, *args, **kwargs): HomePageV2.as_view(), name="home", ), + path( + "advisories/packages/", + AdvisoryPackagesDetails.as_view(), + name="advisory_package_details", + ), path( "advisories/", AdvisoryDetails.as_view(), @@ -143,11 +148,6 @@ def __init__(self, *args, **kwargs): VulnerabilityPackagesDetails.as_view(), name="vulnerability_package_details", ), - path( - "advisories//packages", - AdvisoryPackagesDetails.as_view(), - name="advisory_package_details", - ), path( "api/", include(api_router.urls), From 0998cc7bfb91bc57fb9ca81009843f8f8decb70c Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 4 Jul 2025 19:26:48 +0530 Subject: [PATCH 2/2] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/pipelines/__init__.py | 10 +++++++--- .../pipelines/test_elixir_security_v2_importer.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index 897e3b9cf..aea3b761e 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -359,11 +359,15 @@ def get_published_package_versions( """ versions_before_until = [] try: - if until and until.tzinfo is None: - breakpoint() - until = until.replace(tzinfo=timezone.utc) versions = package_versions.versions(str(package_url)) for version in versions or []: + if ( + version.release_date + and version.release_date.tzinfo + and until + and until.tzinfo is None + ): + until = until.replace(tzinfo=timezone.utc) if until and version.release_date and version.release_date > until: continue versions_before_until.append(version.value) diff --git a/vulnerabilities/tests/pipelines/test_elixir_security_v2_importer.py b/vulnerabilities/tests/pipelines/test_elixir_security_v2_importer.py index 96359ca3c..4c763ab53 100644 --- a/vulnerabilities/tests/pipelines/test_elixir_security_v2_importer.py +++ b/vulnerabilities/tests/pipelines/test_elixir_security_v2_importer.py @@ -72,7 +72,7 @@ def test_collect_advisories(mock_fetch_via_vcs, mock_vcs_response): assert len(advisories) == 1 advisory: AdvisoryData = advisories[0] - assert advisory.advisory_id == "CVE-2022-9999" + assert advisory.advisory_id == "some_package/CVE-2022-9999" assert advisory.summary.startswith("Cross-site scripting vulnerability") assert advisory.affected_packages[0].package.name == "plug" assert advisory.affected_packages[0].package.type == "hex"