Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
3e094c9
Add migrations and code to infer patched package.
sbs2001 Apr 4, 2021
7021526
Refactor the import process to implement patched_package
sbs2001 Apr 9, 2021
d684871
Refactor alpine importer and it's tests fwrt new models
sbs2001 Apr 9, 2021
8bc91ff
Refactor debian importer and it's tests wrt to new models
sbs2001 Apr 9, 2021
cea3738
Refactor apache kafka and it's tests wrt new models
sbs2001 Apr 9, 2021
b0ccf83
Refactor apache tomcata and it's importers for new models
sbs2001 Apr 9, 2021
5344e95
Refactor archlinux importer wrt new models
sbs2001 Apr 9, 2021
6607c29
Refactor elixr security importer wrt new models
sbs2001 Apr 9, 2021
d430a21
Refactor gentoo importer wrt new models and update univers
sbs2001 Apr 9, 2021
46fd0a0
Use AffectedPackageWithPatched dataclass and refactor the Advisory in…
sbs2001 Apr 9, 2021
1508274
Refactor github importer wrt new models
sbs2001 Apr 9, 2021
2b3334f
Refactor istio importer wrt new models
sbs2001 Apr 9, 2021
1cae542
Refactor kaybee importer wrt new models
sbs2001 Apr 9, 2021
f73fbc9
Refactor nginx importer wrt new models
sbs2001 Apr 9, 2021
e97c415
Refactor npm importer wrt new models
sbs2001 Apr 10, 2021
c1c0d14
Refactor openssl wrt new models
sbs2001 Apr 10, 2021
e9cd094
Refactor postgresql importer
sbs2001 Apr 10, 2021
db17f09
Refactor msr2019 importer
sbs2001 Apr 10, 2021
9f6bb60
Refactor redhat importer
sbs2001 Apr 10, 2021
92fd1ad
Refactor retire dot net
sbs2001 Apr 10, 2021
30b0b49
Refactor ruby importer
sbs2001 Apr 10, 2021
c852afc
Refactor rust importer and simplify it's tests
sbs2001 Apr 10, 2021
d1f5485
Refactor safetydb wrt new models and simplify tests
sbs2001 Apr 12, 2021
527324f
Disable suse backport importer and refactor suse score importer
sbs2001 Apr 12, 2021
29f1185
Refactor ubuntu usn importer and tests
sbs2001 Apr 12, 2021
c12e985
Refactor ubuntu
sbs2001 Apr 12, 2021
cbd6c73
Fix more tests and refactor importer_runner
sbs2001 Apr 12, 2021
9246553
Update fixtures
sbs2001 Apr 15, 2021
36f4d1d
Fix model relations and patch inference in importers
sbs2001 Apr 16, 2021
5999a8a
Simplify patch inference, fix arch importer bug and remove useless code
sbs2001 Apr 19, 2021
d70a931
Fix codestyle
sbs2001 Apr 19, 2021
5973bbc
Add docstrings for model properties
sbs2001 Apr 19, 2021
cc5bbb4
Improve naming and docs
sbs2001 Apr 22, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ cached-property==1.5.1
cffi==1.14.0
contextlib2==0.5.5
decorator==4.4.2
univers==21.4.8
univers==21.4.16.6
dj-database-url==0.4.2
Django==3.0.14
django-filter==2.2.0
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class PackageAdmin(admin.ModelAdmin):

@admin.register(PackageRelatedVulnerability)
class PackageRelatedVulnerabilityAdmin(admin.ModelAdmin):
list_filter = ("is_vulnerable", "package__type", "package__namespace")
list_filter = ("package__type", "package__namespace")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there anything that could replace is_vulnerable?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that should be patched_package adding it.

search_fields = ["vulnerability__vulnerability_id", "package__name"]


Expand Down
42 changes: 17 additions & 25 deletions vulnerabilities/data_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
# Visit https://github.com/nexB/vulnerablecode/ for support and download.

import pickle
import dataclasses
import logging
import os
Expand All @@ -47,6 +46,8 @@
from vulnerabilities.oval_parser import OvalParser
from vulnerabilities.severity_systems import ScoringSystem
from vulnerabilities.helpers import is_cve
from vulnerabilities.helpers import nearest_patched_package
from vulnerabilities.helpers import AffectedPackage

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -87,17 +88,14 @@ class Advisory:

summary: str
vulnerability_id: Optional[str] = None
impacted_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list)
resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list)
affected_packages: List[AffectedPackage] = dataclasses.field(default_factory=list)
references: List[Reference] = dataclasses.field(default_factory=list)

def __post_init__(self):
if self.vulnerability_id and not is_cve(self.vulnerability_id):
raise ValueError("CVE expected, found: {}".format(self.vulnerability_id))

def normalized(self):
impacted_package_urls = {package_url for package_url in self.impacted_package_urls}
resolved_package_urls = {package_url for package_url in self.resolved_package_urls}
references = sorted(
self.references, key=lambda reference: (reference.reference_id, reference.url)
)
Expand All @@ -107,8 +105,7 @@ def normalized(self):
return Advisory(
summary=self.summary,
vulnerability_id=self.vulnerability_id,
impacted_package_urls=impacted_package_urls,
resolved_package_urls=resolved_package_urls,
affected_packages=sorted(self.affected_packages),
references=references,
)

Expand Down Expand Up @@ -531,9 +528,8 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
# connected/linked to an OvalDefinition
vuln_id = definition_data["vuln_id"]
description = definition_data["description"]
affected_purls = set()
safe_purls = set()
references = [Reference(url=url) for url in definition_data["reference_urls"]]
affected_packages = []
for test_data in definition_data["test_data"]:
for package_name in test_data["package_list"]:
if package_name and len(package_name) >= 50:
Expand All @@ -552,35 +548,31 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
# FIXME: we should not drop data this way
# This filter is for filtering out long versions.
# 50 is limit because that's what db permits atm.
all_versions = set(filter(lambda x: len(x) < 50, all_versions))
all_versions = [version for version in all_versions if len(version) < 50]
if not all_versions:
continue
affected_versions = set(
filter(lambda x: version_class(x) in affected_version_range, all_versions)
)
safe_versions = all_versions - affected_versions

for version in affected_versions:
affected_purls = []
safe_purls = []
for version in all_versions:
purl = self.create_purl(
pkg_name=package_name,
pkg_version=version,
pkg_data=pkg_metadata,
)
affected_purls.add(purl)
if version_class(version) in affected_version_range:
affected_purls.append(purl)
else:
safe_purls.append(purl)

for version in safe_versions:
purl = self.create_purl(
pkg_name=package_name,
pkg_version=version,
pkg_data=pkg_metadata,
)
safe_purls.add(purl)
affected_packages.extend(
nearest_patched_package(affected_purls, safe_purls),
)

all_adv.append(
Advisory(
summary=description,
impacted_package_urls=affected_purls,
resolved_package_urls=safe_purls,
affected_packages=affected_packages,
vulnerability_id=vuln_id,
references=references,
)
Expand Down
195 changes: 87 additions & 108 deletions vulnerabilities/fixtures/debian.json
Original file line number Diff line number Diff line change
@@ -1,110 +1,89 @@
[
{
"model": "vulnerabilities.vulnerability",
"pk": 1,
"fields": {
"vulnerability_id": "CVE-2014-8242",
"summary": ""

{
"model": "vulnerabilities.vulnerability",
"pk": 1,
"fields": {
"vulnerability_id": "CVE-2014-8242",
"old_vulnerability_id": null,
"summary": ""
}
},
{
"model": "vulnerabilities.vulnerability",
"pk": 2,
"fields": {
"vulnerability_id": "CVE-2009-1382",
"old_vulnerability_id": null,
"summary": ""
}
},
{
"model": "vulnerabilities.vulnerability",
"pk": 3,
"fields": {
"vulnerability_id": "CVE-2009-2459",
"old_vulnerability_id": null,
"summary": ""
}
},
{
"model": "vulnerabilities.package",
"pk": 1,
"fields": {
"type": "deb",
"namespace": "debian",
"name": "librsync",
"version": "0.9.7-10",
"subpath": "",
"qualifiers": {
"distro": "jessie"
}
}
},
{
"model": "vulnerabilities.package",
"pk": 2,
"fields": {
"type": "deb",
"namespace": "debian",
"name": "mimetex",
"version": "1.74-1",
"subpath": "",
"qualifiers": {
"distro": "jessie"
}
}
},
{
"model": "vulnerabilities.package",
"pk": 3,
"fields": {
"type": "deb",
"namespace": "debian",
"name": "mimetex",
"version": "1.50-1.1",
"subpath": "",
"qualifiers": {
"distro": "jessie"
}
}
},
{
"model": "vulnerabilities.packagerelatedvulnerability",
"pk": 1,
"fields": {
"package": 1,
"vulnerability": 1,
"patched_package": null
}
},
{
"model": "vulnerabilities.packagerelatedvulnerability",
"pk": 4,
"fields": {
"package": 3,
"vulnerability": 3,
"patched_package": 2
}
}
},
{
"model": "vulnerabilities.vulnerability",
"pk": 2,
"fields": {
"vulnerability_id": "CVE-2009-1382",
"summary": ""

}
},
{
"model": "vulnerabilities.vulnerability",
"pk": 3,
"fields": {
"vulnerability_id": "CVE-2009-2459",
"summary": ""

}
},
{
"model": "vulnerabilities.package",
"pk": 1,
"fields": {
"type": "deb",
"namespace": "debian",
"name": "librsync",
"version": "0.9.7-10",
"qualifiers": {"distro":"jessie"},
"subpath": ""
}
},
{
"model": "vulnerabilities.package",
"pk": 2,
"fields": {
"type": "deb",
"namespace": "debian",
"name": "mimetex",
"version": "1.74-1",
"qualifiers": {"distro":"jessie"},
"subpath": ""
}
},
{
"model": "vulnerabilities.package",
"pk": 3,
"fields": {
"type": "deb",
"namespace": "debian",
"name": "mimetex",
"version": "1.50-1.1",
"qualifiers": {"distro":"jessie"},
"subpath": ""
}
},
{
"model": "vulnerabilities.packagerelatedvulnerability",
"pk": 1,
"fields": {
"vulnerability": 1,
"package": 1,
"is_vulnerable": true
}
},
{
"model": "vulnerabilities.packagerelatedvulnerability",
"pk": 10,
"fields": {
"vulnerability": 2,
"package": 2,
"is_vulnerable": false
}
},
{
"model": "vulnerabilities.packagerelatedvulnerability",
"pk": 2,
"fields": {
"vulnerability": 2,
"package": 3,
"is_vulnerable": false
}
},
{
"model": "vulnerabilities.packagerelatedvulnerability",
"pk": 3,
"fields": {
"vulnerability": 3,
"package": 2,
"is_vulnerable": false
}
},
{
"model": "vulnerabilities.packagerelatedvulnerability",
"pk": 4,
"fields": {
"vulnerability": 3,
"package": 3,
"is_vulnerable": false
}
}
]
]
13 changes: 2 additions & 11 deletions vulnerabilities/fixtures/github.json
Original file line number Diff line number Diff line change
Expand Up @@ -74,22 +74,13 @@
"qualifiers": {}
}
},
{
"model": "vulnerabilities.packagerelatedvulnerability",
"pk": 3844,
"fields": {
"package": 3469,
"vulnerability": 60,
"is_vulnerable": false
}
},
{
"model": "vulnerabilities.packagerelatedvulnerability",
"pk": 3845,
"fields": {
"package": 3467,
"vulnerability": 60,
"is_vulnerable": true
"patched_package": 3469
}
},
{
Expand All @@ -98,7 +89,7 @@
"fields": {
"package": 3468,
"vulnerability": 60,
"is_vulnerable": true
"patched_package": 3469
}
},
{
Expand Down
Loading