Merge branch 'main' into feature/mula/improve-ranking

* main:
  Add bit to set default values for FindingType risk levels in Octopoes (#1075)
  Fix thread termination in Mula (#1003)
  test(boefjes): snyk (#1116)
  Add endpoints in Octopoes for bulk operations in the object list page (#1067)
  Remove tasks and items on mutation delete (#1090)
  Persist impact, recommendation and source fields in FindingType objects in XTDB (#1126)
  Handle an empty plugin.consumes field for the plugin detail page (#1104)
  Add script to automatically backport PR to release branch (#1097)
  Fix typos in 'no organizations found' message (#1123)
  Finding Types Boefjes (#1056)
  add findingtype files (#1117)
  Remove containers after `docker-compose run` (#1112)
  Bump cryptography from 39.0.1 to 41.0.0 in /boefjes/boefjes/plugins/kat_ssl_certificates (#1099)
  Bump cryptography from 40.0.2 to 41.0.1 (#1108)
  Bump cryptography from 40.0.2 to 41.0.0 in /bytes (#1100)
  Fix failing test-debian-install in CI (#1111)
  Remove unused boefje fields when creating a BoefjeTask object to send to the scheduler (#1103)
  add 'ideas' as a category in project guidelines (#1105)

.github/workflows/test_debian_packages_on_ubuntu.yml

@@ -217,7 +217,7 @@ jobs:
         run: sudo sed -i "s/XTDB_TYPE=\"xtdb\"/XTDB_TYPE=\"xtdb-multinode\"/g" /etc/kat/octopoes.conf
 
       - name: Restart KAT
-        run: sudo systemctl restart kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker xtdb-http-multinode
+        run: sudo systemctl restart kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 
       - name: Setup accounts in Rocky
         run: |
@@ -246,7 +246,7 @@ jobs:
 
       - name: Check XTDB health or print response and logs
         run: |
-          for i in {1..15}; do curl -s -H "Accept: application/edn" http://localhost:3000/_dev/_xtdb/test/status && s=0 && break || s=$? && sleep 1 ; done
+          for i in {1..30}; do curl -s -H "Accept: application/edn" http://localhost:3000/_dev/_xtdb/test/status && s=0 && break || s=$? && sleep 1 ; done
           if [ $s != 0 ]; then echo $(curl -s -H "Accept: application/edn" http://localhost:3000/_dev/_xtdb/test/status) || true && journalctl --no-pager -u xtdb-http-multinode.service && exit $s ; fi
 
       - name: Create _dev node in Octopoes

boefjes/Makefile

@@ -31,7 +31,7 @@ help: ## Show this help.
 build: seed
 
 seed:  # Seed the katalogus database
-	-docker-compose run katalogus python -m boefjes.seed
+	-docker-compose run --rm katalogus python -m boefjes.seed
 
 ##
 ##|------------------------------------------------------------------------|
@@ -42,12 +42,12 @@ migrations: ## Generate a migration using alembic
 ifeq ($(m),)
 	$(HIDE) (echo "Specify a message with m={message} and a rev-id with revid={revid} (e.g. 0001 etc.)"; exit 1)
 else
-	docker-compose run katalogus python -m alembic --config /app/boefjes/boefjes/alembic.ini revision --autogenerate -m "$(m)"
+	docker-compose run --rm katalogus python -m alembic --config /app/boefjes/boefjes/alembic.ini revision --autogenerate -m "$(m)"
 endif
 
 
 sql: ## Generate raw sql for the migrations
-	docker-compose run katalogus python -m alembic --config /app/boefjes/boefjes/alembic.ini upgrade $(rev1):$(rev2) --sql
+	docker-compose run --rm katalogus python -m alembic --config /app/boefjes/boefjes/alembic.ini upgrade $(rev1):$(rev2) --sql
 
 check:
 	pre-commit run --all-files --color always

boefjes/boefjes/plugins/kat_adr_finding_types/boefje.json

@@ -0,0 +1,14 @@
+{
+    "id": "adr-finding-types",
+    "name": "ADR Finding Types",
+    "description": "Hydrate information of ADR finding types",
+    "consumes": [
+        "ADRFindingType"
+    ],
+    "produces": [
+        "ADRFindingType"
+    ],
+    "environment_keys": [],
+    "scan_level": 0,
+    "enabled": true
+}

boefjes/boefjes/plugins/kat_adr_finding_types/main.py

@@ -0,0 +1,15 @@
+from typing import List, Tuple, Union
+
+import requests
+
+from boefjes.job_models import BoefjeMeta
+
+FINDING_TYPES_JSON_LOCATION = (
+    "https://raw.githubusercontent.com/minvws/nl-kat-coordination/main/data/adr_finding_types.json"
+)
+
+
+def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
+    response = requests.get(f"{FINDING_TYPES_JSON_LOCATION}")
+
+    return [(set(), response.content)]

boefjes/boefjes/plugins/kat_adr_finding_types/normalize.py

@@ -0,0 +1,36 @@
+import json
+import logging
+from typing import Iterable, Union
+
+from boefjes.job_models import NormalizerMeta
+from octopoes.models import OOI
+from octopoes.models.ooi.findings import ADRFindingType, RiskLevelSeverity
+
+logger = logging.getLogger(__name__)
+
+
+SEVERITY_SCORE_LOOKUP = {
+    RiskLevelSeverity.CRITICAL: 10.0,
+    RiskLevelSeverity.HIGH: 8.9,
+    RiskLevelSeverity.MEDIUM: 6.9,
+    RiskLevelSeverity.LOW: 3.9,
+    RiskLevelSeverity.RECOMMENDATION: 0.0,
+}
+
+
+def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterable[OOI]:
+    adr_finding_type_id = normalizer_meta.raw_data.boefje_meta.arguments["input"]["id"]
+    data = json.loads(raw)
+
+    finding_type_information = data[adr_finding_type_id]
+    logger.info(finding_type_information["risk"].lower())
+    risk_severity = RiskLevelSeverity(finding_type_information["risk"].lower())
+
+    risk_score = SEVERITY_SCORE_LOOKUP[risk_severity]
+
+    yield ADRFindingType(
+        id=adr_finding_type_id,
+        description=finding_type_information["description"],
+        risk_severity=risk_severity,
+        risk_score=risk_score,
+    )

boefjes/boefjes/plugins/kat_adr_finding_types/normalizer.json

@@ -0,0 +1,10 @@
+{
+    "id": "kat_adr_finding_types_normalize",
+    "consumes": [
+        "adr-finding-types"
+    ],
+    "produces": [
+        "ADRFindingType"
+    ],
+    "enabled": true
+}

boefjes/boefjes/plugins/kat_cve_finding_types/boefje.json

@@ -0,0 +1,14 @@
+{
+    "id": "cve-finding-types",
+    "name": "CVE Finding Types",
+    "description": "Hydrate information of CVE finding types from the CVE API",
+    "consumes": [
+        "CVEFindingType"
+    ],
+    "produces": [
+        "CVEFindingType"
+    ],
+    "environment_keys": [],
+    "scan_level": 0,
+    "enabled": true
+}

boefjes/boefjes/plugins/kat_cve_finding_types/main.py

@@ -0,0 +1,12 @@
+from typing import List, Tuple, Union
+
+import requests
+
+from boefjes.job_models import BoefjeMeta
+
+
+def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
+    cve_id = boefje_meta.arguments["input"]["id"]
+    response = requests.get(f"https://v1.cveapi.com/{cve_id}.json")
+
+    return [(set(), response.content)]

boefjes/boefjes/plugins/kat_cve_finding_types/normalize.py

@@ -0,0 +1,48 @@
+import json
+import logging
+from typing import Iterable, Union
+
+from boefjes.job_models import NormalizerMeta
+from octopoes.models import OOI
+from octopoes.models.ooi.findings import CVEFindingType, RiskLevelSeverity
+
+logger = logging.getLogger(__name__)
+
+
+SEVERITY_SCORE_LOOKUP = {
+    RiskLevelSeverity.CRITICAL: 10.0,
+    RiskLevelSeverity.HIGH: 8.9,
+    RiskLevelSeverity.MEDIUM: 6.9,
+    RiskLevelSeverity.LOW: 3.9,
+    RiskLevelSeverity.RECOMMENDATION: 0.0,
+}
+
+
+def get_risk_level(severity_score):
+    for risk_level, score in SEVERITY_SCORE_LOOKUP.items():
+        if severity_score >= score:
+            return risk_level
+    return None
+
+
+def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterable[OOI]:
+    cve_finding_type_id = normalizer_meta.raw_data.boefje_meta.arguments["input"]["id"]
+    data = json.loads(raw)
+
+    descriptions = data["cve"]["description"]["description_data"]
+    english_description = [description for description in descriptions if description["lang"] == "en"][0]
+
+    if data["impact"] == {}:
+        risk_severity = RiskLevelSeverity.UNKNOWN
+        risk_score = None
+    else:
+        risk_score = data["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]
+        risk_severity = get_risk_level(risk_score)
+
+    yield CVEFindingType(
+        id=cve_finding_type_id,
+        description=english_description["value"],
+        source=f"https://cve.circl.lu/cve/{cve_finding_type_id}",
+        risk_severity=risk_severity,
+        risk_score=risk_score,
+    )

boefjes/boefjes/plugins/kat_cve_finding_types/normalizer.json

@@ -0,0 +1,10 @@
+{
+    "id": "kat_cve_finding_types_normalize",
+    "consumes": [
+        "cve-finding-types"
+    ],
+    "produces": [
+        "CVEFindingType"
+    ],
+    "enabled": true
+}

boefjes/boefjes/plugins/kat_kat_finding_types/boefje.json

@@ -0,0 +1,14 @@
+{
+    "id": "kat-finding-types",
+    "name": "KAT Finding Types",
+    "description": "Hydrate information of KAT finding types",
+    "consumes": [
+        "KATFindingType"
+    ],
+    "produces": [
+        "KATFindingType"
+    ],
+    "environment_keys": [],
+    "scan_level": 0,
+    "enabled": true
+}

boefjes/boefjes/plugins/kat_kat_finding_types/main.py

@@ -0,0 +1,15 @@
+from typing import List, Tuple, Union
+
+import requests
+
+from boefjes.job_models import BoefjeMeta
+
+FINDING_TYPES_JSON_LOCATION = (
+    "https://raw.githubusercontent.com/minvws/nl-kat-coordination/main/data/kat_finding_types.json"
+)
+
+
+def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
+    response = requests.get(f"{FINDING_TYPES_JSON_LOCATION}")
+
+    return [(set(), response.content)]

boefjes/boefjes/plugins/kat_kat_finding_types/normalize.py

@@ -0,0 +1,39 @@
+import json
+import logging
+from typing import Iterable, Union
+
+from boefjes.job_models import NormalizerMeta
+from octopoes.models import OOI
+from octopoes.models.ooi.findings import KATFindingType, RiskLevelSeverity
+
+logger = logging.getLogger(__name__)
+
+
+SEVERITY_SCORE_LOOKUP = {
+    RiskLevelSeverity.CRITICAL: 10.0,
+    RiskLevelSeverity.HIGH: 8.9,
+    RiskLevelSeverity.MEDIUM: 6.9,
+    RiskLevelSeverity.LOW: 3.9,
+    RiskLevelSeverity.RECOMMENDATION: 0.0,
+}
+
+
+def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterable[OOI]:
+    kat_finding_type_id = normalizer_meta.raw_data.boefje_meta.arguments["input"]["id"]
+    data = json.loads(raw)
+
+    finding_type_information = data[kat_finding_type_id]
+    logger.info(finding_type_information["risk"].lower())
+    risk_severity = RiskLevelSeverity(finding_type_information["risk"].lower())
+
+    risk_score = SEVERITY_SCORE_LOOKUP[risk_severity]
+
+    yield KATFindingType(
+        id=kat_finding_type_id,
+        description=finding_type_information.get("description", None),
+        source=finding_type_information.get("source", None),
+        impact=finding_type_information.get("impact", None),
+        recommendation=finding_type_information.get("recommendation", None),
+        risk_severity=risk_severity,
+        risk_score=risk_score,
+    )

boefjes/boefjes/plugins/kat_kat_finding_types/normalizer.json

@@ -0,0 +1,10 @@
+{
+    "id": "kat_kat_finding_types_normalize",
+    "consumes": [
+        "kat-finding-types"
+    ],
+    "produces": [
+        "KATFindingType"
+    ],
+    "enabled": true
+}

boefjes/boefjes/plugins/kat_retirejs_finding_types/boefje.json

@@ -0,0 +1,14 @@
+{
+    "id": "retirejs-finding-types",
+    "name": "RetireJS Finding Types",
+    "description": "Hydrate information of RetireJS finding types",
+    "consumes": [
+        "RetireJSFindingType"
+    ],
+    "produces": [
+        "RetireJSFindingType"
+    ],
+    "environment_keys": [],
+    "scan_level": 0,
+    "enabled": true
+}

boefjes/boefjes/plugins/kat_retirejs_finding_types/main.py

@@ -0,0 +1,11 @@
+from typing import List, Tuple, Union
+
+import requests
+
+from boefjes.job_models import BoefjeMeta
+
+
+def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
+    response = requests.get("https://raw.githubusercontent.com/RetireJS/retire.js/v3/repository/jsrepository.json")
+
+    return [(set(), response.content)]

boefjes/boefjes/plugins/kat_retirejs_finding_types/normalize.py

@@ -0,0 +1,71 @@
+import hashlib
+import json
+import logging
+from typing import Dict, Iterable, List, Union
+
+from boefjes.job_models import NormalizerMeta
+from octopoes.models import OOI
+from octopoes.models.ooi.findings import RetireJSFindingType, RiskLevelSeverity
+
+logger = logging.getLogger(__name__)
+
+
+SEVERITY_SCORE_LOOKUP = {
+    RiskLevelSeverity.CRITICAL: 10.0,
+    RiskLevelSeverity.HIGH: 8.9,
+    RiskLevelSeverity.MEDIUM: 6.9,
+    RiskLevelSeverity.LOW: 3.9,
+    RiskLevelSeverity.RECOMMENDATION: 0.0,
+}
+
+
+def _hash_identifiers(identifiers: Dict[str, Union[str, List[str]]]) -> str:
+    pre_hash = ""
+    for identifier in identifiers.values():
+        pre_hash += "".join(identifier)
+    return hashlib.sha1(pre_hash.encode()).hexdigest()[:4]
+
+
+def _create_description(finding: dict) -> str:
+    if "summary" in finding["identifiers"]:
+        description = finding["identifiers"]["summary"] + ". More information at: "
+    else:
+        description = "No summary available. Find more information at: "
+
+    info = finding["info"]
+    description += ", ".join(info[:-1])
+    if len(info) > 1:
+        description += " or " + info[-1]
+    else:
+        description += info[0]
+
+    return description
+
+
+def run(normalizer_meta: NormalizerMeta, raw: Union[bytes, str]) -> Iterable[OOI]:
+    retirejs_finding_type_id = normalizer_meta.raw_data.boefje_meta.arguments["input"]["id"]
+    data = json.loads(raw)
+
+    _, name, hashed_id = retirejs_finding_type_id.split("-")
+
+    software = [
+        brand
+        for brand in data
+        if name == brand.lower().replace(" ", "").replace("_", "").replace("-", "").replace(".", "")
+    ][0]
+    issues = data[software]["vulnerabilities"]
+
+    finding = [issue for issue in issues if _hash_identifiers(issue["identifiers"]) == hashed_id]
+
+    if not finding:
+        return
+
+    risk_severity = RiskLevelSeverity(finding[0]["severity"].lower())
+    risk_score = SEVERITY_SCORE_LOOKUP[risk_severity]
+
+    yield RetireJSFindingType(
+        id=retirejs_finding_type_id,
+        description=_create_description(finding[0]),
+        risk_severity=risk_severity,
+        risk_score=risk_score,
+    )