Merge c118e7e into 9697906

.coveragerc

@@ -2,7 +2,11 @@
 # https://coverage.readthedocs.io/en/latest/config.html
 
 [run]
-source = src/example
+source =
+    src/md_from_template
+    src/mdyml
+    src/yml
+    src/ymlmd
 omit =
 branch = true
 

.github/workflows/build.yml

@@ -23,7 +23,7 @@ jobs:
       - id: setup-python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: "3.10"
       # We need the Go version and Go cache location for the actions/cache step,
       # so the Go installation must happen before that.
       - uses: actions/setup-go@v2
@@ -112,10 +112,6 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - "3.6"
-          - "3.7"
-          - "3.8"
-          - "3.9"
           - "3.10"
     steps:
       - uses: actions/checkout@v2
@@ -165,7 +161,7 @@ jobs:
       - id: setup-python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: "3.10"
       - uses: actions/cache@v2
         env:
           BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
@@ -199,10 +195,6 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - "3.6"
-          - "3.7"
-          - "3.8"
-          - "3.9"
           - "3.10"
     steps:
       - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 default_language_version:
   # force all unspecified python hooks to run python3
-  python: python3
+  python: python3.10
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -91,9 +91,11 @@ repos:
         name: bandit (everything else)
         exclude: tests
   - repo: https://github.com/psf/black
-    rev: 21.9b0
+    rev: 21.12b0
     hooks:
       - id: black
+        args:
+          - --target-version=py310
   - repo: https://gitlab.com/pycqa/flake8
     rev: 3.9.2
     hooks:
@@ -104,12 +106,17 @@ repos:
     rev: 5.9.3
     hooks:
       - id: isort
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
-    hooks:
-      - id: mypy
-        additional_dependencies:
-          - types-setuptools
+  # mypy does not currently support the match statement added in 3.10. Please
+  # check the status of https://github.com/python/mypy/pull/10191 for the
+  # current status of support.
+  # - repo: https://github.com/pre-commit/mirrors-mypy
+  #   rev: v0.920
+  #   hooks:
+  #     - id: mypy
+  #       additional_dependencies:
+  #         - types-dateparser
+  #         - types-pytz
+  #         - types-PyYAML
   - repo: https://github.com/asottile/pyupgrade
     rev: v2.29.0
     hooks:

CONTRIBUTING.md

@@ -15,7 +15,7 @@ all of which should be in this repository.
 
 If you want to report a bug or request a new feature, the most direct
 method is to [create an
-issue](https://github.com/cisagov/md-table-to-yml/issues) in
+issue](https://github.com/cisagov/log4j-md-yml/issues) in
 this repository.  We recommend that you first search through existing
 issues (both open and closed) to check if your particular issue has
 already been reported.  If it has then you might want to add a comment
@@ -25,7 +25,7 @@ one.
 ## Pull requests ##
 
 If you choose to [submit a pull
-request](https://github.com/cisagov/md-table-to-yml/pulls),
+request](https://github.com/cisagov/log4j-md-yml/pulls),
 you will notice that our continuous integration (CI) system runs a
 fairly extensive set of linters, syntax checkers, system, and unit tests.
 Your pull request may fail these checks, and that's OK.  If you want
@@ -135,9 +135,9 @@ can create and configure the Python virtual environment with these
 commands:
 
 ```console
-cd md-table-to-yml
-pyenv virtualenv <python_version_to_use> md-table-to-yml
-pyenv local md-table-to-yml
+cd log4j-md-yml
+pyenv virtualenv <python_version_to_use> log4j-md-yml
+pyenv local log4j-md-yml
 pip install --requirement requirements-dev.txt
 ```
 

README.md

@@ -1,25 +1,77 @@
-# md-table-to-yml #
-
-[![GitHub Build Status](https://github.com/cisagov/md-table-to-yml/workflows/build/badge.svg)](https://github.com/cisagov/md-table-to-yml/actions)
-[![Coverage Status](https://coveralls.io/repos/github/cisagov/md-table-to-yml/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/md-table-to-yml?branch=develop)
-[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/md-table-to-yml.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/md-table-to-yml/alerts/)
-[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/md-table-to-yml.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/md-table-to-yml/context:python)
-[![Known Vulnerabilities](https://snyk.io/test/github/cisagov/md-table-to-yml/develop/badge.svg)](https://snyk.io/test/github/cisagov/md-table-to-yml)
-
-This is a generic skeleton project that can be used to quickly get a
-new [cisagov](https://github.com/cisagov) Python library GitHub
-project started.  This skeleton project contains [licensing
-information](LICENSE), as well as
-[pre-commit hooks](https://pre-commit.com) and
-[GitHub Actions](https://github.com/features/actions) configurations
-appropriate for a Python library project.
-
-## New Repositories from a Skeleton ##
-
-Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup)
-for step-by-step instructions on how to start a new repository from
-a skeleton. This will save you time and effort when configuring a
-new repository!
+# log4j-md-yml #
+
+[![GitHub Build Status](https://github.com/cisagov/log4j-md-yml/workflows/build/badge.svg)](https://github.com/cisagov/log4j-md-yml/actions)
+[![Coverage Status](https://coveralls.io/repos/github/cisagov/log4j-md-yml/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/log4j-md-yml?branch=develop)
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/log4j-md-yml.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/log4j-md-yml/alerts/)
+[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/log4j-md-yml.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/log4j-md-yml/context:python)
+[![Known Vulnerabilities](https://snyk.io/test/github/cisagov/log4j-md-yml/develop/badge.svg)](https://snyk.io/test/github/cisagov/log4j-md-yml)
+
+This repository contains Python code to:
+
+1. (Work in Progress) Translate the Markdown tables of vulnerable software
+   from [NCSC-NL/log4shell](https://github.com/NCSC-NL/log4shell) into a
+   common YAML format: [`convert-nscs-nl`](src/mdyml/convert_ncsc_nl.py)
+1. Merge the YAML from the previous step with the YAML from
+   [cisagov/log4j-affected-db](https://github.com/cisagov/log4j-affected-db)
+   into one grand YAML file: [`normalize-yml`](src/yml/normalize_yml.py)
+1. Generate a Markdown table from the YAML output of the previous step:
+   [`yml2md`](src/ymlmd/yml2md.py)
+1. Generate Markdown based on a Markdown template file and a file containing
+   a Markdown table such as the output of the previous step:
+   [`md-from-template`](src/md_from_template/md_from_template.py)
+
+## Common YAML format ##
+
+The common YAML format looks like this:
+
+```yaml
+---
+version: '1.0'
+software:
+  -  cves:
+      - affected_versions:
+          - 1.0
+          - 1.1
+        cve: cve-2021-44228
+        fixed_versions:
+          - 1.2
+        investigated: true
+        unaffected_versions: []
+    notes: Blah blah blah
+    product: ProductA
+    references:
+      - https://www.reddit.com/r/Vendor1/comments/abcdef/log4j
+    reporter: cisagov
+    vendor: Vendor1
+    vendor_links:
+      - https://vendor1.com/discussion/comment/622612/#Comment_622612
+    last_updated: "2021-12-09"
+  ⋮
+...
+```
+
+The fields and their descriptions are as follows:
+
+| Field  | Description |
+| ------ | ----------- |
+| `cves` | A list of dictionaries containing a CVE ID together with vulnerability information about the product specific to that CVE. |
+| `last_updated` | The date of the last update for this product in ISO 8601 timestamp format. |
+| `notes` | A free-form text field for additional notes. |
+| `product` | The name of the software product. |
+| `references` | A list of links to non-vendor sources concerning the software product and the log4j vulnerabilities. |
+| `reporter` | The entity reporting information about the software product. |
+| `vendor` | The name of the software vendor. |
+| `vendor_links` | A list of links to the vendor's website concerning the software product and the log4j vulnerabilities. |
+
+The subfields in the `cves` entries are as follows:
+
+| `cves` subfield  | Description |
+| ---------------- | ----------- |
+| `cve` | The [CVE ID](https://www.cve.org/) of the particular log4j vulnerability.  The only valid value is [`cve-2021-44228`](https://www.cve.org/CVERecord?id=CVE-2021-44228). |
+| `affected_versions` | A list of the versions of the product that are vulnerable to the particular CVE. |
+| `fixed_versions` | A list of the versions of the product that are patched and therefore unaffected by the particular CVE. |
+| `investigated` | A Boolean value indicating whether or not the product's vulnerability to the particular CVE has been investigated. |
+| `unaffected_versions` | A list of the versions of the product that are completely unaffected by the particular CVE. |
 
 ## Contributing ##
 

bump_version.sh

@@ -6,7 +6,7 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
-VERSION_FILE=src/example/_version.py
+VERSION_FILE=src/_log4j_md_yml_version.py
 
 HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)"
 

setup.py

@@ -1,5 +1,5 @@
 """
-This is the setup module for the example project.
+This is the setup module for the log4j-md-yml project.
 
 Based on:
 
@@ -42,19 +42,19 @@ def get_version(version_file):
 
 
 setup(
-    name="example",
+    name="log4j-md-yml",
     # Versions should comply with PEP440
-    version=get_version("src/example/_version.py"),
-    description="Example Python library",
+    version=get_version("src/_log4j_md_yml_version.py"),
+    description="log4j-md-yml Python library",
     long_description=readme(),
     long_description_content_type="text/markdown",
     # Landing page for CISA's cybersecurity mission
     url="https://www.cisa.gov/cybersecurity",
     # Additional URLs for this project per
     # https://packaging.python.org/guides/distributing-packages-using-setuptools/#project-urls
     project_urls={
-        "Source": "https://github.com/cisagov/md-table-to-yml",
-        "Tracker": "https://github.com/cisagov/md-table-to-yml/issues",
+        "Source": "https://github.com/cisagov/log4j-md-yml",
+        "Tracker": "https://github.com/cisagov/log4j-md-yml/issues",
     },
     # Author details
     author="Cybersecurity and Infrastructure Security Agency",
@@ -74,20 +74,23 @@ def get_version(version_file):
         # Specify the Python versions you support here. In particular, ensure
         # that you indicate whether you support Python 2, Python 3 or both.
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
-    python_requires=">=3.6",
+    python_requires=">=3.10",
     # What does your project relate to?
-    keywords="skeleton",
+    keywords="log4j markdown yaml",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
-    package_data={"example": ["data/*.txt"]},
     py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")],
-    include_package_data=True,
-    install_requires=["docopt", "schema", "setuptools >= 24.2.0"],
+    install_requires=[
+        "chevron",
+        "dateparser",
+        "docopt",
+        "pyyaml",
+        "ruamel.yaml",
+        "schema",
+        "setuptools >= 24.2.0",
+    ],
     extras_require={
         "test": [
             "coverage",
@@ -103,6 +106,14 @@ def get_version(version_file):
             "pytest",
         ]
     },
-    # Conveniently allows one to run the CLI tool as `example`
-    entry_points={"console_scripts": ["example = example.example:main"]},
+    # Conveniently allows one to run the CLI tools
+    entry_points={
+        "console_scripts": [
+            "convert-cisagov = mdyml.convert_cisagov:main",
+            "convert-nscs-nl = mdyml.convert_ncsc_nl:main",
+            "md-from-template = md_from_template.md_from_template:main",
+            "normalize-yml = yml.normalize_yml:main",
+            "yml2md = ymlmd.yml2md:main",
+        ]
+    },
 )

src/example/_version.py → src/_log4j_md_yml_version.py

@@ -1,2 +1,2 @@
 """This file defines the version of this module."""
-__version__ = "0.0.1"
+__version__ = "1.0.0"