Refactor FIPS & unify CLI actions (#275)

* Unifies CLIs into sine CLI
* Merge much of dataset functionality into `Dataset` baseclass
* Refactor `unittest` -> `pytest`
* Unify processing pipeline across datasets (common public functions)
* Unify and refactor processing of auxillary datasets (CPE, CVE, PP, Algorithms, ...)
* Upgrade Github Actions pipelines
* Allow for failure of some tests
* Refactor type hints

.flake8

@@ -1,6 +1,6 @@
 [flake8]
 max-line-length = 120
-exclude = 
+exclude =
     .git,
     __pycache__,
     build,
@@ -13,10 +13,13 @@ max-complexity = 10
 
 # Ignore complexity for CLI
 per-file-ignores =
-    cc_cli.py: C901, 
+    cc_cli.py: C901,
     fips_cli.py: C901
 
-ignore = 
-    E501,  # line length, should be handleded by black
-    W503,  # line break before binary operator, depracated
-    E203,  # whitespace before :, not PEP8 compliant
+ignore =
+    # line length, should be handleded by black
+    E501,
+    # line break before binary operator, depracated
+    W503,
+    # whitespace before :, not PEP8 compliant
+    E203,

.github/workflows/docs.yml

@@ -11,7 +11,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.8
       - name: Install external dependencies

.github/workflows/draft_release.yml

@@ -11,7 +11,7 @@ jobs:
     if: (github.repository == 'crocs-muni/sec-certs') && (github.ref == 'refs/heads/main')
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Release
         uses: softprops/action-gh-release@v1
         with:

.github/workflows/lint.yml

@@ -6,11 +6,11 @@ jobs:
   run-mypy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name : Setup python
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - name: Setup python
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: "3.9"
       - name: Install external dependencies
         run: sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
       - name: Install python dependencies
@@ -22,27 +22,45 @@ jobs:
   black:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: psf/black@stable
   isort:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.8
       - uses: isort/isort-action@master
         with:
-            requirementsFiles: "requirements/requirements.txt requirements/dev_requirements.txt"
+          requirementsFiles: "requirements/requirements.txt requirements/dev_requirements.txt"
+  pyupgrade:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+      - name: Install external dependencies
+        run: sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
+      - name: Install python dependencies
+        run: |
+          pip install -r requirements/requirements.txt
+          pip install -r requirements/dev_requirements.txt
+      - name: Run pyupgrade
+        run: pre-commit run pyupgrade --all-files
   flake8-lint:
     runs-on: ubuntu-latest
     name: Flake8
     steps:
       - name: Check out source repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Set up Python environment
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: "3.8"
       - name: flake8 Lint
-        uses: py-actions/flake8@v2
+        uses: py-actions/flake8@v2
+        with:
+          plugins: "flake8-future-annotations"

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: "3.8"
       - name: Install build dependencies

.github/workflows/tests.yml

@@ -7,27 +7,18 @@ env:
 
 jobs:
   run-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - name: Install Poppler 21.06
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y software-properties-common
-          sudo apt-get update
-          sudo add-apt-repository ppa:hlprasu/poppler
-          sudo apt-get update
-          sudo apt-get install -y build-essential libpoppler-cpp-dev pkg-config python3-dev
-      - uses: actions/checkout@v2
+      - name: Install Poppler
+        run: sudo apt-get install -y build-essential libpoppler-cpp-dev pkg-config python3-dev
+      - uses: actions/checkout@v3
       - name: Setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: "3.8"
       - name: Install python dependencies
-        run: pip install -r requirements/requirements.txt
-      - name: Download spacy language model
-        run: python -m spacy download en_core_web_sm
-      - name: Install pytest and package
         run: |
+          pip install -r requirements/requirements.txt
           pip install -r requirements/test_requirements.txt
       - name: Run tests
         run: pytest --cov=sec_certs tests

.pre-commit-config.yaml

@@ -1,24 +1,31 @@
 repos:
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.2.3
+    hooks:
+      - id: pyupgrade
+        args: ["--py38-plus"]
   - repo: https://github.com/psf/black
     rev: 22.6.0
     hooks:
       - id: black
-        args: ['--check', '--target-version', 'py38']
+        args: ["--check", "--target-version", "py38"]
   - repo: https://github.com/pycqa/isort
     rev: 5.10.1
     hooks:
       - id: isort
-        args: ['--check-only']
+        args: ["--check-only"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.982'
+    rev: "v0.982"
     hooks:
       - id: mypy
         additional_dependencies:
-          - 'numpy'
-          - 'types-PyYAML'
-          - 'types-python-dateutil'
-          - 'types-requests'
+          - "numpy"
+          - "types-PyYAML"
+          - "types-python-dateutil"
+          - "types-requests"
   - repo: https://github.com/pycqa/flake8
-    rev: '4.0.1'
+    rev: "4.0.1"
     hooks:
-    -   id: flake8
+      - id: flake8
+        additional_dependencies:
+          - "flake8-future-annotations"

CONTRIBUTING.md

@@ -49,6 +49,7 @@ All commits shall pass the lint pipeline of the following tools:
 - Black (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings)
 - isort (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings)
 - Flake8 (see [.flake8](https://github.com/crocs-muni/sec-certs/blob/main/.flake8) for settings)
+- PyUpgrade
 
 These tools can be installed via [dev_requirements.txt](https://github.com/crocs-muni/sec-certs/blob/main/dev_requirements.txt) You can use [pre-commit](https://pre-commit.com/) tool register git hook that will evalute these checks prior to any commit and abort the commit for you. Note that the pre-commit is not meant to automatically fix the issues, just warn you.
 
@@ -65,6 +66,7 @@ To ivoke the tools manually, you can, in the repository root, use:
 - Black: `black --check .` (without the flag to reformat)
 - isort: `isort --check-only .` (without the flag to actually fix the issue)
 - Flake8: `flake8 .`
+- PyUpgrade: `pyupgrade --py38-plus 'find ./sec_certs/ -name "*.py" -type f'`
 
 ## Documentation
 

Dockerfile

@@ -18,7 +18,6 @@ RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata
 RUN apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
 RUN apt-get install libqpdf-dev -y
 RUN apt-get install default-jdk -y
-RUN apt-get install graphviz -y
 RUN apt-get install tesseract-ocr tesseract-ocr-eng tesseract-ocr-deu tesseract-ocr-fra -y
 
 
@@ -51,9 +50,6 @@ RUN \
   pip3 install --no-cache notebook jupyterlab && \
   pip3 install -e .
 
-# Download spacy language model
-RUN python3 -m spacy download en_core_web_sm
-
 # #just to be sure that pdftotext is in $PATH
 ENV PATH /usr/bin/pdftotext:${PATH}