Merge pull request #521 from cisagov/lineage/skeleton

⚠️ CONFLICT! Lineage pull request for: skeleton

.github/CODEOWNERS

@@ -3,8 +3,8 @@
 # These owners will be the default owners for everything in the
 # repo. Unless a later match takes precedence, these owners will be
 # requested for review when someone opens a pull request.
-* @aloftus23 @cduhn17 @dav3r @DJensen94 @elr64 @felddy @jsf9k @mcdonnnj @schmelz-ctr @stewartl97
+* @aloftus23 @cduhn17 @dav3r @DJensen94 @felddy @jasonodoom @jsf9k @mcdonnnj @stewartl97
 
 # These folks own any files in the .github directory at the root of
 # the repository and any of its subdirectories.
-/.github/ @dav3r @felddy @jsf9k @mcdonnnj
+/.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj

.github/workflows/build.yml

@@ -23,11 +23,25 @@ jobs:
       - id: setup-python
         uses: actions/setup-python@v4
         with:
+          # Temporarily hold back support for Python 3.11 due to the
+          # version pin for the PyMuPDF package. Pre-built wheels for
+          # Python 3.11 were not made available until the 1.21.0
+          # release and building the PyMuPDF package from source
+          # proves problematic due to dependencies. Please see the
+          # following issue for more information:
+          # https://github.com/cisagov/pe-reports/issues/441
+          #
+          # It's also the case that we cannot build the wheel for
+          # reportlab under Python 3.11 because its C code accesses
+          # the C struct members of PyFrameObject directly, and these
+          # struct members are no longer public as of Python 3.11.
+          # See here for more details:
+          # https://docs.python.org/3/whatsnew/3.11.html#pyframeobject-3-11-hiding
           python-version: "3.10"
       # We need the Go version and Go cache location for the actions/cache step,
       # so the Go installation must happen before that.
       - id: setup-go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: "1.19"
       - name: Lookup Go cache directory
@@ -91,9 +105,14 @@ jobs:
           PACKAGE_URL: github.com/terraform-docs/terraform-docs
           PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }}
         run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
+      # Install some necessary packages for building wheels
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: libmupdf-dev
+          version: 1.3.0
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade pip setuptools wheel
           pip install --upgrade --requirement requirements-test.txt
       - name: Set up pre-commit hook environments
         run: pre-commit install-hooks
@@ -145,6 +164,11 @@ jobs:
             ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
+      # Install some necessary packages for building wheels
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: libmupdf-dev
+          version: 1.3.0
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -189,6 +213,11 @@ jobs:
             ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
+      # Install some necessary packages for building wheels
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: libmupdf-dev
+          version: 1.3.0
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -245,6 +274,11 @@ jobs:
             ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
+      # Install some necessary packages for building wheels
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: libmupdf-dev
+          version: 1.3.0
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel

.pre-commit-config.yaml

@@ -37,26 +37,26 @@ repos:
         args:
           - --config=.mdl_config.yaml
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.0-alpha.4
+    rev: v3.0.0-alpha.6
     hooks:
       - id: prettier
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.29.0
+    rev: v1.30.0
     hooks:
       - id: yamllint
         args:
           - --strict
 
   # GitHub Actions hooks
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.21.0
+    rev: 0.22.0
     hooks:
       - id: check-github-actions
       - id: check-github-workflows
 
   # pre-commit hooks
   - repo: https://github.com/pre-commit/pre-commit
-    rev: v3.0.2
+    rev: v3.2.1
     hooks:
       - id: validate_manifest
 
@@ -83,7 +83,7 @@ repos:
   # Python hooks
   # Run bandit on the "tests" tree with a configuration
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.4
+    rev: 1.7.5
     hooks:
       - id: bandit
         name: bandit (tests tree)
@@ -98,7 +98,7 @@ repos:
         name: bandit (everything else)
         exclude: tests
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.1.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/flake8
@@ -112,7 +112,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.1.1
     hooks:
       - id: mypy
         additional_dependencies:
@@ -142,7 +142,7 @@ repos:
 
   # Terraform hooks
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.77.0
+    rev: v1.77.1
     hooks:
       - id: terraform_fmt
       - id: terraform_validate

src/pe_reports/helpers/bulletin/bulletin_generator.py

@@ -105,7 +105,6 @@ def generate_cybersix_bulletin(
     resp = get_post(id)
 
     for post in resp["intel_items"]:
-
         outputText = template.render(
             post,
             user_provided_content=user_text,

src/pe_reports/stakeholder/views.py

@@ -136,7 +136,6 @@ def setStakeholder(customer):
         conn = psycopg2.connect(**params)
 
         if conn:
-
             LOGGER.info(
                 "There was a connection made to "
                 "the database and the query was executed "
@@ -171,7 +170,6 @@ def setCustRootDomain(customer, rootdomain, orgUUID):
         conn = psycopg2.connect(**params)
 
         if conn:
-
             LOGGER.info(
                 "There was a connection made to "
                 "the database and the query was executed "
@@ -204,15 +202,13 @@ def setCustSubDomain(subdomain, rootUUID, rootname):
     global conn, cursor
 
     try:
-
         LOGGER.info("Starting insert into database...")
 
         params = config()
 
         conn = psycopg2.connect(**params)
 
         if conn:
-
             LOGGER.info(
                 "There was a connection made to "
                 "the database and the query to "
@@ -260,7 +256,6 @@ def setCustomerExternalCSG(
         conn = psycopg2.connect(**params)
 
         if conn:
-
             LOGGER.info(
                 "There was a connection made to"
                 " the database and the query was executed "
@@ -322,9 +317,7 @@ def getSubdomain(domain):
 
     subisolated = ""
     for sub in subdomains:
-
         if sub != f"www.{domain}":
-
             LOGGER.info(sub)
             subisolated = sub.rsplit(".")[:-2]
             LOGGER.info(
@@ -460,7 +453,6 @@ def setOrganizationUsers(org_id):
             or userrole == role2
             and user_id != id_role1
         ):
-
             url = (
                 f"https://api.cybersixgill.com/multi-tenant/organization/"
                 f"{org_id}/user/{user_id}?role_id={userrole}"
@@ -546,7 +538,6 @@ def stakeholder():
         allValidIP = getallsubdomainIPS(custRootDomainValue)
 
         try:
-
             if cust not in allDomain.values():
                 flash(f"You successfully submitted a new customer {cust} ", "success")
 

src/pe_source/dnstwistscript.py

@@ -176,7 +176,6 @@ def run_dnstwist(orgs_list):
 
         # Only run on orgs in the org list
         if pe_org_id in orgs_list or orgs_list == "all":
-
             LOGGER.info("Running dnstwist on %s", pe_org_id)
 
             """Collect dnstwist data from Crossfeed"""