Skip to content
This repository has been archived by the owner on Feb 14, 2024. It is now read-only.

Commit

Permalink
Merge branch 'develop' into DJ-SQLupdate-update
Browse files Browse the repository at this point in the history
  • Loading branch information
DJensen94 committed Nov 8, 2021
2 parents c5d4151 + 46c05c1 commit 9c3feb5
Show file tree
Hide file tree
Showing 23 changed files with 1,829 additions and 117 deletions.
22 changes: 22 additions & 0 deletions .ansible-lint
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
# See https://ansible-lint.readthedocs.io/en/latest/configuring.html
# for a list of the configuration elements that can exist in this
# file.
enable_list:
# Useful checks that one must opt-into. See here for more details:
# https://ansible-lint.readthedocs.io/en/latest/rules.html
- fcqn-builtins
- no-log-password
- no-same-owner
exclude_paths:
# This exclusion is implicit, unless exclude_paths is defined
- .cache
# Seems wise to ignore this too
- .github
# ansible-lint doesn't like the role name in this playbook, but it's
# what molecule requires
- molecule/default/converge.yml
# These two are Molecule configuration files, not Ansible playbooks
- molecule/default/molecule-no-systemd.yml
- molecule/default/molecule-with-systemd.yml
use_default_rules: true
5 changes: 5 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,8 @@ updates:
directory: "/"
schedule:
interval: "weekly"

- package-ecosystem: "terraform"
directory: "/"
schedule:
interval: "weekly"
62 changes: 33 additions & 29 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,23 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: cisagov/setup-env-github-action@develop
- id: setup-env
uses: cisagov/setup-env-github-action@develop
- uses: actions/checkout@v2
- id: setup-python
uses: actions/setup-python@v2
with:
python-version: 3.9
# GO_VERSION and GOCACHE are used by the cache task, so the Go
# installation must happen before that.
# We need the Go version and Go cache location for the actions/cache step,
# so the Go installation must happen before that.
- uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Store installed Go version
id: go-version
run: |
echo "GO_VERSION="\
"$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \
>> $GITHUB_ENV
echo "::set-output name=version::"\
"$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')"
- name: Lookup Go cache directory
id: go-cache
run: |
Expand All @@ -41,9 +42,9 @@ jobs:
env:
BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
py${{ steps.setup-python.outputs.python-version }}-\
go${{ env.GO_VERSION }}-\
packer${{ env.PACKER_VERSION }}-\
tf${{ env.TERRAFORM_VERSION }}-"
go${{ steps.go-version.outputs.version }}-\
packer${{ steps.setup-env.outputs.packer-version }}-\
tf${{ steps.setup-env.outputs.terraform-version }}-"
with:
# Note that the .terraform directory IS NOT included in the
# cache because if we were caching, then we would need to use
Expand All @@ -69,6 +70,8 @@ jobs:
- name: Setup curl cache
run: mkdir -p ${{ env.CURL_CACHE_DIR }}
- name: Install Packer
env:
PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }}
run: |
PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip"
curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \
Expand All @@ -81,20 +84,17 @@ jobs:
sudo ln -s /opt/packer/packer /usr/local/bin/packer
- uses: hashicorp/setup-terraform@v1
with:
terraform_version: ${{ env.TERRAFORM_VERSION }}
terraform_version: ${{ steps.setup-env.outputs.terraform-version }}
- name: Install shfmt
run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION}
env:
PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt
PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }}
run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
- name: Install Terraform-docs
run: |
go install \
github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION}
- name: Find and initialize Terraform directories
run: |
for path in $(find . -not \( -type d -name ".terraform" -prune \) \
-type f -iname "*.tf" -exec dirname "{}" \; | sort -u); do \
echo "Initializing '$path'..."; \
terraform init -input=false -backend=false "$path"; \
done
env:
PACKAGE_URL: github.com/terraform-docs/terraform-docs
PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }}
run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
Expand All @@ -109,12 +109,14 @@ jobs:
test:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- 3.6
- 3.7
- 3.8
- 3.9
- "3.6"
- "3.7"
- "3.8"
- "3.9"
- "3.10"
steps:
- uses: actions/checkout@v2
- id: setup-python
Expand Down Expand Up @@ -194,12 +196,14 @@ jobs:
runs-on: ubuntu-latest
needs: [lint, test]
strategy:
fail-fast: false
matrix:
python-version:
- 3.6
- 3.7
- 3.8
- 3.9
- "3.6"
- "3.7"
- "3.8"
- "3.9"
- "3.10"
steps:
- uses: actions/checkout@v2
- id: setup-python
Expand Down
39 changes: 12 additions & 27 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,25 +31,25 @@ repos:

# Text file hooks
- repo: https://github.com/igorshubovych/markdownlint-cli
rev: v0.28.1
rev: v0.29.0
hooks:
- id: markdownlint
args:
- --config=.mdl_config.yaml
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v2.3.2
rev: v2.4.1
hooks:
- id: prettier
- repo: https://github.com/adrienverge/yamllint
rev: v1.26.2
rev: v1.26.3
hooks:
- id: yamllint
args:
- --strict

# pre-commit hooks
- repo: https://github.com/pre-commit/pre-commit
rev: v2.14.0
rev: v2.15.0
hooks:
- id: validate_manifest

Expand Down Expand Up @@ -91,7 +91,7 @@ repos:
name: bandit (everything else)
exclude: tests
- repo: https://github.com/psf/black
rev: 21.7b0
rev: 21.9b0
hooks:
- id: black
- repo: https://gitlab.com/pycqa/flake8
Expand All @@ -105,46 +105,31 @@ repos:
hooks:
- id: isort
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v0.910
rev: v0.910-1
hooks:
- id: mypy
additional_dependencies:
- types-PyYAML
- types-requests
- types-setuptools
- repo: https://github.com/asottile/pyupgrade
rev: v2.23.3
rev: v2.29.0
hooks:
- id: pyupgrade

# Ansible hooks
- repo: https://github.com/ansible-community/ansible-lint
# This is intentionally being held back because of issues in v5 per
# https://github.com/cisagov/skeleton-ansible-role/issues/69
rev: v4.3.7
rev: v5.2.1
hooks:
- id: ansible-lint
# files: molecule/default/playbook.yml

# Terraform hooks
- repo: https://github.com/antonbabenko/pre-commit-terraform
rev: v1.50.0
rev: v1.52.0
hooks:
- id: terraform_fmt
# There are ongoing issues with how this command works. This issue
# documents the core issue:
# https://github.com/hashicorp/terraform/issues/21408
# We have seen issues primarily with proxy providers and Terraform code
# that uses remote state. The PR
# https://github.com/hashicorp/terraform/pull/24887
# has been approved and is part of the 0.13 release to resolve the issue
# with remote states.
# The PR
# https://github.com/hashicorp/terraform/pull/24896
# is a proprosed fix to deal with `terraform validate` with proxy
# providers (among other configurations).
# We have decided to disable the terraform_validate hook until the issues
# above have been resolved, which we hope will be with the release of
# Terraform 0.13.
# - id: terraform_validate
- id: terraform_validate

# Docker hooks
- repo: https://github.com/IamTheFij/docker-pre-commit
Expand Down
106 changes: 70 additions & 36 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,68 +1,102 @@
# Posture and Exposure (P&E) Reports #
# Posture & Exposure Reports (P&E Reports) #

[![GitHub Build Status](https://github.com/cisagov/pe-reports/workflows/build/badge.svg)](https://github.com/cisagov/pe-reports/actions)
[![Coverage Status](https://coveralls.io/repos/github/cisagov/pe-reports/badge.svg?branch=develop)](https://coveralls.io/github/cisagov/pe-reports?branch=develop)
[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/pe-reports.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/pe-reports/alerts/)
[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/pe-reports.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/pe-reports/context:python)
[![Known Vulnerabilities](https://snyk.io/test/github/cisagov/pe-reports/develop/badge.svg)](https://snyk.io/test/github/cisagov/pe-reports)

This package is used to generate encrypted Posture and Exposure (P&E) PDF
reports using raw_data.xlsx files.
This package is used to generate and deliver CISA Posture & Exposure Reports
(P&E Reports). Reports are delivered by email and include an encrypted PDF
attachment with a series of embedded raw-data files of the collected materials.
The reports are delivered in a two step process. First the `pe_reports` module
collects the raw data and creates the encrypted PDFs. The `pe_mailer` then
securely delivers the content.

## Device Setup ##
Topics of interest include *Exposed Credentials, Domain Masquerading, Malware,
Inferred Vulnerabilities and the Dark Web*. The data collected for the reports
is gathered on the 1st and 15th of each month.

Install [LibreOffice](https://www.libreoffice.org/get-help/install-howto/macos/)
for PowerPoint to PDF conversion.
## Requirements ##

Install python 3
- [Python Environment](CONTRIBUTING.md#creating-the-python-virtual-environment)

(Optional) [Setting up your Mac](https://github.com/cisagov/development-guide/blob/develop/dev_envs/mac-env-setup.md)
- [cisagov MongoDB](https://github.com/cisagov/mongo-db-from-config)

- [cisagov AWS SES](https://github.com/cisagov/cool-dns-cyber.dhs.gov)

## Installation ##

Please see the
[Creating the Python virtual environment](CONTRIBUTING.md#creating-the-python-virtual-environment)
section of the [CONTRIBUTING](CONTRIBUTING.md) document for information about
setting up a Python virtual environment.
- `git clone https://github.com/cisagov/pe-reports.git`

- `pip install -e .`

Required configurations:
*You must have access to the cyhy database
Install [cisagov/mongo-db-from-config](https://github.com/cisagov/mongo-db-from-config)
and follow the instructions to create the yaml file.
The report generator will read `/secrets/database_creds.yml` by default if no
yaml filepath is provided.
## Create P&E Reports ##

To generate a P&E report:
- Configure [cisagov MongoDB connection](https://github.com/cisagov/mongo-db-from-config)

```console
python3 /pe-reports/src/pe_reports YYYY-MM-DD DATA_DIRECTORY OUTPUT_DIRECTORY [OPTIONS]
Usage:
pe-reports REPORT_DATE DATA_DIRECTORY OUTPUT_DIRECTORY [--db-creds-file=FILENAME] [--log-level=LEVEL]

Arguments:
REPORT_DATE Date of the report, format YYYY-MM-DD.
DATA_DIRECTORY The directory where the Excel data files are located.
Organized by owner.
OUTPUT_DIRECTORY The directory where the final PDF reports should be saved.
-c --db-creds-file=FILENAME A YAML file containing the Cyber
Hygiene database credentials.
[default: /secrets/database_creds.yml]
Options:
-h --help Show this message.
-v --version Show version information.
--log-level=LEVEL If specified, then the log level will be set to
the specified value. Valid values are "debug", "info",
"warning", "error", and "critical". [default: info]
```

## Making Changes ##

To change any general report format/standard visuals edit
`/src/data/shell/pe_shell.pptx`
## Deliver P&E Reports ##

To make any style changes, edit `/src/pe_reports/stylesheet.py`
- Configure [cisagov MongoDB connection](https://github.com/cisagov/mongo-db-from-config)

To change metrics, edit `/src/pe_reports/report_metrics.py`
- Load an AWS profile that assumes [this role](https://github.com/cisagov/cool-dns-cyber.dhs.gov/blob/develop/sessendemail_rolerole.tf#L33-L39)

To change page setups/graphs, edit `/src/pe_reports/pages.py`
```console
Usage:
pe-mailer [--pe-report-dir=DIRECTORY] [--db-creds-file=FILENAME] [--log-level=LEVEL]

Arguments:
-p --pe-report-dir=DIRECTORY Directory containing the pe-reports output.
-c --db-creds-file=FILENAME A YAML file containing the Cyber
Hygiene database credentials.
[default: /secrets/database_creds.yml]
Options:
-h --help Show this message.
-v --version Show version information.
-s --summary-to=EMAILS A comma-separated list of email addresses
to which the summary statistics should be
sent at the end of the run. If not
specified then no summary will be sent.
-t --test_emails=EMAILS A comma-separated list of email addresses
to which to test email send process. If not
specified then no test will be sent.
-l --log-level=LEVEL If specified, then the log level will be set to
the specified value. Valid values are "debug", "info",
"warning", "error", and "critical". [default: info]
```

## Contributing ##

We welcome contributions! Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for
details.
We welcome contributions! Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for details.

## License ##

This project is in the worldwide [public domain](LICENSE).

This project is in the public domain within the United States, and
copyright and related rights in the work worldwide are waived through
the [CC0 1.0 Universal public domain
dedication](https://creativecommons.org/publicdomain/zero/1.0/).
This project is in the public domain within the United States, and copyright
and related rights in the work worldwide are waived through the
[CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).

All contributions to this project will be released under the CC0
dedication. By submitting a pull request, you are agreeing to comply
with this waiver of copyright interest.
All contributions to this project will be released under the CC0 dedication.
By submitting a pull request, you are agreeing to comply with this waiver
of copyright interest.

0 comments on commit 9c3feb5

Please sign in to comment.