Makefile

SHELL=/bin/bash

# settings
app_name = websecmap

# configure virtualenv to be created in OS specific cache directory
ifeq ($(shell uname -s),Darwin)
# macOS cache location
CACHEDIR ?= ~/Library/Caches
else
# User customized cache location or Linux default
XDG_CACHE_HOME ?= ~/.cache
CACHEDIR ?= ${XDG_CACHE_HOME}
endif
VIRTUAL_ENV ?= ${CACHEDIR}/virtualenvs/$(notdir ${PWD})

# variables for environment
bin = ${VIRTUAL_ENV}/bin
env = env PATH=${bin}:$$PATH

# shortcuts for common used binaries
python = ${bin}/python
pip = ${bin}/pip
pip-compile = ${bin}/pip-compile
pip-sync = ${bin}/pip-sync

# application binary
app = ${bin}/${app_name}

ifeq (${MAKECMDGOALS},)
$(info Virtualenv path: ${VIRTUAL_ENV})
$(info Run App: ${env} ${app})
$(info )
$(info Run `make help` for available commands or use tab-completion.)
$(info )
$(info Running complete setup and test of development environment now.)
$(info )
endif

pysrcdirs = ${app_name}/ tests/
pysrc = $(shell find ${pysrcdirs} -name \*.py)
shsrc = $(shell find * ! -path vendor\* -name \*.sh)

.PHONY: test check setup run fix autofix clean mrproper test_integration

# default action to run
all: check test setup

# setup entire dev environment
setup: ${app}	## setup development environment and application
	@test \! -z "$$PS1" || (echo -ne "Development environment is tested and ready."; \
	if command -v websecmap &>/dev/null;then \
		echo -e " Development shell is activated."; \
	else \
		echo -e "\n\nTo activate development shell run:"; \
		echo -e "\n\t. ${VIRTUAL_ENV}/bin/activate$$([[ "$$SHELL" =~ "fish" ]] && echo .fish)\n"; \
		echo -e "Or refer to Direnv in README.md for automatic activation."; \
	fi)

# install application and all its (python) dependencies
${app}: ${VIRTUAL_ENV}/.requirements.installed | ${pip}
	# install project and its dependencies
	${python} setup.py develop --no-deps
	@test -f $@ && touch $@  # update timestamp, do not create empty file

test: .make.test	## run test suite
.make.test: ${pysrc} ${app}
	# run testsuite
	# #7040: -k no longer matches against the names of the directories outside the test session root.
	# #7122: Expressions given to the -m and -k options are no longer evaluated using Python’s eval().
	# The format supports or, and, not, parenthesis and general identifiers to match against.
	# Python constants, keywords or other operators are no longer evaluated differently.
	DJANGO_SETTINGS_MODULE=${app_name}.settings ${env} coverage run --include '${app_name}/*' --omit '*migrations*' \
		-m pytest -vv -ra -k 'not integration_celery and not integration_scanners and not system' ${testargs}
	# generate coverage
	${env} coverage report
	# and pretty html
	${env} coverage html
	# ensure no model updates are commited without migrations
	${env} ${app} makemigrations --check
	@touch $@  # update timestamp

check: .make.check.py .make.check.sh  ## code quality checks
.make.check.py: ${pysrc} ${app}
	# check code quality
	${env} pylama ${pysrcdirs} --skip "**/migrations/*"
	# check formatting
	${env} black --line-length 120 --check ${pysrcdirs}
	@touch $@  # update timestamp

.make.check.sh: ${shsrc}
	# shell script checks (if installed)
	if command -v shellcheck &>/dev/null;then ${env} shellcheck --version; ${env} shellcheck ${shsrc}; fi
	@touch $@  # update timestamp

autofix fix: .make.fix  ## automatic fix of trivial code quality issues
.make.fix: ${pysrc} ${app}
	# remove unused imports
	${env} autoflake -ri --remove-all-unused-imports ${pysrcdirs}
	# autoformat code
	# -q is used because a few files cannot be formatted with black, and will raise errors
	${env} black --line-length 120 -q ${pysrcdirs}
	# replaced by black: fix trivial pep8 style issues
	# replaced by black: ${env} autopep8 -ri ${pysrcdirs}
	# replaced by black: sort imports
	# replaced by black: ${env} isort -rc ${pysrcdirs}
	# do a check after autofixing to show remaining problems
	${MAKE} check
	@touch $@  # update timestamp

run: ${app}  ## run complete application stack (frontend, worker, broker)
	# start server (this can take a while)
	DEBUG=1 NETWORK_SUPPORTS_IPV6=1 ${env} ${app} devserver

audit:
	${python} -m bandit -c bandit.yaml -r websecmap

run_no_backend: ${app}  ## run application stack without broker/worker
	# start server (this can take a while)
	DEBUG=1 NETWORK_SUPPORTS_IPV6=1 ${env} ${app} devserver --no-backend

run-frontend: ${app}  ## only run frontend component
	DEBUG=1 DJANGO_SETTINGS_MODULE=${app_name}.settings NETWORK_SUPPORTS_IPV6=1 ${env} ${app} runserver

run-nonlocal-frontend: ${app}  ## only run frontend component
	DEBUG=1 NETWORK_SUPPORTS_IPV6=1 ${env} ${app} runserver 0.0.0.0:8000

app: ${app}  ## perform arbitrary app commands
    # make app cmd="help"
    # make app cmd="report -y municipality"
    # make app cmd="makemigrations"
    # make app cmd="migrate"
	DEBUG=1 NETWORK_SUPPORTS_IPV6=1 ${env} ${app} ${cmd}

run-worker: ${app}  ## only run worker component
	DEBUG=1 NETWORK_SUPPORTS_IPV6=1 ${env} ${app} celery worker -ldebug

run-broker:  ## only run broker
	docker run --rm --name=redis -p 6379:6379 redis

testcase: ${app}
	# run specific testcase
	# example: make testcase case=test_openstreetmaps
	DJANGO_SETTINGS_MODULE=${app_name}.settings DB_NAME=test.sqlite3 ${env} pytest -vvv --log-cli-level=10 -k ${case}

test_integration: ${app}
	# run integration tests
	# see docs at make test why the -k option is so explicit (no globbing anymore).
	${env} DJANGO_SETTINGS_MODULE=${app_name}.settings DB_NAME=test.sqlite3 \
		${env} pytest -vv -ra -k 'integration_celery or integration_scanners' ${testargs}

test_system:
	# run system tests
	${env} pytest -vv --setup-show tests/system ${testargs}

test_datasets: ${app}
	${env} /bin/sh -ec "find websecmap -path '*/fixtures/*.json' -print0 | \
		xargs -0n1 basename -s .yaml | uniq | \
		xargs -n1 ${app} test_dataset"

test_deterministic: | ${VIRTUAL_ENV}
	${env} /bin/bash tools/compare_differences.sh HEAD HEAD tools/show_ratings.sh testdata

test_mysql:
	docker run --name mysql -d --rm -p 3306:3306 \
		-e MYSQL_ROOT_PASSWORD=failmap \
		-e MYSQL_DATABASE=failmap \
		-e MYSQL_USER=failmap \
		-e MYSQL_PASSWORD=failmap \
		-v $$PWD/tests/etc/mysql-minimal-memory.cnf:/etc/mysql/conf.d/mysql.cnf \
		mysql:5.6
	DJANGO_DATABASE=production DB_USER=root DB_HOST=127.0.0.1 \
		$(MAKE) test; e=$$?; docker stop mysql; exit $$e

test_postgres:
	docker run --name postgres -d --rm -p 5432:5432 \
		-e POSTGRES_DB=failmap \
		-e POSTGRES_USER=root \
		-e POSTGRES_PASSWORD=failmap \
		postgres:9.5
	DJANGO_DATABASE=production DB_ENGINE=postgresql_psycopg2 DB_USER=root DB_HOST=127.0.0.1 \
		$(MAKE) test; e=$$?; docker stop postgres; exit $$e

clean:  ## cleanup build artifacts, caches, databases, etc.
	# remove python cache files
	-find * -name __pycache__ -print0 | xargs -0 rm -rf
	# remove state files
	-rm -f .make.*
	# remove test artifacts
	-rm -rf .pytest_cache htmlcov/
	# remove build artifacts
	-rm -rf *.egg-info dist/ pip-wheel-metadata/
	# remove runtime state files
	-rm -rf *.sqlite3

clean_virtualenv:  ## cleanup virtualenv and installed app/dependencies
	# remove virtualenv
	-rm -fr ${VIRTUAL_ENV}/

mrproper: clean clean_virtualenv ## thorough cleanup, also removes virtualenv

${VIRTUAL_ENV}/.requirements.installed: requirements.txt requirements-dev.txt| ${pip-sync}
	${env} ${pip-sync} $^
	@touch $@  # update timestamp

requirements = requirements.txt requirements-dev.txt requirements-deploy.txt
requirements: ${requirements}

# perform 'pip freeze' on first class requirements in .in files.
${requirements}: %.txt: %.in | ${pip-compile}
	${env} ${pip-compile} ${pip_compile_args} --output-file $@ $<

pip-sync:
	# synchronizes the .venv with the state of requirements.txt
	python3 -m piptools sync requirements.txt requirements-dev.txt

update_requirements: pip_compile_args=--upgrade
update_requirements: _mark_outdated requirements.txt requirements-dev.txt requirements-deploy.txt _commit_update

_mark_outdated:
	touch requirements*.in

_commit_update: requirements.txt
	git add requirements*.txt requirements*.in
	git commit -m "Updated requirements."

${pip-compile} ${pip-sync}: | ${pip}
	${env} ${pip} install --quiet pip-tools

${python} ${pip}:
	@if ! command -v python3 &>/dev/null;then \
		echo "Python 3 is not available. Please refer to installation instructions in README.md"; \
	fi
	# create virtualenv
	${env} python3 -mvenv ${VIRTUAL_ENV}

# utility
help:           ## Show this help.
	@IFS=$$'\n' ; \
	help_lines=(`fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##/:/'`); \
	printf "\nRun \`make\` with any of the targets below to reach the desired target state.\n" ; \
	printf "\nTargets are complementary. Eg: the \`run\` target requires \`setup\` which is automatically executed.\n\n" ; \
	printf "%-30s %s\n" "target" "help" ; \
	printf "%-30s %s\n" "------" "----" ; \
	for help_line in $${help_lines[@]}; do \
		IFS=$$':' ; \
		help_split=($$help_line) ; \
		help_command=`echo $${help_split[0]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \
		help_info=`echo $${help_split[2]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \
		printf '\033[36m'; \
		printf "%-30s %s" $$help_command ; \
		printf '\033[0m'; \
		printf "%s\n" $$help_info; \
	done

check-commit: fix test