-
Notifications
You must be signed in to change notification settings - Fork 2
/
Makefile
251 lines (193 loc) · 8.16 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
# Makefile for Planttracer web application
# - Creates CI/CD environment in GitHub
# - Manages deployemnt to AWS Linux
# - Updated to handle virtual environment.
PYLINT_FILES=$(shell /bin/ls *.py | grep -v bottle.py | grep -v app_wsgi.py)
PYLINT_THRESHOLD=9.5
TS_FILES := $(wildcard *.ts */*.ts)
JS_FILES := $(TS_FILES:.ts=.js)
################################################################
# Manage the virtual environment
ACTIVATE = . venv/bin/activate
REQ = venv/pyvenv.cfg
PY=python3.11
PYTHON=$(ACTIVATE) ; $(PY)
PIP_INSTALL=$(PYTHON) -m pip install --no-warn-script-location
venv/pyvenv.cfg:
$(PY) -m venv venv
venv:
$(PY) -m venv venv
################################################################
#
# By default, PYLINT generates an error if your code does not rank 10.0.
# This makes us tolerant of minor problems.
all:
@echo verify syntax and then restart
make pylint
check:
make pylint
make pytest
################################################################
## Program testing
##
## Static Analysis
pylint: $(REQ)
$(PYTHON) -m pylint --rcfile .pylintrc --fail-under=$(PYLINT_THRESHOLD) --verbose $(PYLINT_FILES)
mypy:
mypy --show-error-codes --pretty --ignore-missing-imports --strict .
black:
black --line-length 127 .
black-check:
black --line-length 127 . --check
@echo "If this fails, simply run: make black"
isort:
isort . --profile=black
isort-check:
isort --check . --profile=black
flake:
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
flake8 . --count --exit-zero --max-complexity=55 --max-line-length=127 --statistics --ignore F403,F405,E203,E231,E252,W503
#
# In the tests below, we always test the database connectivity first
# It makes no sense to run the tests otherwise
##
## Dynamic Analysis
pytest-db: $(REQ)
$(PYTHON) -m pytest --log-cli-level=DEBUG tests/dbreader_test.py
@echo dbreader_test is successful
pytest: $(REQ)
$(PYTHON) -m pytest --log-cli-level=DEBUG tests/dbreader_test.py
@echo dbreader_test is successful
$(PYTHON) -m pytest -v --log-cli-level=INFO .
# Set these during development to speed testing of the one function you care about:
TEST1MODULE=tests/movie_test.py
TEST1FUNCTION="-k test_new_movie"
pytest1:
$(PYTHON) -m pytest --log-cli-level=DEBUG tests/dbreader_test.py
@echo dbreader_test is successful
$(PYTHON) -m pytest -v --log-cli-level=DEBUG --maxfail=1 $(TEST1MODULE) $(TEST1FUNCTION)
pytest-selenium:
$(PYTHON) -m pytest -v --log-cli-level=INFO tests/sitetitle_test.py
pytest-debug:
$(PYTHON) -m pytest --log-cli-level=DEBUG tests/dbreader_test.py
@echo dbreader_test is successful
$(PYTHON) -m pytest -v --log-cli-level=DEBUG
pytest-debug1:
@echo run in debug mode but stop on first error
$(PYTHON) -m pytest --log-cli-level=DEBUG --maxfail=1 tests/dbreader_test.py
@echo dbreader_test is successful
$(PYTHON) -m pytest -v --log-cli-level=DEBUG -k test_new_movie tests/movie_test.py
pytest-app-framework:
@echo validate app framework
$(PYTHON) -m pytest -x --log-cli-level=DEBUG tests/app_test.py -k test_templates
pytest-quiet:
@echo quietly make pytest and stop at the firt error
$(PYTHON) -m pytest --log-cli-level=ERROR tests/dbreader_test.py
@echo dbreader_test is successful
$(PYTHON) -m pytest --log-cli-level=ERROR
test-schema-upgrade:
$(PYTHON) dbmaint.py --rootconfig etc/mysql-root-localhost.ini --dropdb test_db1 || echo database does not exist
$(PYTHON) dbmaint.py --rootconfig etc/mysql-root-localhost.ini --createdb test_db1 --schema etc/schema_0.sql
$(PYTHON) dbmaint.py --rootconfig etc/mysql-root-localhost.ini --upgradedb test_db1
$(PYTHON) dbmaint.py --rootconfig etc/mysql-root-localhost.ini --dropdb test_db1
################################################################
### Database management for testing and CI/CD
create_localdb:
@echo Creating local database, exercise the upgrade code and write credentials to etc/credentials.ini using etc/github_actions_mysql_rootconfig.ini
@echo etc/credentials.ini will be used automatically by other tests
$(PYTHON) dbmaint.py --create_client=$$MYSQL_ROOT_PASSWORD --writeconfig etc/github_actions_mysql_rootconfig.ini
$(PYTHON) dbmaint.py --rootconfig etc/github_actions_mysql_rootconfig.ini --createdb actions_test --schema etc/schema_0.sql --writeconfig etc/credentials.ini
$(PYTHON) dbmaint.py --upgradedb --loglevel DEBUG
$(PYTHON) -m pytest -x --log-cli-level=DEBUG tests/dbreader_test.py
remove_localdb:
@echo Removing local database using etc/github_actions_mysql_rootconfig.ini
$(PYTHON) dbmaint.py --rootconfig etc/github_actions_mysql_rootconfig.ini --dropdb actions_test --writeconfig etc/credentials.ini
/bin/rm -f etc/credentials.ini
coverage:
$(PYTHON) -m pip install --upgrade pip
$(PIP_INSTALL) codecov pytest pytest_cov
$(PYTHON) -m pytest -v --cov=. --cov-report=xml tests
debug:
make debug-local
DEBUG:=$(PYTHON) bottle_app.py --loglevel DEBUG
debug-single:
@echo run bottle locally in debug mode single-threaded
$(DEBUG) --dbcredentials etc/credentials.ini
debug-multi:
@echo run bottle locally in debug mode multi-threaded
$(DEBUG) --dbcredentials etc/credentials.ini --multi
debug-local:
@echo run bottle locally in debug mode, storing new data in database
$(DEBUG) --storelocal --dbcredentials etc/credentials-local.ini
debug-dev:
@echo run bottle locally in debug mode, storing new data in S3, with the dev.planttracer.com database
$(DEBUG) --dbcredentials etc/credentials-dev.ini
freeze:
$(PYTHON) -m pip freeze > requirements.txt
clean:
find . -name '*~' -exec rm {} \;
/bin/rm -rf __pycache__ */__pycache__
tracker-debug:
/bin/rm -f outfile.mp4
$(PYTHON) tracker.py --moviefile="tests/data/2019-07-12 circumnutation.mp4" --outfile=outfile.mp4
open outfile.mp4
eslint:
(cd static;make eslint)
(cd templates;make eslint)
jscoverage:
npm run coverage
################################################################
# Installations are used by the CI pipeline:
# Generic:
install-python-dependencies: $(REQ)
$(PYTHON) -m pip install --upgrade pip
if [ -r requirements.txt ]; then $(PIP_INSTALL) -r requirements.txt ; else echo no requirements.txt ; fi
install-chromium-browser-ubuntu: $(REQ)
sudo apt-get install -y chromium-browser
chromium --version
install-chromium-browser-macos: $(REQ)
brew install chromium --no-quarantine
# Includes ubuntu dependencies
install-ubuntu: $(REQ)
echo on GitHub, we use this action instead: https://github.com/marketplace/actions/setup-ffmpeg
which ffmpeg || sudo apt install ffmpeg
which node || sudo apt-get install nodejs
which npm || sudo apt-get install npm
npm ci
$(PYTHON) -m pip install --upgrade pip
if [ -r requirements-ubuntu.txt ]; then $(PIP_INSTALL) -r requirements-ubuntu.txt ; else echo no requirements-ubuntu.txt ; fi
if [ -r requirements.txt ]; then $(PIP_INSTALL) -r requirements.txt ; else echo no requirements.txt ; fi
# Includes MacOS dependencies managed through Brew
install-macos:
brew update
brew upgrade
brew install python3
brew install ffmpeg
brew install node
brew install npm
npm ci
npm install -g typescript webpack webpack-cli
$(PYTHON) -m pip install --upgrade pip
if [ -r requirements-macos.txt ]; then $(PIP_INSTALL) -r requirements-macos.txt ; else echo no requirements-ubuntu.txt ; fi
if [ -r requirements.txt ]; then $(PIP_INSTALL) -r requirements.txt ; else echo no requirements.txt ; fi
# Includes Windows dependencies
install-windows:
$(PYTHON) -m pip install --upgrade pip
if [ -r requirements-windows.txt ]; then $(PIP_INSTALL) -r requirements-windows.txt ; else echo no requirements-ubuntu.txt ; fi
if [ -r requirements.txt ]; then $(PIP_INSTALL) -r requirements.txt ; else echo no requirements.txt ; fi
################################################################
## Python maintence and Zappa deployment
# https://stackoverflow.com/questions/24764549/upgrade-python-packages-from-requirements-txt-using-pip-command
update-python:
cat requirements.txt | cut -f1 -d= | xargs pip install -U
update-dev:
$(ACTIVATE) && pip freeze > requirements.txt
$(ACTIVATE) && zappa update dev
update-prod:
$(ACTIVATE) && pip freeze > requirements.txt
$(ACTIVATE) && zappa update production
update-demo:
$(ACTIVATE) && pip freeze > requirements.txt
$(ACTIVATE) && zappa update demo
%.js: %.ts
tsc $<