forked from distributed-system-analysis/pbench
-
Notifications
You must be signed in to change notification settings - Fork 0
/
shell.py
227 lines (202 loc) · 8.64 KB
/
shell.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
from configparser import NoOptionError, NoSectionError
from logging import Logger
import os
from pathlib import Path
import site
import subprocess
import sys
from flask import Flask
import sdnotify
from pbench.common import wait_for_uri
from pbench.common.exceptions import BadConfig
from pbench.common.logger import get_pbench_logger
from pbench.server import PbenchServerConfig
from pbench.server.api import create_app, get_server_config
from pbench.server.auth import OpenIDClient
from pbench.server.database import init_db
from pbench.server.database.database import Database
from pbench.server.indexer import init_indexing
PROG = "pbench-shell"
def app() -> Flask:
"""External gunicorn application entry point."""
return create_app(get_server_config())
def find_the_unicorn(logger: Logger):
"""Add the location of the `pip install --user` version of gunicorn to the
PATH if it exists.
"""
local = Path(site.getuserbase()) / "bin"
if (local / "gunicorn").exists():
# Use a `pip install --user` version of gunicorn
os.environ["PATH"] = ":".join([str(local), os.environ["PATH"]])
logger.info(
"Found a local unicorn: augmenting server PATH to {}",
os.environ["PATH"],
)
def run_gunicorn(server_config: PbenchServerConfig, logger: Logger) -> int:
"""Setup of the Gunicorn Pbench Server Flask application.
Returns:
1 on error, or the gunicorn sub-process status code
"""
notifier = sdnotify.SystemdNotifier()
notifier.notify("STATUS=Identifying configuration")
if site.ENABLE_USER_SITE:
find_the_unicorn(logger)
try:
db_uri = server_config.get("database", "uri")
db_wait_timeout = int(server_config.get("database", "wait_timeout"))
es_uri = server_config.get("Indexing", "uri")
es_wait_timeout = int(server_config.get("Indexing", "wait_timeout"))
workers = str(server_config.get("pbench-server", "workers"))
worker_timeout = str(server_config.get("pbench-server", "worker_timeout"))
server_config.get("flask-app", "secret-key")
except (NoOptionError, NoSectionError) as exc:
logger.error("Error fetching required configuration: {}", exc)
notifier.notify("STOPPING=1")
notifier.notify("STATUS=Unable to configure gunicorn")
return 1
notifier.notify("STATUS=Waiting for database")
logger.info(
"Waiting at most {:d} seconds for database instance {} to become available.",
db_wait_timeout,
db_uri,
)
try:
wait_for_uri(db_uri, db_wait_timeout)
except BadConfig as exc:
logger.error(f"{exc}")
notifier.notify("STOPPING=1")
notifier.notify(f"STATUS=Bad DB config {exc}")
return 1
except ConnectionRefusedError:
logger.error("Database {} not responding", db_uri)
notifier.notify("STOPPING=1")
notifier.notify("STATUS=DB not responding")
return 1
notifier.notify("STATUS=Waiting for Elasticsearch instance")
logger.info(
"Waiting at most {:d} seconds for the Elasticsearch instance {} to become available.",
es_wait_timeout,
es_uri,
)
try:
wait_for_uri(es_uri, es_wait_timeout)
except BadConfig as exc:
logger.error(f"{exc}")
notifier.notify("STOPPING=1")
notifier.notify(f"STATUS=Bad index config {exc}")
return 1
except ConnectionRefusedError:
logger.error("Index {} not responding", es_uri)
notifier.notify("STOPPING=1")
notifier.notify("STATUS=Index service not responding")
return 1
notifier.notify("STATUS=Initializing OIDC")
try:
oidc_server = OpenIDClient.wait_for_oidc_server(server_config, logger)
except OpenIDClient.NotConfigured as exc:
logger.warning("OpenID Connect client not configured, {}", exc)
notifier.notify("STOPPING=1")
notifier.notify("STATUS=OPENID broker not configured")
return 1
except (
OpenIDClient.ServerConnectionError,
OpenIDClient.ServerConnectionTimedOut,
) as exc:
logger.warning("OpenID Connect client not reachable, {}", exc)
notifier.notify("STOPPING=1")
notifier.notify("STATUS=OPENID broker not responding")
return 1
else:
logger.info("Pbench server using OIDC server {}", oidc_server)
# Multiple gunicorn workers will attempt to connect to the DB; rather than
# attempt to synchronize them, detect a missing DB (from the database URI)
# and create it here. It's safer to do this here, where we're
# single-threaded.
notifier.notify("STATUS=Initializing database")
logger.info("Performing database setup")
Database.create_if_missing(db_uri, logger)
try:
init_db(server_config, logger)
except (NoOptionError, NoSectionError) as exc:
logger.error("Invalid database configuration: {}", exc)
notifier.notify("STOPPING=1")
notifier.notify(f"STATUS=Error initializing database: {exc}")
return 1
# The server and indexer both attempt to initialize the Elasticsearch
# instance, which can cause a race and messy logging. To avoid that,
# initialize the indexing sub-system here.
notifier.notify("STATUS=Initializing Elasticsearch")
logger.info("Performing Elasticsearch indexing setup")
try:
init_indexing(PROG, server_config, logger)
except (NoOptionError, NoSectionError) as exc:
logger.error("Invalid indexing configuration: {}", exc)
notifier.notify("STOPPING=1")
notifier.notify(f"STATUS=Invalid indexing config {exc}")
return 1
notifier.notify("READY=1")
# Beginning of the gunicorn command to start the pbench-server.
cmd_line = [
"gunicorn",
"--workers",
workers,
"--timeout",
worker_timeout,
"--pid",
"/run/pbench-server/gunicorn.pid",
"--bind",
"unix:/run/pbench-server/pbench-server.sock",
"--log-syslog",
"--log-syslog-prefix",
"pbench-server",
]
# If PB_PROFILE_DUMP_FILE is defined, enable profiling of the server's
# execution of requests. If defined to an empty string, profiling
# results are dumped to the log; otherwise, the value is treated as the
# name of a file to receive the data. (An excellent choice is
# "/srv/pbench/public_html/pbench_server.prof", because this is writable
# by the server and easily accessed by the user via the browser by hitting
# "https://<server>/pbench_server.prof".) Note that the file is
# overwritten for each request.
if os.environ.get("PB_PROFILE_DUMP_FILE") is not None:
cmd_line += ["--config", "/opt/pbench-server/lib/pbench/profiling.conf.py"]
# When installed via RPM, the shebang in the gunicorn script includes a -s
# which prevents Python from implicitly including the user site packages in
# the sys.path configuration. (Note that, when installed via Pip, the
# shebang does not include this switch.) This means that gunicorn itself,
# but, more importantly, the user application which it runs, won't be able
# to use any packages installed with the Pip --user switch, like our
# requirements.txt contents. However, gunicorn provides the --pythonpath
# switch which adds entries to the PYTHONPATH used to run the application.
# So, we request that gunicorn add our current user site packages location
# to the app's PYTHONPATH so that it can actually find the locally installed
# packages as well as the pbench.pth file which points to the Pbench Server
# package.
if site.ENABLE_USER_SITE:
adds = f"{site.getusersitepackages()},{server_config.LIBDIR}"
cmd_line += ["--pythonpath", adds]
cmd_line.append("pbench.cli.server.shell:app()")
logger.info("Starting Gunicorn Pbench Server application")
notifier.notify("STATUS=Starting gunicorn")
cp = subprocess.run(cmd_line, cwd=server_config.log_dir)
logger.info("Gunicorn Pbench Server application exited with {}", cp.returncode)
notifier.notify(f"STATUS=Gunicorn terminated with {cp.returncode}")
notifier.notify("STOPPING=1")
return cp.returncode
def main() -> int:
"""Wrapper performing general error handling here allowing for the heavy
lifting to be performed in run_gunicorn().
Returns:
0 on success, 1 on error
"""
try:
server_config = get_server_config()
logger = get_pbench_logger(PROG, server_config)
except Exception as exc:
print(exc, file=sys.stderr)
return 1
try:
return run_gunicorn(server_config, logger)
except Exception:
logger.exception("Unhandled exception running gunicorn")
return 1