From c23f59f1d5e79a9179c2da6c2f6456de6e56c567 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Sat, 23 Dec 2023 19:54:48 +0000 Subject: [PATCH] Reorganise h's config files This commit reorganises h's `conf/` directory following a schema that I plan to apply to our cookiecutter and the rest of our apps. h's `conf/` directory presents some unique problems that don't apply to our other apps because it has some extra configuration files that our other apps don't have: * Several `conf/*websocket*` config files for h's separate websocket app, which even needs an extra set of config files for when it's running in "monolithic" mode (used by `make run-docker` in development and in the Canada production environment). * `websocket-newrelic.ini`: a New Relic configuration used by h's separate websocket app * An `nginx.conf` file used by h's "internal" NGINX instance in production (there's no NGINX in h's development environment) --- .github/workflows/ci.yml | 4 ++-- .github/workflows/data_tasks.yml | 2 +- .github/workflows/deploy.yml | 2 +- .github/workflows/report_refresh.yml | 2 +- conf/{development-app.ini => development.ini} | 9 --------- conf/gunicorn-dev.conf.py | 3 +++ conf/gunicorn-websocket-dev.conf.py | 5 +++++ conf/gunicorn-websocket-monolithic.conf.py | 5 +++++ conf/gunicorn-websocket.conf.py | 5 +++++ conf/gunicorn.conf.py | 1 + conf/{app.ini => production.ini} | 5 ----- conf/supervisord-dev.conf | 4 ++-- conf/supervisord.conf | 13 +++++++++++-- conf/websocket-dev.ini | 11 ----------- conf/websocket-monolithic.ini | 9 --------- conf/{websocket-separate.ini => websocket.ini} | 10 ---------- h/cli/__init__.py | 2 +- tests/functional/bin/run_data_task_test.py | 2 +- 18 files changed, 39 insertions(+), 55 deletions(-) rename conf/{development-app.ini => development.ini} (88%) create mode 100644 conf/gunicorn-dev.conf.py create mode 100644 conf/gunicorn-websocket-dev.conf.py create mode 100644 conf/gunicorn-websocket-monolithic.conf.py create mode 100644 conf/gunicorn-websocket.conf.py create mode 100644 conf/gunicorn.conf.py rename conf/{app.ini => production.ini} (88%) rename conf/{websocket-separate.ini => websocket.ini} (75%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b59f975830e..4488e37d6ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: - '.github/workflows/deploy.yml' - '.github/workflows/redeploy.yml' - 'bin/logger' - - 'conf/development-app.ini' - - 'app.ini' + - 'conf/development.ini' + - 'production.ini' - 'conf/supervisord*.conf' - 'docs/*' - 'requirements/*.in' diff --git a/.github/workflows/data_tasks.yml b/.github/workflows/data_tasks.yml index 8f150465282..cc33e1f1ef2 100644 --- a/.github/workflows/data_tasks.yml +++ b/.github/workflows/data_tasks.yml @@ -55,5 +55,5 @@ jobs: Env: ${{ inputs.Environment }} Timeout: 7200 Region: ${{ inputs.Region }} - Command: 'newrelic-admin run-program python bin/run_data_task.py --config-file conf/app.ini --task ${{ inputs.Task }}' + Command: 'newrelic-admin run-program python bin/run_data_task.py --config-file conf/production.ini --task ${{ inputs.Task }}' secrets: inherit diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9421174795e..478172b653c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,7 +12,7 @@ on: - '.github/*' - 'bin/create-testdb' - 'bin/install-python' - - 'conf/development-app.ini' + - 'conf/development.ini' - 'conf/supervisord-dev.conf' - 'conf/websocket-dev.ini' - 'docs/*' diff --git a/.github/workflows/report_refresh.yml b/.github/workflows/report_refresh.yml index c2dc08a8b0f..657c3850e69 100644 --- a/.github/workflows/report_refresh.yml +++ b/.github/workflows/report_refresh.yml @@ -15,5 +15,5 @@ jobs: Env: 'prod' Timeout: 3600 Region: 'all' - Command: 'newrelic-admin run-program python bin/run_data_task.py --config-file conf/app.ini --task report/refresh' + Command: 'newrelic-admin run-program python bin/run_data_task.py --config-file conf/production.ini --task report/refresh' secrets: inherit diff --git a/conf/development-app.ini b/conf/development.ini similarity index 88% rename from conf/development-app.ini rename to conf/development.ini index 6812311572b..fa41d6b17e5 100644 --- a/conf/development-app.ini +++ b/conf/development.ini @@ -22,15 +22,6 @@ secret_key: notverysecretafterall sqlalchemy.url: postgresql://postgres@localhost/postgres -[server:main] -use: egg:gunicorn#main -host: 0.0.0.0 -port: 5000 -proc_name: web -graceful_timeout: 0 -timeout: 0 -errorlog: - - [loggers] keys = root, gunicorn.error, h diff --git a/conf/gunicorn-dev.conf.py b/conf/gunicorn-dev.conf.py new file mode 100644 index 00000000000..9b2156f02a1 --- /dev/null +++ b/conf/gunicorn-dev.conf.py @@ -0,0 +1,3 @@ +bind = "0.0.0.0:5000" +graceful_timeout = 0 +timeout = 0 diff --git a/conf/gunicorn-websocket-dev.conf.py b/conf/gunicorn-websocket-dev.conf.py new file mode 100644 index 00000000000..a1b4a398a8c --- /dev/null +++ b/conf/gunicorn-websocket-dev.conf.py @@ -0,0 +1,5 @@ +bind = "localhost:5001" +worker_class = "h.streamer.Worker" +graceful_timeout = 0 +workers = 2 +worker_connections = 8 diff --git a/conf/gunicorn-websocket-monolithic.conf.py b/conf/gunicorn-websocket-monolithic.conf.py new file mode 100644 index 00000000000..2bc7f926c90 --- /dev/null +++ b/conf/gunicorn-websocket-monolithic.conf.py @@ -0,0 +1,5 @@ +bind = "unix:/tmp/gunicorn-websocket.sock" +worker_class = "h.streamer.Worker" +graceful_timeout = 0 +workers = 2 +worker_connections = 8192 diff --git a/conf/gunicorn-websocket.conf.py b/conf/gunicorn-websocket.conf.py new file mode 100644 index 00000000000..12ed214038f --- /dev/null +++ b/conf/gunicorn-websocket.conf.py @@ -0,0 +1,5 @@ +bind = "0.0.0.0:5000" +worker_class = "h.streamer.Worker" +graceful_timeout = 0 +workers = 2 +worker_connections = 8192 diff --git a/conf/gunicorn.conf.py b/conf/gunicorn.conf.py new file mode 100644 index 00000000000..d59b887b3d1 --- /dev/null +++ b/conf/gunicorn.conf.py @@ -0,0 +1 @@ +bind = "unix:/tmp/gunicorn-web.sock" diff --git a/conf/app.ini b/conf/production.ini similarity index 88% rename from conf/app.ini rename to conf/production.ini index bd2ade24cc8..bb45d3e8f02 100644 --- a/conf/app.ini +++ b/conf/production.ini @@ -9,11 +9,6 @@ use: call:h.app:create_app [filter:proxy-prefix] use: egg:PasteDeploy#prefix -[server:main] -use: egg:gunicorn#main -bind: unix:/tmp/gunicorn-web.sock -proc_name: web - [loggers] keys = root, alembic, gunicorn.error, h diff --git a/conf/supervisord-dev.conf b/conf/supervisord-dev.conf index 00745af10a6..6d6faec3f2a 100644 --- a/conf/supervisord-dev.conf +++ b/conf/supervisord-dev.conf @@ -3,7 +3,7 @@ nodaemon = true silent = true [program:web] -command = pserve --reload conf/development-app.ini +command = gunicorn --paste conf/development.ini --config conf/gunicorn-dev.conf.py stdout_events_enabled=true stderr_events_enabled=true stopsignal = KILL @@ -11,7 +11,7 @@ stopasgroup = true autostart = %(ENV_ENABLE_WEB)s [program:websocket] -command = pserve --reload conf/websocket-dev.ini +command = gunicorn --paste conf/websocket-dev.ini --config conf/gunicorn-websocket-dev.conf.py stdout_events_enabled=true stderr_events_enabled=true stopsignal = KILL diff --git a/conf/supervisord.conf b/conf/supervisord.conf index 322a6a8f017..5c681fbd067 100644 --- a/conf/supervisord.conf +++ b/conf/supervisord.conf @@ -13,7 +13,7 @@ stderr_events_enabled=true autostart = %(ENV_ENABLE_NGINX)s [program:web] -command=newrelic-admin run-program pserve conf/app.ini +command=newrelic-admin run-program gunicorn --paste conf/production.ini --config conf/gunicorn.conf.py stdout_logfile=NONE stderr_logfile=NONE stdout_events_enabled=true @@ -21,13 +21,22 @@ stderr_events_enabled=true autostart = %(ENV_ENABLE_WEB)s [program:websocket] -command=pserve %(ENV_WEBSOCKET_CONFIG)s +command=newrelic-admin run-program gunicorn --paste conf/websocket.ini --config conf/gunicorn-websocket.conf.py stdout_logfile=NONE stderr_logfile=NONE stdout_events_enabled=true stderr_events_enabled=true autostart = %(ENV_ENABLE_WEBSOCKET)s +[program:websocket-monolithic] +command=newrelic-admin run-program gunicorn --paste conf/websocket-monolithic.ini --config conf/gunicorn-websocket-monolithic.conf.py +stdout_logfile=NONE +stderr_logfile=NONE +stdout_events_enabled=true +stderr_events_enabled=true +autostart = %(ENV_ENABLE_WEBSOCKET_MONOLITHIC)s +process_name = websocket + [program:worker] command=newrelic-admin run-program hypothesis celery worker --loglevel=INFO stdout_logfile=NONE diff --git a/conf/websocket-dev.ini b/conf/websocket-dev.ini index 4f4a327b937..30f8123faa8 100644 --- a/conf/websocket-dev.ini +++ b/conf/websocket-dev.ini @@ -10,17 +10,6 @@ secret_key: notverysecretafterall # SQLAlchemy configuration -- See SQLAlchemy documentation sqlalchemy.url: postgresql://postgres@localhost/postgres -[server:main] -use: egg:gunicorn#main -host: localhost -port: 5001 -worker_class: h.streamer.Worker -graceful_timeout: 0 -proc_name: websocket -# This is very low so you can see what happens when we run out -workers: 2 -worker_connections: 8 - [loggers] keys = root, gunicorn.error, h diff --git a/conf/websocket-monolithic.ini b/conf/websocket-monolithic.ini index ddab2880dfe..ada4eee45f8 100644 --- a/conf/websocket-monolithic.ini +++ b/conf/websocket-monolithic.ini @@ -1,15 +1,6 @@ [app:main] use: call:h.streamer:create_app -[server:main] -use: egg:gunicorn#main -bind: unix:/tmp/gunicorn-websocket.sock -worker_class: h.streamer.Worker -graceful_timeout: 0 -proc_name: websocket -workers: 2 -worker_connections: 8192 - [loggers] keys = root, gunicorn.error, h.streamer diff --git a/conf/websocket-separate.ini b/conf/websocket.ini similarity index 75% rename from conf/websocket-separate.ini rename to conf/websocket.ini index 50642cf5753..ada4eee45f8 100644 --- a/conf/websocket-separate.ini +++ b/conf/websocket.ini @@ -1,16 +1,6 @@ [app:main] use: call:h.streamer:create_app -[server:main] -use: egg:gunicorn#main -host: 0.0.0.0 -port: 5000 -worker_class: h.streamer.Worker -graceful_timeout: 0 -proc_name: websocket -workers: 2 -worker_connections: 8192 - [loggers] keys = root, gunicorn.error, h.streamer diff --git a/h/cli/__init__.py b/h/cli/__init__.py index fc8bde7383c..508d7005f13 100644 --- a/h/cli/__init__.py +++ b/h/cli/__init__.py @@ -40,7 +40,7 @@ def bootstrap(app_url, dev=False): else: raise click.ClickException("the app URL must be set in production mode!") - config = "conf/development-app.ini" if dev else "conf/app.ini" + config = "conf/development.ini" if dev else "conf/production.ini" paster.setup_logging(config) request = Request.blank("/", base_url=app_url) diff --git a/tests/functional/bin/run_data_task_test.py b/tests/functional/bin/run_data_task_test.py index fffff26854e..c3efc443b34 100644 --- a/tests/functional/bin/run_data_task_test.py +++ b/tests/functional/bin/run_data_task_test.py @@ -29,7 +29,7 @@ def test_reporting_tasks(self, environ): sys.executable, "bin/run_data_task.py", "--config-file", - "conf/development-app.ini", + "conf/development.ini", "--task", task_name, ],