api: put all env vars in supervisord conf

.gitignore

@@ -22,10 +22,11 @@ cloudformation/kong/
 */dist
 mpcontribs-client/build/*
 mpcontribs-io/build/*
-mpcontribs-api/mpcontribs/api/swagger/
+mpcontribs-api/mpcontribs/api/swagger*/
 .cache/
 .ipython/
 .jupyter/
 .local/
 mpcontribs-portal/mpcontribs/portal/templates/notebooks
 coverage.xml
+mpcontribs-api/supervisord.conf

mpcontribs-api/mpcontribs/api/config.py

@@ -33,7 +33,7 @@
 }
 REDIS_ADDRESS = os.environ.get("REDIS_ADDRESS", "redis")
 REDIS_URL = RQ_REDIS_URL = RQ_DASHBOARD_REDIS_URL = f"redis://{REDIS_ADDRESS}"
-DOC_DIR = os.path.join(os.path.dirname(__file__), "swagger")
+DOC_DIR = os.path.join(os.path.dirname(__file__), f"swagger-{MPCONTRIBS_DB}")
 
 SWAGGER = {
     "swagger_ui_bundle_js": "//unpkg.com/swagger-ui-dist@3/swagger-ui-bundle.js",

mpcontribs-api/scripts/start.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-set -a
-source $ENV_FILE
-env
-
 wait-for-it.sh $JUPYTER_GATEWAY_HOST -t 50 -- gunicorn -c gunicorn.conf.py \
     -b 0.0.0.0:$API_PORT -k gevent -w $NWORKERS \
     --access-logfile - --error-logfile - --log-level debug $RELOAD \

mpcontribs-api/scripts/start_rq.sh

@@ -1,7 +1,3 @@
 #!/bin/bash
 
-set -a
-source $ENV_FILE
-env
-
 wait-for-it.sh $JUPYTER_GATEWAY_HOST -s -t 50 -- wait-for-it.sh $MPCONTRIBS_API_HOST -s -t 15 -- ddtrace-run flask rq $1

mpcontribs-api/supervisord/conf.py

@@ -2,13 +2,34 @@
 from jinja2 import Environment, FileSystemLoader
 
 DIR = os.path.abspath(os.path.dirname(__file__))
+PRODUCTION = int(os.environ.get("PRODUCTION", "1"))
 
-deployments = [
-    entry.name.split(".")[0]
-    for entry in os.scandir(os.environ["ENV_FILES"])
-    if entry.is_file()
-]
+deployments = {}
+
+for deployment in os.environ.get("DEPLOYMENTS", "ml:5002").split(","):
+    name, db, s3, tm, api_port = deployment.split(":")
+    portal_port = 8080 + int(api_port) % 5000
+    deployments[name] = {
+        "api_port": api_port,
+        "portal_port": portal_port,
+        "db": db,
+        "s3": s3,
+        "tm": tm.upper()
+    }
+
+kwargs = {
+    "deployments": deployments,
+    "nworkers": 2 if PRODUCTION else 1,
+    "reload": "" if PRODUCTION else "--reload",
+    "node_env": "production" if PRODUCTION else "development",
+    "flask_log_level": "INFO" if PRODUCTION else "DEBUG",
+    "jupyter_gateway_host": "localhost:8888" if PRODUCTION else "kernel-gateway:8888",
+    "dd_agent_host": "localhost" if PRODUCTION else "datadog",
+    "mpcontribs_api_host": "localhost" if PRODUCTION else "contribs-apis",
+}
+kwargs["flask_env"] = kwargs["node_env"]
+kwargs["jupyter_gateway_url"] = "http://" + kwargs["jupyter_gateway_host"]
 
 env = Environment(loader=FileSystemLoader(DIR))
 template = env.get_template("supervisord.conf.jinja")
-template.stream(deployments=deployments).dump("supervisord.conf")
+template.stream(**kwargs).dump("supervisord.conf")

mpcontribs-api/supervisord/supervisord.conf.jinja

@@ -7,14 +7,31 @@ environment=
     MPCONTRIBS_MONGO_HOST="%(ENV_MPCONTRIBS_MONGO_HOST)s",
     AWS_ACCESS_KEY_ID="%(ENV_AWS_ACCESS_KEY_ID)s",
     AWS_SECRET_ACCESS_KEY="%(ENV_AWS_SECRET_ACCESS_KEY)s",
-    AWS_SNS_TOPIC_ARN="%(ENV_AWS_SNS_TOPIC_ARN)s"
+    AWS_SNS_TOPIC_ARN="%(ENV_AWS_SNS_TOPIC_ARN)s",
+    METADATA_URI="%(ENV_ECS_CONTAINER_METADATA_URI_V4)s",
+    REDIS_ADDRESS="%(ENV_REDIS_ADDRESS)s",
+    AWS_REGION="us-east-1",
+    AWS_DEFAULT_REGION="us-east-1",
+    MAIL_DEFAULT_SENDER="contribs@materialsproject.org",
+    DD_PROFILING_ENABLED="false",
+    DD_LOGS_INJECTION="true",
+    DD_PATCH_MODULES="aioredis:false,aredis:false,boto:false,bottle:false,cassandra:false,consul:false,django:false,algoliasearch:false,grpc:false,httpx:false,mysql:false,mysqldb:false,pymysql:false,mariadb:false,pylibmc:false,pymemcache:false",
+    FLASK_APP="mpcontribs.api",
+    PYTHONUNBUFFERED=1,
+    MAX_REQUESTS=100000,
+    MAX_REQUESTS_JITTER=10000,
+    NWORKERS={{ nworkers }},
+    RELOAD="{{ reload }}",
+    NODE_ENV="{{ node_env }}",
+    FLASK_ENV="{{ flask_env }}",
+    FLASK_LOG_LEVEL="{{ flask_log_level }}",
+    JUPYTER_GATEWAY_URL="{{ jupyter_gateway_url }}",
+    JUPYTER_GATEWAY_HOST="{{ jupyter_gateway_host }}",
+    DD_AGENT_HOST="{{ dd_agent_host }}"
 
 [program:main]
 command=./main.py
 directory=/app
-environment=
-    METADATA_URI="%(ENV_ECS_CONTAINER_METADATA_URI_V4)s",
-    AWS_DEFAULT_REGION="us-east-1"
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/fd/2
@@ -29,13 +46,15 @@ supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
 [supervisorctl]
 serverurl=unix:///var/run/supervisor.sock
 
+{% set names = deployments.keys() %}
+
 [group:apis]
-programs={{ deployments|join('-api,') }}-api
+programs={{ names|join('-api,') }}-api
 
 [group:rq]
-programs={{ deployments|join('-worker,') }}-worker,{{ deployments|join('-scheduler,') }}-scheduler
+programs={{ names|join('-worker,') }}-worker,{{ names|join('-scheduler,') }}-scheduler
 
-{% for deployment in deployments %}
+{% for deployment, cfg in deployments.items() %}
 {% set defaults %}
 directory=/app
 autostart=false
@@ -47,7 +66,16 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/fd/2
 stderr_logfile_maxbytes=0
 environment=
-    ENV_FILE="%(ENV_ENV_FILES)s/{{ deployment }}.env",
+    DEPLOYMENT={{ loop.index0 }},
+    API_PORT={{ cfg.api_port }},
+    PORTAL_PORT={{ cfg.portal_port }},
+    MPCONTRIBS_API_HOST="{{ mpcontribs_api_host }}:{{ cfg.api_port }}",
+    MPCONTRIBS_DB_NAME="mpcontribs-{{ cfg.db }}",
+    TRADEMARK="{{ cfg.tm }}",
+    S3_DOWNLOADS_BUCKET="mpcontribs-downloads-{{ cfg.s3 }}",
+    S3_ATTACHMENTS_BUCKET="mpcontribs-attachments-{{ cfg.s3 }}",
+    S3_IMAGES_BUCKET="mpcontribs-images-{{ cfg.s3 }}",
+    ADMIN_GROUP="admin_{{ deployment }}.materialsproject.org"
 {% endset %}
 
 [program:{{ deployment }}-worker]