diff --git a/.github/workflows/dockerfile.yaml b/.github/workflows/dockerfile.yaml index 9669ef5bd6..8f694c3882 100644 --- a/.github/workflows/dockerfile.yaml +++ b/.github/workflows/dockerfile.yaml @@ -154,3 +154,61 @@ jobs: pod=$(kubectl get pods -n "$namespace" -l app.kubernetes.io/name=zulip --output name) kubectl -n "$namespace" logs "$pod" kubectl -n "$namespace" exec "$pod" -c zulip -- cat /var/log/zulip/errors.log + + docker-compose-test: + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: + - build + env: + GITHUB_CI_IMAGE: ghcr.io/${{ github.repository }}:pr-${{ github.event.pull_request.number }} + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Verify Docker Compose config validation + run: | + docker compose \ + -f compose.yaml \ + -f ci/compose.override.yaml \ + --env-file ci/env \ + config + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Start Docker Compose services + run: | + docker compose \ + -f compose.yaml \ + -f ci/compose.override.yaml \ + --env-file ci/env \ + up -d --no-build + + - name: Wait for services to be healthy + run: | + echo "Waiting for zulip service to be healthy..." + timeout 300 bash -c \ + 'until docker inspect --format "{{.State.Health.Status}}" $(docker compose ps -q zulip) | grep -q healthy; do sleep 5; done' + + - name: Verify all services are running + run: | + docker compose ps + # Check that no services are in a failed state + if docker compose ps | grep -E "(Exit|Restarting)"; then + exit 1 + fi + + - name: Check service logs for critical errors + if: success() || failure() + continue-on-error: true + run: | + docker compose ps + docker compose logs zulip diff --git a/.gitignore b/.gitignore index dca4d1c90c..7eabd1dc8d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ .idea/ *.tmproj +# Secrets +/.env + # dev files docker-compose-dev.yml kubernetes/*-dev.yml diff --git a/Dockerfile b/Dockerfile index 65cb53ef40..7e4bc037aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,5 +73,8 @@ COPY certbot-deploy-hook /sbin/certbot-deploy-hook VOLUME ["$DATA_DIR"] EXPOSE 25 80 443 +HEALTHCHECK --interval=10s --timeout=5s --retries=3 --start-period=300s \ + CMD curl -isfL --insecure http://localhost/health || exit 1 + ENTRYPOINT ["/sbin/entrypoint.sh"] CMD ["app:run"] diff --git a/ci/compose.override.yaml b/ci/compose.override.yaml new file mode 100644 index 0000000000..90aff181d0 --- /dev/null +++ b/ci/compose.override.yaml @@ -0,0 +1,21 @@ +--- +secrets: + zulip__postgres_password: + environment: "ZULIP__POSTGRES_PASSWORD" + zulip__memcached_password: + environment: "ZULIP__MEMCACHED_PASSWORD" + zulip__rabbitmq_password: + environment: "ZULIP__RABBITMQ_PASSWORD" + zulip__redis_password: + environment: "ZULIP__REDIS_PASSWORD" + zulip__secret_key: + environment: "ZULIP__SECRET_KEY" + zulip__email_password: + environment: "ZULIP__EMAIL_PASSWORD" + +services: + zulip: + image: "${GITHUB_CI_IMAGE:?error}" + environment: + SETTING_EXTERNAL_HOST: "zulip.example.net" + SETTING_ZULIP_ADMINISTRATOR: "admin@example.net" diff --git a/ci/env b/ci/env new file mode 100644 index 0000000000..32d0df73e4 --- /dev/null +++ b/ci/env @@ -0,0 +1,6 @@ +ZULIP__POSTGRES_PASSWORD=postgres_password +ZULIP__MEMCACHED_PASSWORD=memcached_password +ZULIP__RABBITMQ_PASSWORD=rabbitmq_password +ZULIP__REDIS_PASSWORD=redis_password +ZULIP__SECRET_KEY=django_secret_key +ZULIP__EMAIL_PASSWORD=outgoing_email_password diff --git a/compose.override.yaml b/compose.override.yaml new file mode 100644 index 0000000000..bdc16bf547 --- /dev/null +++ b/compose.override.yaml @@ -0,0 +1,128 @@ +--- +secrets: + ## You can either set these secrets in a file named .env in this directory, or + ## change these from "environment:" to "file:" with a path to store them on + ## disk. Be sure that files, if you use them, do not contain trailing + ## newlines! + ## + ## You may also add additional secrets here, prefixed with zulip__, (and also + ## in the "secrets:" block inside the "zulip" service, below) in order to + ## propagate them into Zulip's /etc/zulip/zulip-secrets.conf + ## + ## --> https://docs.docker.com/reference/compose-file/secrets/ + ## https://docs.docker.com/compose/how-tos/use-secrets/ + zulip__postgres_password: + ## Note that you need to do a manual `ALTER ROLE` query if you + ## change this on a system after booting the postgres container + ## the first time on a host. Instructions are available in README.md. + environment: "ZULIP__POSTGRES_PASSWORD" + zulip__memcached_password: + environment: "ZULIP__MEMCACHED_PASSWORD" + zulip__rabbitmq_password: + environment: "ZULIP__RABBITMQ_PASSWORD" + zulip__redis_password: + environment: "ZULIP__REDIS_PASSWORD" + zulip__secret_key: + environment: "ZULIP__SECRET_KEY" + zulip__email_password: + environment: "ZULIP__EMAIL_PASSWORD" + +services: + zulip: + build: + args: + ## If you want to build zulip from a different repo/branch, you can + ## change these and run: + ## + ## docker compose build zulip + ZULIP_GIT_URL: https://github.com/zulip/zulip.git + ZULIP_GIT_REF: "11.4" + environment: + ## See https://github.com/zulip/docker-zulip#configuration for + ## details on this section and how to discover the many + ## additional settings that are supported here. + ## + ## The following two settings are required: + SETTING_EXTERNAL_HOST: "localhost.localdomain" + SETTING_ZULIP_ADMINISTRATOR: "admin@example.com" + + ## Most deploys do SSL termination outside of the container; Zulip + ## automatically generates a self-signed certificate to use on port443. + ## Uncomment this to set up an auto-renewed Lets Encrypt certificate + ## inside the container -- this requires that SETTING_EXTERNAL_HOST be + ## accessible from the public network. + ## + # SSL_CERTIFICATE_GENERATION: "certbot" + + ## By default, port 80 redirects to port 443, as is suitable for exposing + ## publicly. To handle traffic directly on port 80 (if doing SSL + ## termination in an outer reverse proxy), uncomment this. + ## + # DISABLE_HTTPS: True + + ## If you're using a reverse proxy, you will also need to provide the + ## comma-separated set of IP addresses (or CIDR ranges) to trust here. + ## + ## --> https://zulip.readthedocs.io/en/stable/production/reverse-proxies.html + ## + # LOADBALANCER_IPS: "10.0.0.0/8" + + ## Outgoing email settings + ## + ## --> https://zulip.readthedocs.io/en/stable/production/email.html + ## + # SETTING_EMAIL_HOST: "smtp.example.com" + # SETTING_EMAIL_HOST_USER: "noreply@example.com" + # SETTING_EMAIL_PORT: "587" + # SETTING_EMAIL_USE_SSL: False + # SETTING_EMAIL_USE_TLS: True + + ## Uncomment to enable the incoming email gateway. You will need to + ## ensure that email to emaildomain.example.com is routed to this host + ## (e.g. via MX record) + ## + ## --> https://zulip.readthedocs.io/en/stable/production/email-gateway.html + ## + # SETTING_EMAIL_GATEWAY_PATTERN: "%s@emaildomain.example.com" + + ## A comma-separated list of authentication backends to enable. Note that + ## this ZULIP_AUTH_BACKENDS takes the place of + ## SETTINGS_AUTHENTICATION_BACKENDS. This defaults to just + ## EmailAuthBackend. + ## + ## --> https://zulip.readthedocs.io/en/stable/production/authentication-methods.html + ## + # ZULIP_AUTH_BACKENDS: "EmailAuthBackend,GoogleAuthBackend" + + ## Uncomment this when configuring the mobile push notifications service. + ## After setting these, you will need to register the server: + ## + ## docker compose exec -it -u zulip zulip /home/zulip/deployments/current/manage.py register_server + ## + ## --> https://zulip.readthedocs.io/en/stable/production/mobile-push-notifications.html + ## + # SETTING_ZULIP_SERVICE_PUSH_NOTIFICATIONS: True + # SETTING_ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS: True + + ## By default, files uploaded by users and profile pictures are + ## stored directly on the Zulip server. You can configure files + ## to be stored in Amazon S3 or a compatible data store + ## here. + ## + ## If you want to use the S3 backend, you must set + ## SETTING_LOCAL_UPLOADS_DIR to None as well as configuring the + ## other fields. + ## + ## --> https://zulip.readthedocs.io/en/latest/production/upload-backends.html + ## + # SETTING_LOCAL_UPLOADS_DIR: "None" + # SETTING_S3_AUTH_UPLOADS_BUCKET: "" + # SETTING_S3_AVATAR_BUCKET: "" + # SETTING_S3_ENDPOINT_URL: "None" + # SETTING_S3_REGION: "None" + + ## For a complete list of possible settings, see: + ## --> https://github.com/zulip/zulip/blob/11.4/zproject/prod_settings_template.py + secrets: + ## Add any additional zulip__ secrets that you defined above. + [] diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000000..d5be196377 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,113 @@ +--- +services: + database: + image: "zulip/zulip-postgresql:14" + restart: unless-stopped + secrets: + - zulip__postgres_password + environment: + POSTGRES_DB: "zulip" + POSTGRES_USER: "zulip" + POSTGRES_PASSWORD_FILE: /run/secrets/zulip__postgres_password + volumes: + - "postgresql-14:/var/lib/postgresql/data:rw" + attach: false + memcached: + image: "memcached:alpine" + restart: unless-stopped + command: + - "sh" + - "-euc" + - | + echo 'mech_list: plain' > "$$SASL_CONF_PATH" + echo "zulip@$$HOSTNAME:$$(cat $$MEMCACHED_PASSWORD_FILE)" > "$$MEMCACHED_SASL_PWDB" + echo "zulip@localhost:$$(cat $$MEMCACHED_PASSWORD_FILE)" >> "$$MEMCACHED_SASL_PWDB" + exec memcached -S + secrets: + - zulip__memcached_password + environment: + SASL_CONF_PATH: "/home/memcache/memcached.conf" + MEMCACHED_SASL_PWDB: "/home/memcache/memcached-sasl-db" + MEMCACHED_PASSWORD_FILE: /run/secrets/zulip__memcached_password + attach: false + rabbitmq: + image: "rabbitmq:4.1" + restart: unless-stopped + command: + - "sh" + - "-euc" + - | + export RABBITMQ_DEFAULT_PASS="$$(cat $$RABBITMQ_PASSWORD_FILE)" + echo 'default_user = $$(RABBITMQ_DEFAULT_USER)' >> /etc/rabbitmq/rabbitmq.conf + echo 'default_pass = $$(RABBITMQ_DEFAULT_PASS)' >> /etc/rabbitmq/rabbitmq.conf + exec docker-entrypoint.sh rabbitmq-server + secrets: + - zulip__rabbitmq_password + environment: + RABBITMQ_DEFAULT_USER: "zulip" + RABBITMQ_PASSWORD_FILE: /run/secrets/zulip__rabbitmq_password + volumes: + - "rabbitmq:/var/lib/rabbitmq:rw" + attach: false + redis: + image: "redis:alpine" + restart: unless-stopped + command: + - "sh" + - "-euc" + - '/usr/local/bin/docker-entrypoint.sh --requirepass "$$(cat $$REDIS_PASSWORD_FILE)"' + secrets: + - zulip__redis_password + environment: + REDIS_PASSWORD_FILE: /run/secrets/zulip__redis_password + volumes: + - "redis:/data:rw" + attach: false + zulip: + image: "zulip/docker-zulip:11.4-0" + restart: unless-stopped + build: + context: . + ports: + - name: smtp + target: 25 + published: 25 + app_protocol: smtp + - name: http + target: 80 + published: 80 + app_protocol: http + - name: https + target: 443 + published: 443 + app_protocol: https + secrets: + - zulip__postgres_password + - zulip__memcached_password + - zulip__rabbitmq_password + - zulip__redis_password + - zulip__secret_key + - zulip__email_password + environment: + # Default hostnames; configure your application in compose.override.yaml + DB_HOST: "database" + DB_HOST_PORT: "5432" + SETTING_MEMCACHED_LOCATION: "memcached:11211" + SETTING_RABBITMQ_HOST: "rabbitmq" + SETTING_REDIS_HOST: "redis" + volumes: + - "zulip:/data:rw" + ulimits: + nofile: + soft: 1000000 + hard: 1048576 + depends_on: + - database + - memcached + - rabbitmq + - redis +volumes: + zulip: + postgresql-14: + rabbitmq: + redis: diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index a652aad46a..0000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,133 +0,0 @@ -services: - database: - image: "zulip/zulip-postgresql:14" - restart: unless-stopped - environment: - POSTGRES_DB: "zulip" - POSTGRES_USER: "zulip" - ## Note that you need to do a manual `ALTER ROLE` query if you - ## change this on a system after booting the postgres container - ## the first time on a host. Instructions are available in README.md. - POSTGRES_PASSWORD: "REPLACE_WITH_SECURE_POSTGRES_PASSWORD" - volumes: - - "postgresql-14:/var/lib/postgresql/data:rw" - memcached: - image: "memcached:alpine" - restart: unless-stopped - command: - - "sh" - - "-euc" - - | - echo 'mech_list: plain' > "$$SASL_CONF_PATH" - echo "zulip@$$HOSTNAME:$$MEMCACHED_PASSWORD" > "$$MEMCACHED_SASL_PWDB" - echo "zulip@localhost:$$MEMCACHED_PASSWORD" >> "$$MEMCACHED_SASL_PWDB" - exec memcached -S - environment: - SASL_CONF_PATH: "/home/memcache/memcached.conf" - MEMCACHED_SASL_PWDB: "/home/memcache/memcached-sasl-db" - MEMCACHED_PASSWORD: "REPLACE_WITH_SECURE_MEMCACHED_PASSWORD" - rabbitmq: - image: "rabbitmq:4.1" - restart: unless-stopped - environment: - RABBITMQ_DEFAULT_USER: "zulip" - RABBITMQ_DEFAULT_PASS: "REPLACE_WITH_SECURE_RABBITMQ_PASSWORD" - volumes: - - "rabbitmq:/var/lib/rabbitmq:rw" - redis: - image: "redis:alpine" - restart: unless-stopped - command: - - "sh" - - "-euc" - - | - echo "requirepass '$$REDIS_PASSWORD'" > /etc/redis.conf - exec redis-server /etc/redis.conf - environment: - REDIS_PASSWORD: "REPLACE_WITH_SECURE_REDIS_PASSWORD" - volumes: - - "redis:/data:rw" - zulip: - image: "zulip/docker-zulip:11.4-0" - restart: unless-stopped - build: - context: . - args: - ## Change these if you want to build zulip from a different repo/branch - ZULIP_GIT_URL: https://github.com/zulip/zulip.git - ZULIP_GIT_REF: "11.4" - ports: - - "25:25" - - "80:80" - - "443:443" - environment: - ## See https://github.com/zulip/docker-zulip#configuration for - ## details on this section and how to discover the many - ## additional settings that are supported here. - DB_HOST: "database" - DB_HOST_PORT: "5432" - DB_USER: "zulip" - SSL_CERTIFICATE_GENERATION: "self-signed" - SETTING_MEMCACHED_LOCATION: "memcached:11211" - SETTING_RABBITMQ_HOST: "rabbitmq" - SETTING_REDIS_HOST: "redis" - SECRETS_email_password: "123456789" - ## These should match RABBITMQ_DEFAULT_PASS, POSTGRES_PASSWORD, - ## MEMCACHED_PASSWORD, and REDIS_PASSWORD above. - SECRETS_rabbitmq_password: "REPLACE_WITH_SECURE_RABBITMQ_PASSWORD" - SECRETS_postgres_password: "REPLACE_WITH_SECURE_POSTGRES_PASSWORD" - SECRETS_memcached_password: "REPLACE_WITH_SECURE_MEMCACHED_PASSWORD" - SECRETS_redis_password: "REPLACE_WITH_SECURE_REDIS_PASSWORD" - SECRETS_secret_key: "REPLACE_WITH_SECURE_SECRET_KEY" - SETTING_EXTERNAL_HOST: "localhost.localdomain" - SETTING_ZULIP_ADMINISTRATOR: "admin@example.com" - SETTING_EMAIL_HOST: "" # e.g. smtp.example.com - SETTING_EMAIL_HOST_USER: "noreply@example.com" - SETTING_EMAIL_PORT: "587" - ## It seems that the email server needs to use ssl or tls and can't be used without it - SETTING_EMAIL_USE_SSL: "False" - SETTING_EMAIL_USE_TLS: "True" - ## Uncomment to enable the incoming email gateway. You will need to - ## ensure that email to emaildomain.example.com is routed to this host - ## (e.g. via MX record) - # SETTING_EMAIL_GATEWAY_PATTERN: "%s@emaildomain.example.com" - ZULIP_AUTH_BACKENDS: "EmailAuthBackend" - ## Uncomment this when configuring the mobile push notifications service - # SETTING_ZULIP_SERVICE_PUSH_NOTIFICATIONS: "True" - # SETTING_ZULIP_SERVICE_SUBMIT_USAGE_STATISTICS: "True" - - ## If you're using a reverse proxy, you'll want to provide the - ## comma-separated set of IP addresses (or CIDR ranges) to trust here. - # LOADBALANCER_IPS: "" - - ## By default, files uploaded by users and profile pictures are - ## stored directly on the Zulip server. You can configure files - ## to be stored in Amazon S3 or a compatible data store - ## here. See docs at: - ## - ## https://zulip.readthedocs.io/en/latest/production/upload-backends.html - ## - ## If you want to use the S3 backend, you must set - ## SETTING_LOCAL_UPLOADS_DIR to None as well as configuring the - ## other fields. - # SETTING_LOCAL_UPLOADS_DIR: "None" - # SETTING_S3_AUTH_UPLOADS_BUCKET: "" - # SETTING_S3_AVATAR_BUCKET: "" - # SETTING_S3_ENDPOINT_URL: "None" - # SETTING_S3_REGION: "None" - volumes: - - "zulip:/data:rw" - ulimits: - nofile: - soft: 1000000 - hard: 1048576 - depends_on: - - database - - memcached - - rabbitmq - - redis -volumes: - zulip: - postgresql-14: - rabbitmq: - redis: diff --git a/entrypoint.sh b/entrypoint.sh index 9af05eb1c2..8207f3eeae 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -9,11 +9,12 @@ set -u shopt -s extglob normalize_bool() { - # Returns either "True" or "False" + # Returns either "True" or "False", or possibly "None" if a third argument is given local varname="$1" local raw_value="${!varname:-}" local value="${raw_value,,}" # Convert to lowercase local default="${2:-False}" + local allow_none="${3:-}" case "$value" in true | enable | enabled | yes | y | 1 | on) @@ -26,8 +27,12 @@ normalize_bool() { echo "$default" ;; *) - echo "WARNING: Invalid boolean ('$raw_value') for '$varname'; defaulting to $default" >&2 - echo "$default" + if [ -n "$allow_none" ] && [ "$value" = "none" ]; then + echo "None" + else + echo "WARNING: Invalid boolean ('$raw_value') for '$varname'; defaulting to $default" >&2 + echo "$default" + fi ;; esac } @@ -137,10 +142,23 @@ setConfigurationValue() { literal) VALUE="$1" ;; - bool | boolean | int | integer | array) + bool) + # Note that if any settings were explicitly set as type + # "bool" (which none are at current), this would provide a + # slightly confusing error message with "PROVIDED_SETTING" + # in it, rather than the actual setting name. + # shellcheck disable=SC2034 + local PROVIDED_SETTING="$2" + VALUE="$KEY = $(normalize_bool PROVIDED_SETTING False allow_none)" + ;; + integer | array) VALUE="$KEY = $2" ;; - string | *) + string) + VALUE="$KEY = '${2//\'/\'}'" + ;; + *) + echo "WARNING: Unknown type '$TYPE' for '$KEY' -- treating as string." >&2 VALUE="$KEY = '${2//\'/\'}'" ;; esac @@ -276,9 +294,35 @@ secretsConfiguration() { [[ "$key" == SECRETS_*([0-9A-Z_a-z-]) ]] || continue local SECRET_KEY="${key#SECRETS_}" local SECRET_VAR="${!key}" + if [[ "$SECRET_KEY" == *"_FILE" ]]; then + SECRET_VAR="$(cat "$SECRET_VAR")" + SECRET_KEY="${SECRET_KEY%_FILE}" + fi if [ -z "$SECRET_VAR" ]; then echo "Empty secret for key \"$SECRET_KEY\"." + elif [[ "$SECRET_VAR" =~ $'\n' ]]; then + echo "ERROR: Secret \"$SECRET_KEY\" contains a newline!" + exit 1 + fi + echo "Setting $SECRET_KEY from environment variable $key" + crudini --set "$DATA_DIR/zulip-secrets.conf" "secrets" "${SECRET_KEY}" "${SECRET_VAR}" + done + # Secrets detected in /run/secrets/ override those via env vars + shopt -s nullglob + local secrets_path + for secrets_path in /run/secrets/zulip__*; do + local secrets_filename + secrets_filename="$(basename "$secrets_path")" + local SECRET_KEY="${secrets_filename#zulip__}" + local SECRET_VAR + SECRET_VAR="$(cat "$secrets_path")" + if [ -z "$SECRET_VAR" ]; then + echo "Empty secret for key \"$SECRET_KEY\"." + elif [[ "$SECRET_VAR" =~ $'\n' ]]; then + echo "ERROR: Secret \"$SECRET_KEY\" contains a newline!" + exit 1 fi + echo "Setting $SECRET_KEY from secret in $secrets_path" crudini --set "$DATA_DIR/zulip-secrets.conf" "secrets" "${SECRET_KEY}" "${SECRET_VAR}" done echo "Zulip secrets configuration succeeded." @@ -353,7 +397,7 @@ zulipConfiguration() { fi setConfigurationValue "$setting_key" "$setting_var" "$type" done - if ! su zulip -c "/home/zulip/deployments/current/manage.py checkconfig"; then + if ! su zulip -c "/home/zulip/deployments/current/manage.py check"; then echo "Error in the Zulip configuration. Exiting." exit 1 fi @@ -404,7 +448,7 @@ initialConfiguration() { ls -l /etc/zulip/ exit 1 fi - if ! su zulip -c "/home/zulip/deployments/current/manage.py checkconfig"; then + if ! su zulip -c "/home/zulip/deployments/current/manage.py check"; then echo "Error in the Zulip configuration. Exiting." exit 1 fi @@ -416,7 +460,9 @@ initialConfiguration() { waitingForDatabase() { local TIMEOUT=60 echo "Waiting for database server to allow connections ..." - while ! PGPASSWORD="${SECRETS_postgres_password?}" /usr/bin/pg_isready -h "$DB_HOST" -p "$DB_HOST_PORT" -U "$DB_USER" -t 1 >/dev/null 2>&1; do + local PGPASSWORD + PGPASSWORD="$(crudini --get /etc/zulip/zulip-secrets.conf secrets postgres_password)" + while ! PGPASSWORD="$PGPASSWORD" /usr/bin/pg_isready -h "$DB_HOST" -p "$DB_HOST_PORT" -U "$DB_USER" -t 1 >/dev/null 2>&1; do if ! ((TIMEOUT--)); then echo "Could not connect to database server. Exiting." exit 1