From 1bc48a02983b6de196b269ddee528c2bce316438 Mon Sep 17 00:00:00 2001 From: Jonathan Gray Date: Wed, 15 May 2024 11:17:26 +0100 Subject: [PATCH 1/3] Switch default log format to JSON --- CHANGELOG.md | 2 ++ flowdb/bin/build/pg_config_template.conf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9859e6d387..36f3ace268 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - FlowDB now triggers an ANALYZE on newly created cache tables to generate statistics rather than waiting for autovacuum +- FlowDB now produces JSON formatted logs by default + ### Fixed diff --git a/flowdb/bin/build/pg_config_template.conf b/flowdb/bin/build/pg_config_template.conf index b3fc25876d..a9041a46ed 100644 --- a/flowdb/bin/build/pg_config_template.conf +++ b/flowdb/bin/build/pg_config_template.conf @@ -20,7 +20,7 @@ # # Logging options # -log_destination = 'csvlog' +log_destination = 'jsonlog' log_directory = 'pg_log' logging_collector = 'on' log_filename = 'postgres-%Y-%m-%d_%H%M%S' From 9491b059968c1038909b90142a54ee86d75a3946 Mon Sep 17 00:00:00 2001 From: Jonathan Gray Date: Wed, 15 May 2024 11:33:17 +0100 Subject: [PATCH 2/3] Allow configuring log destination --- .circleci/config.yml | 1 + CHANGELOG.md | 3 ++- flowdb.Dockerfile | 3 +++ flowdb/bin/build/configurate.py | 9 +++++++++ flowdb/bin/build/pg_config_template.conf | 4 ++-- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b7a384bdd..72c69f09bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -159,6 +159,7 @@ executors: SHARED_BUFFERS_SIZE: "1GB" EFFECTIVE_CACHE_SIZE: "1GB" STATS_TARGET: 100 + FLOWDB_LOG_DEST: stderr - image: bitnami/redis:latest environment: REDIS_PASSWORD: "fm_redis" diff --git a/CHANGELOG.md b/CHANGELOG.md index 36f3ace268..80e56c54af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - FlowDB now triggers an ANALYZE on newly created cache tables to generate statistics rather than waiting for autovacuum -- FlowDB now produces JSON formatted logs by default +- FlowDB now produces JSON formatted logs by default. Set `FLOWDB_LOG_DEST=csvlog` for the old default behaviour. +- The logging destination of FlowDB can now be configured at init by setting the `FLOWDB_LOG_DEST` environment variable, valid options are `stderr`, `csvlog`, and `jsonlog`. ### Fixed diff --git a/flowdb.Dockerfile b/flowdb.Dockerfile index 97612ee7b7..faf0bacbaa 100644 --- a/flowdb.Dockerfile +++ b/flowdb.Dockerfile @@ -139,6 +139,9 @@ ENV FLOWAPI_FLOWDB_USER=flowapi # Default location table ENV LOCATION_TABLE=infrastructure.cells +# Default logging destination +ENV FLOWDB_LOG_DEST=jsonlog + # # Copy file spinup build scripts to be execed. # diff --git a/flowdb/bin/build/configurate.py b/flowdb/bin/build/configurate.py index bedc397c85..635ab4e7f7 100644 --- a/flowdb/bin/build/configurate.py +++ b/flowdb/bin/build/configurate.py @@ -80,6 +80,13 @@ def bool_env(var): if bool_env("FLOWDB_ENABLE_POSTGRES_DEBUG_MODE"): preload_libraries.append("plugin_debugger") +possible_log_destinations = ["stderr", "jsonlog", "csvlog"] +log_destination = os.getenv("FLOWDB_LOG_DEST", "jsonlog").lower() +if log_destination not in possible_log_destinations: + raise ValueError( + f"Invalid log destination. Valid values for FLOWDB_LOG_DEST are {possible_log_destinations}" + ) + with open("/docker-entrypoint-initdb.d/pg_config_template.conf") as fin: config_file = fin.read().format( cores=cores, @@ -92,6 +99,8 @@ def bool_env(var): stats_target=stats_target, use_jit=use_jit, max_locks=max_locks, + log_destination=log_destination, + collecter_on="on" if log_destination != "stderr" else "off", ) print("Writing config file to", config_path) diff --git a/flowdb/bin/build/pg_config_template.conf b/flowdb/bin/build/pg_config_template.conf index a9041a46ed..9a5e3babc4 100644 --- a/flowdb/bin/build/pg_config_template.conf +++ b/flowdb/bin/build/pg_config_template.conf @@ -20,9 +20,9 @@ # # Logging options # -log_destination = 'jsonlog' +log_destination = '{log_destination}' log_directory = 'pg_log' -logging_collector = 'on' +logging_collector = '{collecter_on}' log_filename = 'postgres-%Y-%m-%d_%H%M%S' log_rotation_age = 1d log_rotation_size = 1GB From 36a332031a2ef97c768fc1605d538c501dcc0135 Mon Sep 17 00:00:00 2001 From: Jonathan Gray Date: Wed, 15 May 2024 13:50:50 +0100 Subject: [PATCH 3/3] Update docs --- docs/source/administrator/deployment.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/source/administrator/deployment.md b/docs/source/administrator/deployment.md index 877c793d0a..8f8e94ff02 100644 --- a/docs/source/administrator/deployment.md +++ b/docs/source/administrator/deployment.md @@ -29,18 +29,19 @@ FlowDB is distributed as a docker container. To run it, you will need to provide You may also provide the following environment variables: -| Variable name | Purpose | Default value | -|-----------------------------------| ------- | ------------- | -| CACHE_SIZE | Maximum size of the cache schema | 1 tenth of available space in pgdata directory | -| CACHE_PROTECTED_PERIOD | Amount of time to protect cache tables from being cleaned up | 86400 (24 hours) | -| CACHE_HALF_LIFE | Speed at which cache tables expire when not used | 1000 | -| MAX_CPUS | Maximum number of CPUs that may be used for parallelising queries | The greater of 1 or 1 less than all CPUs | -| SHARED_BUFFERS_SIZE | Size of shared buffers | 16GB | -| MAX_WORKERS | Maximum number of CPUs that may be used for parallelising one query | MAX_CPUS/2 | -| MAX_WORKERS_PER_GATHER | Maximum number of CPUs that may be used for parallelising part of one query | MAX_CPUS/2 | -| EFFECTIVE_CACHE_SIZE | Postgres cache size | 25% of total RAM | -| FLOWDB_ENABLE_POSTGRES_DEBUG_MODE | When set to TRUE, enables use of the [pgadmin debugger](https://www.pgadmin.org/docs/pgadmin4/4.13/debugger.html) | FALSE | - | MAX_LOCKS_PER_TRANSACTION | Controls the maximum number of locks one transaction can take, you may wish to reduce this on low-memory servers. | 36500 | +| Variable name | Purpose | Default value | +|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|------------------------------------------------| +| CACHE_SIZE | Maximum size of the cache schema | 1 tenth of available space in pgdata directory | +| CACHE_PROTECTED_PERIOD | Amount of time to protect cache tables from being cleaned up | 86400 (24 hours) | +| CACHE_HALF_LIFE | Speed at which cache tables expire when not used | 1000 | +| MAX_CPUS | Maximum number of CPUs that may be used for parallelising queries | The greater of 1 or 1 less than all CPUs | +| SHARED_BUFFERS_SIZE | Size of shared buffers | 16GB | +| MAX_WORKERS | Maximum number of CPUs that may be used for parallelising one query | MAX_CPUS/2 | +| MAX_WORKERS_PER_GATHER | Maximum number of CPUs that may be used for parallelising part of one query | MAX_CPUS/2 | +| EFFECTIVE_CACHE_SIZE | Postgres cache size | 25% of total RAM | +| FLOWDB_ENABLE_POSTGRES_DEBUG_MODE | When set to TRUE, enables use of the [pgadmin debugger](https://www.pgadmin.org/docs/pgadmin4/4.13/debugger.html) | FALSE | +| MAX_LOCKS_PER_TRANSACTION | Controls the maximum number of locks one transaction can take, you may wish to reduce this on low-memory servers. | 36500 | +| FLOWDB_LOG_DEST | Controls the logging destination, may be be one of `stderr`, `jsonlog`, `csvlog`. | `jsonlog` | However in most cases, the defaults will be adequate.