From 11eb89e45c8a6d4a5b07134b8957a1298f9305c8 Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Wed, 2 Nov 2022 20:52:54 -0700 Subject: [PATCH] feat(stac-db): Provide sensible defaults for pgSTAC db parameters (#1) --- lib/database/index.ts | 104 +++++- lib/database/instance-memory.json | 525 ++++++++++++++++++++++++++++++ 2 files changed, 626 insertions(+), 3 deletions(-) create mode 100644 lib/database/instance-memory.json diff --git a/lib/database/index.ts b/lib/database/index.ts index 214436d..429e83d 100644 --- a/lib/database/index.ts +++ b/lib/database/index.ts @@ -6,6 +6,8 @@ import { import { Construct } from "constructs"; import { BootstrapPgStac, BootstrapPgStacProps } from "../bootstrapper"; +const instanceSizes: Record = require("./instance-memory.json"); + /** * An RDS instance with pgSTAC installed. This is a wrapper around the * `rds.DatabaseInstance` higher-level construct making use @@ -18,12 +20,22 @@ export class PgStacDatabase extends Construct { constructor(scope: Construct, id: string, props: PgStacDatabaseProps) { super(scope, id); + const defaultParameters = this.getParameters( + props.instanceType?.toString() || "m5.large", + props.parameters + ); const parameterGroup = new rds.ParameterGroup(this, "parameterGroup", { engine: props.engine, parameters: { - max_locks_per_transaction: "1024", - work_mem: "64000", - temp_buffers: "32000", + shared_buffers: defaultParameters.sharedBuffers, + effective_cache_size: defaultParameters.effectiveCacheSize, + work_mem: defaultParameters.workMem, + maintenance_work_mem: defaultParameters.maintenanceWorkMem, + max_locks_per_transaction: defaultParameters.maxLocksPerTransaction, + temp_buffers: defaultParameters.tempBuffers, + seq_page_cost: defaultParameters.seqPageCost, + random_page_cost: defaultParameters.randomPageCost, + ...props.parameters, }, }); @@ -45,6 +57,45 @@ export class PgStacDatabase extends Construct { this.pgstacSecret = bootstrap.secret; } + + public getParameters( + instanceType: string, + parameters: PgStacDatabaseProps["parameters"] + ): DatabaseParameters { + // https://github.com/aws/aws-cli/issues/1279#issuecomment-909318236 + const memory_in_kb = instanceSizes[instanceType] * 1024; + + // It's only necessary to consider passed in parameters for any value that used to + // derive subsequent values. Values that don't have dependencies will be overriden + // when we unpack the passed-in user parameters + const maxConnections = parameters?.maxConnections + ? Number.parseInt(parameters.maxConnections) + : // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.MaxConnections + Math.min(Math.round((memory_in_kb * 1024) / 9531392), 5000); + const sharedBuffers = parameters?.sharedBufers + ? Number.parseInt(parameters.sharedBufers) + : Math.round(0.25 * memory_in_kb); + + const effectiveCacheSize = Math.round(0.75 * memory_in_kb); + const workMem = Math.floor(sharedBuffers / maxConnections); + const maintenanceWorkMem = Math.round(0.25 * sharedBuffers); + + const tempBuffers = 128 * 1024; + const seqPageCost = 1; + const randomPageCost = 1.1; + + return { + maxConnections: `${maxConnections}`, + sharedBuffers: `${sharedBuffers}`, + effectiveCacheSize: `${effectiveCacheSize}`, + workMem: `${workMem}`, + maintenanceWorkMem: `${maintenanceWorkMem}`, + maxLocksPerTransaction: "1024", + tempBuffers: `${tempBuffers}`, + seqPageCost: `${seqPageCost}`, + randomPageCost: `${randomPageCost}`, + }; + } } export interface PgStacDatabaseProps extends rds.DatabaseInstanceProps { @@ -53,3 +104,50 @@ export interface PgStacDatabaseProps extends rds.DatabaseInstanceProps { readonly pgstacUsername?: BootstrapPgStacProps["pgstacUsername"]; readonly secretsPrefix?: BootstrapPgStacProps["secretsPrefix"]; } + +export interface DatabaseParameters { + /** + * @default - LEAST({DBInstanceClassMemory/9531392}, 5000) + */ + readonly maxConnections: string; + + /** + * @default - 25% of instance memory + */ + readonly sharedBuffers: string; + + /** + * @default - 75% of instance memory + */ + readonly effectiveCacheSize: string; + + /** + * @default - shared buffers divided by max connections + */ + readonly workMem: string; + + /** + * @default - 25% of shared buffers + */ + readonly maintenanceWorkMem: string; + + /** + * @default 1024 + */ + readonly maxLocksPerTransaction: string; + + /** + * @default 131172 (128 * 1024) + */ + readonly tempBuffers: string; + + /** + * @default 1 + */ + readonly seqPageCost: string; + + /** + * @default 1.1 + */ + readonly randomPageCost: string; +} diff --git a/lib/database/instance-memory.json b/lib/database/instance-memory.json new file mode 100644 index 0000000..a9be844 --- /dev/null +++ b/lib/database/instance-memory.json @@ -0,0 +1,525 @@ +{ + "c6g.16xlarge": 131072, + "x2gd.metal": 1048576, + "c6a.16xlarge": 131072, + "c5ad.12xlarge": 98304, + "r6a.metal": 1572864, + "is4gen.4xlarge": 98304, + "m6gd.xlarge": 16384, + "m6gd.4xlarge": 65536, + "d3en.2xlarge": 32768, + "c7g.8xlarge": 65536, + "x2gd.8xlarge": 524288, + "m6a.2xlarge": 32768, + "d2.8xlarge": 249856, + "i3en.3xlarge": 98304, + "r6a.2xlarge": 65536, + "r6a.xlarge": 32768, + "m5dn.large": 8192, + "m5a.4xlarge": 65536, + "c6gd.metal": 131072, + "m6id.2xlarge": 32768, + "r5dn.8xlarge": 262144, + "r5a.xlarge": 32768, + "x2iedn.2xlarge": 262144, + "c5ad.24xlarge": 196608, + "m5zn.6xlarge": 98304, + "r6g.medium": 8192, + "im4gn.2xlarge": 32768, + "t4g.nano": 512, + "r6g.large": 16384, + "t2.2xlarge": 32768, + "c5ad.4xlarge": 32768, + "m4.4xlarge": 65536, + "c6id.8xlarge": 65536, + "r5n.12xlarge": 393216, + "g3.16xlarge": 499712, + "r6id.16xlarge": 524288, + "c6a.48xlarge": 393216, + "m5a.2xlarge": 32768, + "g5.4xlarge": 65536, + "m5n.16xlarge": 262144, + "r6gd.xlarge": 32768, + "c5d.large": 4096, + "m6id.xlarge": 16384, + "x2iezn.4xlarge": 524288, + "i4i.16xlarge": 524288, + "c4.4xlarge": 30720, + "m5dn.2xlarge": 32768, + "r6i.xlarge": 32768, + "m5ad.12xlarge": 196608, + "m6gd.12xlarge": 196608, + "m5a.16xlarge": 262144, + "i3en.12xlarge": 393216, + "c6i.2xlarge": 16384, + "trn1.32xlarge": 524288, + "m6id.12xlarge": 196608, + "r6gd.medium": 8192, + "m5d.8xlarge": 131072, + "m5dn.12xlarge": 196608, + "r4.16xlarge": 499712, + "z1d.2xlarge": 65536, + "r6id.xlarge": 32768, + "r5.24xlarge": 786432, + "is4gen.xlarge": 24576, + "x1e.8xlarge": 999424, + "c6gn.16xlarge": 131072, + "m6a.large": 8192, + "r6id.2xlarge": 65536, + "r6g.16xlarge": 524288, + "r6g.8xlarge": 262144, + "c6i.32xlarge": 262144, + "r5ad.24xlarge": 786432, + "c6i.large": 4096, + "c5n.2xlarge": 21504, + "c6a.24xlarge": 196608, + "im4gn.8xlarge": 131072, + "x1e.2xlarge": 249856, + "r6gd.2xlarge": 65536, + "i3en.24xlarge": 786432, + "m6i.large": 8192, + "c4.8xlarge": 61440, + "m5zn.large": 8192, + "x1e.32xlarge": 3997696, + "c6i.8xlarge": 65536, + "m5zn.2xlarge": 32768, + "r5a.2xlarge": 65536, + "x2iedn.xlarge": 131072, + "g5.2xlarge": 32768, + "m6gd.medium": 4096, + "m5d.xlarge": 16384, + "f1.2xlarge": 124928, + "d3en.8xlarge": 131072, + "m5.8xlarge": 131072, + "c6i.4xlarge": 32768, + "x1.16xlarge": 999424, + "m4.xlarge": 16384, + "r5.large": 16384, + "c5d.12xlarge": 98304, + "g4dn.12xlarge": 196608, + "m6g.xlarge": 16384, + "m6i.24xlarge": 393216, + "r6gd.metal": 524288, + "t3.xlarge": 16384, + "m5ad.4xlarge": 65536, + "c7g.16xlarge": 131072, + "c5a.4xlarge": 32768, + "r5a.16xlarge": 524288, + "m5d.large": 8192, + "u-3tb1.56xlarge": 3145728, + "c7g.4xlarge": 32768, + "r6gd.large": 16384, + "r6i.4xlarge": 131072, + "p2.8xlarge": 499712, + "c7g.xlarge": 8192, + "m5d.metal": 393216, + "z1d.xlarge": 32768, + "g3.8xlarge": 249856, + "m5n.8xlarge": 131072, + "m6i.8xlarge": 131072, + "i4i.metal": 1048576, + "c5d.2xlarge": 16384, + "r6id.12xlarge": 393216, + "m6gd.metal": 262144, + "c5n.large": 5376, + "m5dn.8xlarge": 131072, + "h1.8xlarge": 131072, + "g4ad.4xlarge": 65536, + "c6id.24xlarge": 196608, + "m5dn.xlarge": 16384, + "im4gn.xlarge": 16384, + "x2iedn.metal": 4194304, + "trn1.2xlarge": 32768, + "t3.medium": 4096, + "t3a.small": 2048, + "x2gd.4xlarge": 262144, + "c6g.xlarge": 8192, + "c6gn.xlarge": 8192, + "c4.large": 3840, + "m5.metal": 393216, + "c5ad.16xlarge": 131072, + "c5a.12xlarge": 98304, + "r6a.32xlarge": 1048576, + "r6id.4xlarge": 131072, + "m6a.metal": 786432, + "c5ad.xlarge": 8192, + "m5.xlarge": 16384, + "m5.24xlarge": 393216, + "g4ad.2xlarge": 32768, + "t3a.2xlarge": 32768, + "t4g.medium": 4096, + "r5ad.12xlarge": 393216, + "g4dn.metal": 393216, + "t3.2xlarge": 32768, + "r5a.large": 16384, + "inf1.xlarge": 8192, + "g4ad.8xlarge": 131072, + "t4g.2xlarge": 32768, + "c5a.xlarge": 8192, + "d2.2xlarge": 62464, + "r6g.4xlarge": 131072, + "m5ad.large": 8192, + "g5.24xlarge": 393216, + "x2idn.24xlarge": 1572864, + "t2.large": 8192, + "c6gn.4xlarge": 32768, + "r5n.4xlarge": 131072, + "c5n.4xlarge": 43008, + "r5a.12xlarge": 393216, + "d3.8xlarge": 262144, + "x2iezn.8xlarge": 1048576, + "c6i.24xlarge": 196608, + "i4i.4xlarge": 131072, + "r6a.12xlarge": 393216, + "c6id.large": 4096, + "t3.large": 8192, + "m5.12xlarge": 196608, + "m5ad.2xlarge": 32768, + "r5b.metal": 786432, + "r5ad.8xlarge": 262144, + "r6i.24xlarge": 786432, + "r5b.12xlarge": 393216, + "m5d.16xlarge": 262144, + "r6a.8xlarge": 262144, + "m6id.32xlarge": 524288, + "c5.4xlarge": 32768, + "m5ad.8xlarge": 131072, + "m6g.metal": 262144, + "c5.xlarge": 8192, + "c6g.12xlarge": 98304, + "c6a.32xlarge": 262144, + "r5.4xlarge": 131072, + "m5zn.3xlarge": 49152, + "vt1.3xlarge": 24576, + "c5.24xlarge": 196608, + "m5n.12xlarge": 196608, + "r5ad.large": 16384, + "t2.micro": 1024, + "m5.4xlarge": 65536, + "d3en.6xlarge": 98304, + "r5dn.xlarge": 32768, + "c6a.8xlarge": 65536, + "u-12tb1.112xlarge": 12582912, + "r5d.large": 16384, + "p3.2xlarge": 62464, + "x2gd.xlarge": 65536, + "m5n.large": 8192, + "r6id.32xlarge": 1048576, + "inf1.6xlarge": 49152, + "g5g.16xlarge": 131072, + "g5g.2xlarge": 16384, + "u-6tb1.56xlarge": 6291456, + "m6i.32xlarge": 524288, + "m6i.16xlarge": 262144, + "r5b.24xlarge": 786432, + "h1.4xlarge": 65536, + "c6a.metal": 393216, + "t3a.medium": 4096, + "m5zn.12xlarge": 196608, + "m4.large": 8192, + "r5b.xlarge": 32768, + "x2gd.medium": 16384, + "r5a.4xlarge": 131072, + "r5.metal": 786432, + "r5n.24xlarge": 786432, + "r5.16xlarge": 524288, + "mac1.metal": 32768, + "r5b.4xlarge": 131072, + "c6a.12xlarge": 98304, + "m5n.metal": 393216, + "d3.2xlarge": 65536, + "x2gd.2xlarge": 131072, + "i4i.2xlarge": 65536, + "vt1.6xlarge": 49152, + "r5d.16xlarge": 524288, + "r5dn.large": 16384, + "c6i.16xlarge": 131072, + "m6gd.16xlarge": 262144, + "c5a.24xlarge": 196608, + "r5dn.metal": 786432, + "c6gn.medium": 2048, + "t4g.small": 2048, + "c6id.16xlarge": 131072, + "r5b.large": 16384, + "r5d.xlarge": 32768, + "c5d.4xlarge": 32768, + "r5n.16xlarge": 524288, + "c5d.24xlarge": 196608, + "r6id.24xlarge": 786432, + "c6g.large": 4096, + "d3.xlarge": 32768, + "p4d.24xlarge": 1179648, + "t4g.large": 8192, + "r6a.48xlarge": 1572864, + "c5.12xlarge": 98304, + "c6gd.16xlarge": 131072, + "inf1.2xlarge": 16384, + "mac2.metal": 16384, + "m6g.8xlarge": 131072, + "r5b.2xlarge": 65536, + "d3en.4xlarge": 65536, + "m6g.4xlarge": 65536, + "m6g.large": 8192, + "c5n.9xlarge": 98304, + "c5a.2xlarge": 16384, + "r5d.4xlarge": 131072, + "c6gd.xlarge": 8192, + "r5n.2xlarge": 65536, + "r5d.2xlarge": 65536, + "x2idn.metal": 2097152, + "x2gd.16xlarge": 1048576, + "h1.2xlarge": 32768, + "m6a.4xlarge": 65536, + "r6gd.4xlarge": 131072, + "g5.12xlarge": 196608, + "m4.16xlarge": 262144, + "r6g.12xlarge": 393216, + "m5n.4xlarge": 65536, + "m5d.24xlarge": 393216, + "c6gn.12xlarge": 98304, + "m6i.metal": 524288, + "g3.4xlarge": 124928, + "g4dn.xlarge": 16384, + "m6gd.8xlarge": 131072, + "c6a.4xlarge": 32768, + "m5a.8xlarge": 131072, + "r5dn.12xlarge": 393216, + "t3.nano": 512, + "m5ad.24xlarge": 393216, + "c5.large": 4096, + "c6a.xlarge": 8192, + "r6gd.16xlarge": 524288, + "i3.large": 15616, + "x2gd.large": 32768, + "r6a.24xlarge": 786432, + "i4i.large": 16384, + "m5dn.16xlarge": 262144, + "g5.16xlarge": 262144, + "g5g.8xlarge": 65536, + "m4.2xlarge": 32768, + "x2iezn.2xlarge": 262144, + "r6id.metal": 1048576, + "c5a.16xlarge": 131072, + "m6i.4xlarge": 65536, + "t4g.xlarge": 16384, + "is4gen.large": 12288, + "r5n.xlarge": 32768, + "c7g.2xlarge": 16384, + "i3.8xlarge": 249856, + "c5n.18xlarge": 196608, + "x2iezn.12xlarge": 1572864, + "r5dn.16xlarge": 524288, + "m5zn.xlarge": 16384, + "r5ad.16xlarge": 524288, + "r6gd.8xlarge": 262144, + "x2idn.16xlarge": 1048576, + "r5ad.xlarge": 32768, + "r6a.4xlarge": 131072, + "r5ad.2xlarge": 65536, + "c5.9xlarge": 73728, + "c4.2xlarge": 15360, + "c5d.18xlarge": 147456, + "z1d.metal": 393216, + "r4.xlarge": 31232, + "c5d.xlarge": 8192, + "c4.xlarge": 7680, + "r6id.large": 16384, + "t2.medium": 4096, + "r6i.16xlarge": 524288, + "r5.2xlarge": 65536, + "x2gd.12xlarge": 786432, + "r5b.16xlarge": 524288, + "u-6tb1.112xlarge": 6291456, + "c5n.metal": 196608, + "m6g.12xlarge": 196608, + "r5.xlarge": 32768, + "g5g.metal": 131072, + "m5dn.4xlarge": 65536, + "x2iedn.4xlarge": 524288, + "x2iedn.16xlarge": 2097152, + "i3.xlarge": 31232, + "c6id.metal": 262144, + "t3a.large": 8192, + "m5d.12xlarge": 196608, + "m5zn.metal": 196608, + "m6g.16xlarge": 262144, + "m6id.metal": 524288, + "t4g.micro": 1024, + "c5ad.2xlarge": 16384, + "r5.8xlarge": 262144, + "is4gen.2xlarge": 49152, + "c5.metal": 196608, + "m5.16xlarge": 262144, + "m6id.16xlarge": 262144, + "z1d.12xlarge": 393216, + "g5.xlarge": 16384, + "m5dn.metal": 393216, + "m5ad.xlarge": 16384, + "m6a.16xlarge": 262144, + "h1.16xlarge": 262144, + "r6i.metal": 1048576, + "t3.micro": 1024, + "r4.2xlarge": 62464, + "r5d.8xlarge": 262144, + "r6i.12xlarge": 393216, + "r5d.24xlarge": 786432, + "x2idn.32xlarge": 2097152, + "m5d.2xlarge": 32768, + "c5d.metal": 196608, + "c7g.medium": 2048, + "c6id.32xlarge": 262144, + "c5.2xlarge": 16384, + "x1e.16xlarge": 1998848, + "c6gn.2xlarge": 16384, + "c6gd.2xlarge": 16384, + "i4i.32xlarge": 1048576, + "c6g.medium": 2048, + "x2iezn.metal": 1572864, + "m5.2xlarge": 32768, + "c6g.8xlarge": 65536, + "c5ad.large": 4096, + "c6gn.8xlarge": 65536, + "r4.8xlarge": 249856, + "m5dn.24xlarge": 393216, + "i3.2xlarge": 62464, + "inf1.24xlarge": 196608, + "c6gn.large": 4096, + "x2iedn.32xlarge": 4194304, + "i3.metal": 524288, + "r6i.32xlarge": 1048576, + "m5n.xlarge": 16384, + "g4dn.16xlarge": 262144, + "m5n.2xlarge": 32768, + "c6id.xlarge": 8192, + "c7g.large": 4096, + "r6g.2xlarge": 65536, + "c6i.xlarge": 8192, + "c6i.12xlarge": 98304, + "c6gd.4xlarge": 32768, + "c6a.2xlarge": 16384, + "r5d.12xlarge": 393216, + "m6id.24xlarge": 393216, + "r6id.8xlarge": 262144, + "x1e.4xlarge": 499712, + "m5n.24xlarge": 393216, + "c5a.8xlarge": 65536, + "m5ad.16xlarge": 262144, + "c6id.2xlarge": 16384, + "c6id.12xlarge": 98304, + "m6a.32xlarge": 524288, + "r5dn.4xlarge": 131072, + "t2.small": 2048, + "m6a.xlarge": 16384, + "g3s.xlarge": 31232, + "r5d.metal": 786432, + "im4gn.4xlarge": 65536, + "r5ad.4xlarge": 131072, + "r6g.xlarge": 32768, + "r4.large": 15616, + "i3en.2xlarge": 65536, + "g5.8xlarge": 131072, + "r5b.8xlarge": 262144, + "i3.16xlarge": 499712, + "d3.4xlarge": 131072, + "r6a.large": 16384, + "m6i.xlarge": 16384, + "r5.12xlarge": 393216, + "r5a.8xlarge": 262144, + "dl1.24xlarge": 786432, + "im4gn.large": 8192, + "p3.8xlarge": 249856, + "i3en.large": 16384, + "t3a.nano": 512, + "c6i.metal": 262144, + "i3en.xlarge": 32768, + "c6g.4xlarge": 32768, + "m5a.large": 8192, + "r6gd.12xlarge": 393216, + "m6id.4xlarge": 65536, + "p2.16xlarge": 749568, + "t2.xlarge": 16384, + "r6a.16xlarge": 524288, + "r5dn.2xlarge": 65536, + "r5n.8xlarge": 262144, + "r5a.24xlarge": 786432, + "i4i.xlarge": 32768, + "c7g.12xlarge": 98304, + "d2.4xlarge": 124928, + "m6a.24xlarge": 393216, + "p2.xlarge": 62464, + "is4gen.medium": 6144, + "t3.small": 2048, + "p3.16xlarge": 499712, + "m6id.8xlarge": 131072, + "im4gn.16xlarge": 262144, + "c5ad.8xlarge": 65536, + "x2iezn.6xlarge": 786432, + "r6i.8xlarge": 262144, + "c6gd.medium": 2048, + "vt1.24xlarge": 196608, + "r6g.metal": 524288, + "g4ad.16xlarge": 262144, + "m4.10xlarge": 163840, + "m5.large": 8192, + "z1d.6xlarge": 196608, + "x2iedn.24xlarge": 3145728, + "x1e.xlarge": 124928, + "m6gd.2xlarge": 32768, + "z1d.large": 16384, + "x1.32xlarge": 1998848, + "m6a.48xlarge": 786432, + "g5g.xlarge": 8192, + "u-18tb1.112xlarge": 18874368, + "g4dn.4xlarge": 65536, + "c5a.large": 4096, + "g5.48xlarge": 786432, + "c6id.4xlarge": 32768, + "c6a.large": 4096, + "c5d.9xlarge": 73728, + "r6i.large": 16384, + "c6gd.12xlarge": 98304, + "c6gd.large": 4096, + "u-9tb1.112xlarge": 9437184, + "t3a.micro": 1024, + "m6a.8xlarge": 131072, + "d3en.12xlarge": 196608, + "m5a.12xlarge": 196608, + "c5n.xlarge": 10752, + "m5d.4xlarge": 65536, + "p3dn.24xlarge": 786432, + "i3en.6xlarge": 196608, + "m6i.2xlarge": 32768, + "i3.4xlarge": 124928, + "is4gen.8xlarge": 196608, + "x2iedn.8xlarge": 1048576, + "t2.nano": 512, + "c5.18xlarge": 147456, + "r4.4xlarge": 124928, + "r6i.2xlarge": 65536, + "f1.16xlarge": 999424, + "m5a.xlarge": 16384, + "m6gd.large": 8192, + "t3a.xlarge": 16384, + "i3en.metal": 786432, + "g4dn.2xlarge": 32768, + "d3en.xlarge": 16384, + "r5dn.24xlarge": 786432, + "c6gd.8xlarge": 65536, + "m5a.24xlarge": 393216, + "i4i.8xlarge": 262144, + "m6id.large": 8192, + "z1d.3xlarge": 98304, + "c6g.metal": 131072, + "r5n.large": 16384, + "f1.4xlarge": 249856, + "d2.xlarge": 31232, + "m6i.12xlarge": 196608, + "r5n.metal": 786432, + "m6a.12xlarge": 196608, + "g4dn.8xlarge": 131072, + "m6g.2xlarge": 32768, + "g4ad.xlarge": 16384, + "m6g.medium": 4096, + "c6g.2xlarge": 16384, + "g5g.4xlarge": 32768 +}