From 2d1452fecf29141afca720a4bd4f09868b05dbcb Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 16 Mar 2023 23:54:44 -0500 Subject: [PATCH] imp: adds basic metadata server, sync agent, webhook --- flake.lock | 7 +- flake.nix | 2 +- nix/cardano/entrypoints.nix | 72 ++++++ nix/cardano/hydrationProfiles.nix | 16 ++ nix/cardano/nomadCharts/default.nix | 1 + nix/cardano/nomadCharts/metadata.nix | 228 ++++++++++++++++++ .../nomadCharts/srv-metadata-server.nix | 18 ++ .../nomadCharts/srv-metadata-webhook.nix | 18 ++ nix/cardano/oci-images.nix | 44 ++++ nix/cloud/hydrationProfiles.nix | 1 + nix/cloud/kv/vault/.sops.yaml | 2 + nix/cloud/kv/vault/metadata/infra.enc.yaml | 22 ++ nix/cloud/namespaces/infra.nix | 12 +- 13 files changed, 438 insertions(+), 5 deletions(-) create mode 100644 nix/cardano/nomadCharts/metadata.nix create mode 100644 nix/cardano/nomadCharts/srv-metadata-server.nix create mode 100644 nix/cardano/nomadCharts/srv-metadata-webhook.nix create mode 100644 nix/cloud/kv/vault/metadata/infra.enc.yaml diff --git a/flake.lock b/flake.lock index 9ae2e0b4a1..2060f99d00 100644 --- a/flake.lock +++ b/flake.lock @@ -25949,15 +25949,16 @@ "offchain-metadata-tools": { "flake": false, "locked": { - "lastModified": 1670417282, - "narHash": "sha256-FtKcQ0EfwJNg34GNLTCCNHmJyeKqtzhU+Fa261WIQSI=", + "lastModified": 1679016345, + "narHash": "sha256-I/M2t7K1Kjlpu0uf3BEwnGFoqR6VX72rhq1peiaEv84=", "owner": "input-output-hk", "repo": "offchain-metadata-tools", - "rev": "a73f2c8d93518470feae798b4d9f4a33906fadfc", + "rev": "7777fbda679fedd43fa90beeb1595be430154a02", "type": "github" }, "original": { "owner": "input-output-hk", + "ref": "pg-cli-mods", "repo": "offchain-metadata-tools", "type": "github" } diff --git a/flake.nix b/flake.nix index 3b63869b20..b451569f11 100644 --- a/flake.nix +++ b/flake.nix @@ -70,7 +70,7 @@ cardano-node.url = "github:input-output-hk/cardano-node/1.35.6"; cardano-wallet.url = "github:input-output-hk/cardano-wallet/v2022-07-01"; offchain-metadata-tools = { - url = "github:input-output-hk/offchain-metadata-tools"; + url = "github:input-output-hk/offchain-metadata-tools/pg-cli-mods"; flake = false; }; diff --git a/nix/cardano/entrypoints.nix b/nix/cardano/entrypoints.nix index c85e97616e..af7e4ff9d4 100644 --- a/nix/cardano/entrypoints.nix +++ b/nix/cardano/entrypoints.nix @@ -604,4 +604,76 @@ in { exec ${packages.cardano-new-faucet}/bin/cardano-new-faucet ''; }; + + metadata-server = writeShellApplication { + runtimeInputs = prelude-runtime; + debugInputs = []; + name = "entrypoint"; + text = '' + # Build args array + args+=("--db" "$PG_DB") + args+=("--db-user" "$PG_USER") + args+=("--db-pass" "$PG_PASS") + args+=("--db-host" "$PG_HOST") + args+=("--db-table" "$PG_TABLE") + args+=("--db-conns" "$PG_NUM_CONNS") + args+=("--db-ssl-mode" "$PG_SSL_MODE") + args+=("--port" "$PORT") + + echo "Starting metadata server" + exec ${packages.metadata-server}/bin/metadata-server "''${args[@]}" + ''; + }; + + metadata-sync = writeShellApplication { + runtimeInputs = with nixpkgs; prelude-runtime ++ [gitMinimal gnused]; + debugInputs = []; + name = "entrypoint"; + text = '' + # Build args array + args+=("--db" "$PG_DB") + args+=("--db-user" "$PG_USER") + args+=("--db-pass" "$PG_PASS") + args+=("--db-host" "$PG_HOST") + args+=("--db-table" "$PG_TABLE") + args+=("--db-conns" "$PG_NUM_CONNS") + args+=("--db-ssl-mode" "$PG_SSL_MODE") + args+=("--git-url" "$GIT_URL") + args+=("--git-metadata-folder" "$GIT_METADATA_FOLDER") + + LC_TIME=C + export TMPDIR="/local"; + export HOME="/local"; + export SSL_CERT_FILE="/etc/ssl/certs/ca-bundle.crt"; + cd /local + + while true; do + echo "Starting metadata sync at $(date -u)" + ${packages.metadata-sync}/bin/metadata-sync "''${args[@]}" | sed -E 's/password=[^ ]* //g' + echo "Sleeping 1 hour until the next sync..." + echo + sleep 3600 + done + ''; + }; + + metadata-webhook= writeShellApplication { + runtimeInputs = prelude-runtime; + debugInputs = []; + name = "entrypoint"; + text = '' + # Build args array + args+=("--db" "$PG_DB") + args+=("--db-user" "$PG_USER") + args+=("--db-pass" "$PG_PASS") + args+=("--db-host" "$PG_HOST") + args+=("--db-table" "$PG_TABLE") + args+=("--db-conns" "$PG_NUM_CONNS") + args+=("--db-ssl-mode" "$PG_SSL_MODE") + args+=("--port" "$PORT") + + echo "Starting metadata webhook" + exec ${packages.metadata-webhook}/bin/metadata-webhook "''${args[@]}" + ''; + }; }) {} diff --git a/nix/cardano/hydrationProfiles.nix b/nix/cardano/hydrationProfiles.nix index c18fb7cf6e..b9cbf92457 100644 --- a/nix/cardano/hydrationProfiles.nix +++ b/nix/cardano/hydrationProfiles.nix @@ -171,4 +171,20 @@ path."auth/token/roles/submit-api".capabilities = ["read"]; }; }; + + # Metadata + workload-policies-metadata = { + tf.hydrate-cluster.configuration.locals.policies = { + vault.metadata = { + path."kv/data/metadata/*".capabilities = ["read" "list"]; + path."kv/metadata/metadata/*".capabilities = ["read" "list"]; + }; + }; + # FIXME: consolidate policy reconciliation loop with TF + # PROBLEM: requires bootstrapper reconciliation loop + # clients need the capability to impersonate the `metadata` role + services.vault.policies.client = { + path."auth/token/roles/metadata".capabilities = ["read"]; + }; + }; } diff --git a/nix/cardano/nomadCharts/default.nix b/nix/cardano/nomadCharts/default.nix index e30482a7ad..4f4b4252f3 100644 --- a/nix/cardano/nomadCharts/default.nix +++ b/nix/cardano/nomadCharts/default.nix @@ -7,4 +7,5 @@ cardano-db-sync = import ./cardano-db-sync.nix {inherit inputs cell;}; cardano-faucet = import ./cardano-faucet.nix {inherit inputs cell;}; cardano-wallet = import ./cardano-wallet.nix {inherit inputs cell;}; + metadata = import ./metadata.nix {inherit inputs cell;}; } diff --git a/nix/cardano/nomadCharts/metadata.nix b/nix/cardano/nomadCharts/metadata.nix new file mode 100644 index 0000000000..698bf0b867 --- /dev/null +++ b/nix/cardano/nomadCharts/metadata.nix @@ -0,0 +1,228 @@ +{ + inputs, + cell, +}: let + inherit (inputs) data-merge cells; + inherit (inputs.nixpkgs) lib system; + inherit (inputs.bitte-cells) vector _utils; + inherit (cell) healthChecks constants oci-images; + # OCI-Image Namer + ociNamer = oci: l.unsafeDiscardStringContext "${oci.imageName}:${oci.imageTag}"; + l = lib // builtins; +in + { + jobname ? "metadata-server", + namespace, + datacenters ? ["eu-central-1" "eu-west-1" "us-east-2"], + domain, + extraVector ? {}, + nodeClass, + ... + } @ args: let + id = jobname; + type = "service"; + priority = 50; + in + with data-merge; { + job.${id} = { + inherit namespace datacenters id type priority; + # ---------- + # Scheduling + # ---------- + constraint = [ + { + attribute = "\${node.class}"; + operator = "="; + value = "${nodeClass}"; + } + { + operator = "distinct_hosts"; + value = "true"; + } + ]; + spread = [ + { + attribute = "\${attr.platform.aws.placement.availability-zone}"; + weight = "100"; + } + ]; + # ---------- + # Update + # ---------- + # https://www.nomadproject.io/docs/job-specification/update + update.health_check = "checks"; + update.healthy_deadline = "5m0s"; + update.max_parallel = 1; + update.min_healthy_time = "10s"; + update.progress_deadline = "10m0s"; + update.stagger = "30s"; + # ---------- + # Migrate + # ---------- + # https://www.nomadproject.io/docs/job-specification/migrate + migrate.health_check = "checks"; + migrate.healthy_deadline = "8m20s"; + migrate.max_parallel = 1; + migrate.min_healthy_time = "10s"; + # ---------- + # Reschedule + # ---------- + reschedule.delay = "10s"; + reschedule.delay_function = "exponential"; + reschedule.max_delay = "5m"; + reschedule.unlimited = true; + # ---------- + # Task Groups + # ---------- + group.metadata = + merge + # task.vector ... + (vector.nomadTask.default { + inherit namespace; + endpoints = [ + # For when metadata-server metrics metrics scrape endpoint becomes available + ]; + extra = extraVector; + }) + { + count = 1; + network = { + mode = "bridge"; + dns = {servers = ["172.17.0.1"];}; + port = { + server = {}; + varnish = {}; + webhook = {}; + }; + }; + service = [ + (import ./srv-metadata-server.nix {inherit namespace;}) + (import ./srv-metadata-webhook.nix {inherit namespace;}) + ]; + task = { + # ------------ + # Task: server + # ------------ + server = { + env = { + PG_DB = "metadata_server"; + PG_HOST = "master.infra-database.service.consul"; + PG_TABLE = "metadata"; + PG_NUM_CONNS = "1"; + PG_SSL_MODE = "require"; + PORT = "\${NOMAD_PORT_server}"; + }; + + template = [ + { + change_mode = "restart"; + data = '' + PG_USER={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgUser }}{{ end }} + PG_PASS={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgPass }}{{ end }} + ''; + destination = "/secrets/metadata-server-env.sh"; + env = true; + } + ]; + + config.image = ociNamer oci-images.metadata-server; + driver = "docker"; + kill_signal = "SIGINT"; + kill_timeout = "30s"; + resources = { + cpu = 2000; + memory = 4 * 1024; + }; + vault = { + change_mode = "noop"; + env = true; + policies = ["metadata"]; + }; + }; + + # ---------- + # Task: sync + # ---------- + sync = { + env = { + PG_DB = "metadata_server"; + PG_HOST = "master.infra-database.service.consul"; + PG_TABLE = "metadata"; + PG_NUM_CONNS = "1"; + PG_SSL_MODE = "require"; + }; + + template = [ + { + change_mode = "restart"; + data = '' + PG_USER={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgUser }}{{ end }} + PG_PASS={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgPass }}{{ end }} + GIT_URL={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.gitUrl }}{{ end }} + GIT_METADATA_FOLDER={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.gitMetadataFolder }}{{ end }} + ''; + destination = "/secrets/metadata-sync-env.sh"; + env = true; + } + ]; + + config.image = ociNamer oci-images.metadata-sync; + driver = "docker"; + kill_signal = "SIGINT"; + kill_timeout = "30s"; + resources = { + cpu = 2000; + memory = 2 * 1024; + }; + vault = { + change_mode = "noop"; + env = true; + policies = ["metadata"]; + }; + }; + + # ------------- + # Task: webhook + # ------------- + webhook = { + env = { + PG_DB = "metadata_server"; + PG_HOST = "master.infra-database.service.consul"; + PG_TABLE = "metadata"; + PG_NUM_CONNS = "1"; + PG_SSL_MODE = "require"; + PORT = "\${NOMAD_PORT_webhook}"; + }; + + template = [ + { + change_mode = "restart"; + data = '' + PG_USER={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgUser }}{{ end }} + PG_PASS={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgPass }}{{ end }} + METADATA_GITHUB_TOKEN={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.metadataGithubToken }}{{ end }} + METADATA_WEBHOOK_SECRET={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.metadataWebhookSecret }}{{ end }} + ''; + destination = "/secrets/metadata-webhook-env.sh"; + env = true; + } + ]; + + config.image = ociNamer oci-images.metadata-webhook; + driver = "docker"; + kill_signal = "SIGINT"; + kill_timeout = "30s"; + resources = { + cpu = 2000; + memory = 2 * 1024; + }; + vault = { + change_mode = "noop"; + env = true; + policies = ["metadata"]; + }; + }; + }; + }; + }; + } diff --git a/nix/cardano/nomadCharts/srv-metadata-server.nix b/nix/cardano/nomadCharts/srv-metadata-server.nix new file mode 100644 index 0000000000..d17627211e --- /dev/null +++ b/nix/cardano/nomadCharts/srv-metadata-server.nix @@ -0,0 +1,18 @@ +{ + namespace, +}: { + address_mode = "auto"; + check = [ + { + name = "live"; + address_mode = "host"; + port = "server"; + timeout = "2s"; + type = "tcp"; + interval = "1m0s"; + } + ]; + name = "${namespace}-metadata-server"; + port = "server"; + tags = []; +} diff --git a/nix/cardano/nomadCharts/srv-metadata-webhook.nix b/nix/cardano/nomadCharts/srv-metadata-webhook.nix new file mode 100644 index 0000000000..836739b649 --- /dev/null +++ b/nix/cardano/nomadCharts/srv-metadata-webhook.nix @@ -0,0 +1,18 @@ +{ + namespace, +}: { + address_mode = "auto"; + check = [ + { + name = "live"; + address_mode = "host"; + port = "webhook"; + timeout = "2s"; + type = "tcp"; + interval = "1m0s"; + } + ]; + name = "${namespace}-metadata-webhook"; + port = "webhook"; + tags = []; +} diff --git a/nix/cardano/oci-images.nix b/nix/cardano/oci-images.nix index 244c7d7d14..8def737884 100644 --- a/nix/cardano/oci-images.nix +++ b/nix/cardano/oci-images.nix @@ -34,6 +34,7 @@ in { ]; config.User = "65534:65534"; }; + cardano-db-sync = buildDebugImage entrypoints.cardano-db-sync { name = "registry.ci.iog.io/cardano-db-sync"; maxLayers = 25; @@ -54,6 +55,7 @@ in { } ]; }; + cardano-wallet = buildDebugImage entrypoints.cardano-wallet { name = "registry.ci.iog.io/cardano-wallet"; maxLayers = 25; @@ -67,6 +69,7 @@ in { ]; config.User = "65534:65534"; }; + cardano-submit-api = buildDebugImage entrypoints.cardano-submit-api { name = "registry.ci.iog.io/cardano-submit-api"; maxLayers = 25; @@ -80,6 +83,7 @@ in { ]; config.User = "65534:65534"; }; + ogmios = buildDebugImage entrypoints.ogmios { name = "registry.ci.iog.io/ogmios"; maxLayers = 25; @@ -93,6 +97,7 @@ in { ]; config.User = "65534:65534"; }; + cardano-faucet = buildDebugImage entrypoints.cardano-faucet { name = "registry.ci.iog.io/cardano-faucet"; maxLayers = 25; @@ -105,4 +110,43 @@ in { ]; config.User = "65534:65534"; }; + + metadata-server = buildDebugImage entrypoints.metadata-server { + name = "registry.ci.iog.io/metadata-server"; + maxLayers = 25; + layers = [ + (n2c.buildLayer {deps = [packages.metadata-server];}) + ]; + copyToRoot = [nixpkgs.bashInteractive]; + config.Cmd = [ + "${entrypoints.metadata-server}/bin/entrypoint" + ]; + config.User = "65534:65534"; + }; + + metadata-sync = buildDebugImage entrypoints.metadata-sync { + name = "registry.ci.iog.io/metadata-sync"; + maxLayers = 25; + layers = [ + (n2c.buildLayer {deps = [packages.metadata-sync];}) + ]; + copyToRoot = with nixpkgs; [bashInteractive cacert]; + config.Cmd = [ + "${entrypoints.metadata-sync}/bin/entrypoint" + ]; + config.User = "65534:65534"; + }; + + metadata-webhook = buildDebugImage entrypoints.metadata-webhook { + name = "registry.ci.iog.io/metadata-webhook"; + maxLayers = 25; + layers = [ + (n2c.buildLayer {deps = [packages.metadata-webhook];}) + ]; + copyToRoot = [nixpkgs.bashInteractive]; + config.Cmd = [ + "${entrypoints.metadata-webhook}/bin/entrypoint" + ]; + config.User = "65534:65534"; + }; } diff --git a/nix/cloud/hydrationProfiles.nix b/nix/cloud/hydrationProfiles.nix index c2ef5801ae..178602b81d 100644 --- a/nix/cloud/hydrationProfiles.nix +++ b/nix/cloud/hydrationProfiles.nix @@ -16,6 +16,7 @@ in { (cells.cardano.hydrationProfiles.workload-policies-faucet) (cells.cardano.hydrationProfiles.workload-policies-wallet) (cells.cardano.hydrationProfiles.workload-policies-submit-api) + (cells.cardano.hydrationProfiles.workload-policies-metadata) ]; # NixOS-level hydration # -------------- diff --git a/nix/cloud/kv/vault/.sops.yaml b/nix/cloud/kv/vault/.sops.yaml index d629a38a0a..8777b1ee0f 100644 --- a/nix/cloud/kv/vault/.sops.yaml +++ b/nix/cloud/kv/vault/.sops.yaml @@ -11,6 +11,8 @@ creation_rules: # Ops - path_regex: patroni/infra* hc_vault_transit_uri: "https://vault.world.dev.cardano.org/v1/sops/keys/ops" + - path_regex: metadata/infra* + hc_vault_transit_uri: "https://vault.world.dev.cardano.org/v1/sops/keys/ops" - path_regex: db-sync/shelley-qa* hc_vault_transit_uri: "https://vault.world.dev.cardano.org/v1/sops/keys/ops" - path_regex: db-sync/preprod* diff --git a/nix/cloud/kv/vault/metadata/infra.enc.yaml b/nix/cloud/kv/vault/metadata/infra.enc.yaml new file mode 100644 index 0000000000..08c3181e2c --- /dev/null +++ b/nix/cloud/kv/vault/metadata/infra.enc.yaml @@ -0,0 +1,22 @@ +pgUser: ENC[AES256_GCM,data:eSR2DYgwkCdILDtdnA==,iv:eLqKkH4HCNbKA5cgySy0dFAgCltWTjDedFitGP4qHPk=,tag:DXT6JtcS5o0fLdYlga8+iQ==,type:str] +pgPass: ENC[AES256_GCM,data:6BUF0vMWTGDbsr2Ma1EDJ66KiVl1FR7rOuAKCjIJpWmbQHwpd6n4ReJOFNxXTJh1ytWEfbxhGI0f/muiQ//lyg==,iv:eNOga0IQqKMnuLyaNaLARl/feMKU5gDFomGrh8saS30=,tag:1sz8PQ7+pFSuudfRsXgrzQ==,type:str] +gitUrl: ENC[AES256_GCM,data:ZElfT7Um7xDh6KKJS7oCyt5U6P0A4Lts7SFmB1FlfdWluyWp6HHu/1/r62OP8tRr/cC4c25eEXVfcSTeuNIuXA==,iv:x4oflQtrSuMS+mQ1JOCOUd75Sj1Li9YQWe2C/YQVjCI=,tag:yJvTnAe9czT9Iphtd0N2Ug==,type:str] +gitMetadataFolder: ENC[AES256_GCM,data:GRUR3bGbKYA=,iv:e4Oou94IXANkrCjqPynltF3QFkOQv+I8PGYky5FE4h4=,tag:zqLdnVQbcuBnE1eBm6BXJg==,type:str] +metadataWebhookSecret: ENC[AES256_GCM,data:Zxn2TKnfK9OS7gZDN88Q1yDjth7ROhOHfD4sNIXqeBY=,iv:ahS6yG1diPdXfOJcPbyyEpGr3uVMVSd9zwCFIds9l9k=,tag:m+34lfvzA3y5YCMzOHATVg==,type:str] +metadataGithubToken: "" +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: + - vault_address: https://vault.world.dev.cardano.org + engine_path: sops + key_name: ops + created_at: "2023-03-17T01:52:06Z" + enc: vault:v1:uQFF6gz+ycdXVS3ZRIAo8/ZapLZLUo+zJNFhWQJo7fEV7j+xBwIArzEOT/OKsqxjgfoNmujjzczZ0wBd + age: [] + lastmodified: "2023-03-17T04:37:33Z" + mac: ENC[AES256_GCM,data:NRvEBsTTB2iwhaJmvbYV4Eh/4AUzv99ND53dwtpR221XGhe4GOFdAN/T+QfbdVT2m2rqRw4O0zhxFHAxGdeSwKDr7RYi16IOhKoSiCRPy4soLtwNAItBQPRp5T7FsZGPkpjyswosgWfRxKASsP3/9k0LoeI+FIu5mnppqQOtTqg=,iv:nhGrtCqemdK0nLEeRMi1nvMiufg+bSOVzUCSVXR28oY=,tag:E6CFb39UkNCzlsIcSQNPFA==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.7.3 diff --git a/nix/cloud/namespaces/infra.nix b/nix/cloud/namespaces/infra.nix index 52c8c2b11f..b821940266 100644 --- a/nix/cloud/namespaces/infra.nix +++ b/nix/cloud/namespaces/infra.nix @@ -4,7 +4,7 @@ }: let inherit (inputs) data-merge; inherit (inputs.bitte-cells) patroni; - inherit (inputs.cells) docs; + inherit (inputs.cells) cardano docs; inherit (cell) constants; WALG_S3_PREFIX = "s3://iog-cardano-bitte/backups/infra/walg"; @@ -17,4 +17,14 @@ in { job.database.group.database.task.patroni.env = {inherit WALG_S3_PREFIX;}; job.database.group.database.task.backup-walg.env = {inherit WALG_S3_PREFIX;}; }; + + metadata = let + jobname = "metadata"; + in cardano.nomadCharts.metadata ( + constants.envs.infra + // { + datacenters = ["eu-central-1"]; + inherit jobname; + } + ); }