Skip to content

Commit

Permalink
imp: adds basic metadata server, sync agent, webhook
Browse files Browse the repository at this point in the history
  • Loading branch information
@johnalotoski
johnalotoski committed Mar 17, 2023
1 parent 880a674 commit 2d1452f
Show file tree
Hide file tree
Showing 13 changed files with 438 additions and 5 deletions.
7 changes: 4 additions & 3 deletions flake.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion flake.nix
View file
Expand Up @@ -70,7 +70,7 @@
cardano-node.url = "github:input-output-hk/cardano-node/1.35.6";
cardano-wallet.url = "github:input-output-hk/cardano-wallet/v2022-07-01";
offchain-metadata-tools = {
url = "github:input-output-hk/offchain-metadata-tools";
url = "github:input-output-hk/offchain-metadata-tools/pg-cli-mods";
flake = false;
};

Expand Down
72 changes: 72 additions & 0 deletions nix/cardano/entrypoints.nix
View file
Expand Up @@ -604,4 +604,76 @@ in {
exec ${packages.cardano-new-faucet}/bin/cardano-new-faucet
'';
};

metadata-server = writeShellApplication {
runtimeInputs = prelude-runtime;
debugInputs = [];
name = "entrypoint";
text = ''
# Build args array
args+=("--db" "$PG_DB")
args+=("--db-user" "$PG_USER")
args+=("--db-pass" "$PG_PASS")
args+=("--db-host" "$PG_HOST")
args+=("--db-table" "$PG_TABLE")
args+=("--db-conns" "$PG_NUM_CONNS")
args+=("--db-ssl-mode" "$PG_SSL_MODE")
args+=("--port" "$PORT")
echo "Starting metadata server"
exec ${packages.metadata-server}/bin/metadata-server "''${args[@]}"
'';
};

metadata-sync = writeShellApplication {
runtimeInputs = with nixpkgs; prelude-runtime ++ [gitMinimal gnused];
debugInputs = [];
name = "entrypoint";
text = ''
# Build args array
args+=("--db" "$PG_DB")
args+=("--db-user" "$PG_USER")
args+=("--db-pass" "$PG_PASS")
args+=("--db-host" "$PG_HOST")
args+=("--db-table" "$PG_TABLE")
args+=("--db-conns" "$PG_NUM_CONNS")
args+=("--db-ssl-mode" "$PG_SSL_MODE")
args+=("--git-url" "$GIT_URL")
args+=("--git-metadata-folder" "$GIT_METADATA_FOLDER")
LC_TIME=C
export TMPDIR="/local";
export HOME="/local";
export SSL_CERT_FILE="/etc/ssl/certs/ca-bundle.crt";
cd /local
while true; do
echo "Starting metadata sync at $(date -u)"
${packages.metadata-sync}/bin/metadata-sync "''${args[@]}" | sed -E 's/password=[^ ]* //g'
echo "Sleeping 1 hour until the next sync..."
echo
sleep 3600
done
'';
};

metadata-webhook= writeShellApplication {
runtimeInputs = prelude-runtime;
debugInputs = [];
name = "entrypoint";
text = ''
# Build args array
args+=("--db" "$PG_DB")
args+=("--db-user" "$PG_USER")
args+=("--db-pass" "$PG_PASS")
args+=("--db-host" "$PG_HOST")
args+=("--db-table" "$PG_TABLE")
args+=("--db-conns" "$PG_NUM_CONNS")
args+=("--db-ssl-mode" "$PG_SSL_MODE")
args+=("--port" "$PORT")
echo "Starting metadata webhook"
exec ${packages.metadata-webhook}/bin/metadata-webhook "''${args[@]}"
'';
};
}) {}
16 changes: 16 additions & 0 deletions nix/cardano/hydrationProfiles.nix
View file
Expand Up @@ -171,4 +171,20 @@
path."auth/token/roles/submit-api".capabilities = ["read"];
};
};

# Metadata
workload-policies-metadata = {
tf.hydrate-cluster.configuration.locals.policies = {
vault.metadata = {
path."kv/data/metadata/*".capabilities = ["read" "list"];
path."kv/metadata/metadata/*".capabilities = ["read" "list"];
};
};
# FIXME: consolidate policy reconciliation loop with TF
# PROBLEM: requires bootstrapper reconciliation loop
# clients need the capability to impersonate the `metadata` role
services.vault.policies.client = {
path."auth/token/roles/metadata".capabilities = ["read"];
};
};
}
1 change: 1 addition & 0 deletions nix/cardano/nomadCharts/default.nix
View file
Expand Up @@ -7,4 +7,5 @@
cardano-db-sync = import ./cardano-db-sync.nix {inherit inputs cell;};
cardano-faucet = import ./cardano-faucet.nix {inherit inputs cell;};
cardano-wallet = import ./cardano-wallet.nix {inherit inputs cell;};
metadata = import ./metadata.nix {inherit inputs cell;};
}
228 changes: 228 additions & 0 deletions nix/cardano/nomadCharts/metadata.nix
View file
@@ -0,0 +1,228 @@
{
inputs,
cell,
}: let
inherit (inputs) data-merge cells;
inherit (inputs.nixpkgs) lib system;
inherit (inputs.bitte-cells) vector _utils;
inherit (cell) healthChecks constants oci-images;
# OCI-Image Namer
ociNamer = oci: l.unsafeDiscardStringContext "${oci.imageName}:${oci.imageTag}";
l = lib // builtins;
in
{
jobname ? "metadata-server",
namespace,
datacenters ? ["eu-central-1" "eu-west-1" "us-east-2"],
domain,
extraVector ? {},
nodeClass,
...
} @ args: let
id = jobname;
type = "service";
priority = 50;
in
with data-merge; {
job.${id} = {
inherit namespace datacenters id type priority;
# ----------
# Scheduling
# ----------
constraint = [
{
attribute = "\${node.class}";
operator = "=";
value = "${nodeClass}";
}
{
operator = "distinct_hosts";
value = "true";
}
];
spread = [
{
attribute = "\${attr.platform.aws.placement.availability-zone}";
weight = "100";
}
];
# ----------
# Update
# ----------
# https://www.nomadproject.io/docs/job-specification/update
update.health_check = "checks";
update.healthy_deadline = "5m0s";
update.max_parallel = 1;
update.min_healthy_time = "10s";
update.progress_deadline = "10m0s";
update.stagger = "30s";
# ----------
# Migrate
# ----------
# https://www.nomadproject.io/docs/job-specification/migrate
migrate.health_check = "checks";
migrate.healthy_deadline = "8m20s";
migrate.max_parallel = 1;
migrate.min_healthy_time = "10s";
# ----------
# Reschedule
# ----------
reschedule.delay = "10s";
reschedule.delay_function = "exponential";
reschedule.max_delay = "5m";
reschedule.unlimited = true;
# ----------
# Task Groups
# ----------
group.metadata =
merge
# task.vector ...
(vector.nomadTask.default {
inherit namespace;
endpoints = [
# For when metadata-server metrics metrics scrape endpoint becomes available
];
extra = extraVector;
})
{
count = 1;
network = {
mode = "bridge";
dns = {servers = ["172.17.0.1"];};
port = {
server = {};
varnish = {};
webhook = {};
};
};
service = [
(import ./srv-metadata-server.nix {inherit namespace;})
(import ./srv-metadata-webhook.nix {inherit namespace;})
];
task = {
# ------------
# Task: server
# ------------
server = {
env = {
PG_DB = "metadata_server";
PG_HOST = "master.infra-database.service.consul";
PG_TABLE = "metadata";
PG_NUM_CONNS = "1";
PG_SSL_MODE = "require";
PORT = "\${NOMAD_PORT_server}";
};

template = [
{
change_mode = "restart";
data = ''
PG_USER={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgUser }}{{ end }}
PG_PASS={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgPass }}{{ end }}
'';
destination = "/secrets/metadata-server-env.sh";
env = true;
}
];

config.image = ociNamer oci-images.metadata-server;
driver = "docker";
kill_signal = "SIGINT";
kill_timeout = "30s";
resources = {
cpu = 2000;
memory = 4 * 1024;
};
vault = {
change_mode = "noop";
env = true;
policies = ["metadata"];
};
};

# ----------
# Task: sync
# ----------
sync = {
env = {
PG_DB = "metadata_server";
PG_HOST = "master.infra-database.service.consul";
PG_TABLE = "metadata";
PG_NUM_CONNS = "1";
PG_SSL_MODE = "require";
};

template = [
{
change_mode = "restart";
data = ''
PG_USER={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgUser }}{{ end }}
PG_PASS={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgPass }}{{ end }}
GIT_URL={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.gitUrl }}{{ end }}
GIT_METADATA_FOLDER={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.gitMetadataFolder }}{{ end }}
'';
destination = "/secrets/metadata-sync-env.sh";
env = true;
}
];

config.image = ociNamer oci-images.metadata-sync;
driver = "docker";
kill_signal = "SIGINT";
kill_timeout = "30s";
resources = {
cpu = 2000;
memory = 2 * 1024;
};
vault = {
change_mode = "noop";
env = true;
policies = ["metadata"];
};
};

# -------------
# Task: webhook
# -------------
webhook = {
env = {
PG_DB = "metadata_server";
PG_HOST = "master.infra-database.service.consul";
PG_TABLE = "metadata";
PG_NUM_CONNS = "1";
PG_SSL_MODE = "require";
PORT = "\${NOMAD_PORT_webhook}";
};

template = [
{
change_mode = "restart";
data = ''
PG_USER={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgUser }}{{ end }}
PG_PASS={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.pgPass }}{{ end }}
METADATA_GITHUB_TOKEN={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.metadataGithubToken }}{{ end }}
METADATA_WEBHOOK_SECRET={{- with secret "kv/data/metadata/${namespace}" }}{{ .Data.data.metadataWebhookSecret }}{{ end }}
'';
destination = "/secrets/metadata-webhook-env.sh";
env = true;
}
];

config.image = ociNamer oci-images.metadata-webhook;
driver = "docker";
kill_signal = "SIGINT";
kill_timeout = "30s";
resources = {
cpu = 2000;
memory = 2 * 1024;
};
vault = {
change_mode = "noop";
env = true;
policies = ["metadata"];
};
};
};
};
};
}
18 changes: 18 additions & 0 deletions nix/cardano/nomadCharts/srv-metadata-server.nix
View file
@@ -0,0 +1,18 @@
{
namespace,
}: {
address_mode = "auto";
check = [
{
name = "live";
address_mode = "host";
port = "server";
timeout = "2s";
type = "tcp";
interval = "1m0s";
}
];
name = "${namespace}-metadata-server";
port = "server";
tags = [];
}
18 changes: 18 additions & 0 deletions nix/cardano/nomadCharts/srv-metadata-webhook.nix
View file
@@ -0,0 +1,18 @@
{
namespace,
}: {
address_mode = "auto";
check = [
{
name = "live";
address_mode = "host";
port = "webhook";
timeout = "2s";
type = "tcp";
interval = "1m0s";
}
];
name = "${namespace}-metadata-webhook";
port = "webhook";
tags = [];
}

0 comments on commit 2d1452f

Please sign in to comment.