Skip to content

Commit

Permalink
snapshot code cleanup and permission fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@manveru @johnalotoski
manveru authored and johnalotoski committed Sep 28, 2021
1 parent 230a72c commit 2228603
Show file tree
Hide file tree
Showing 5 changed files with 249 additions and 212 deletions.
129 changes: 71 additions & 58 deletions modules/consul-snapshots.nix
View file
Expand Up @@ -135,61 +135,68 @@ let
};

snapshotService = job: {
serviceConfig.Type = "oneshot";
path = with pkgs; [ consul coreutils findutils gawk hostname jq ];
script = builtins.readFile "${(pkgs.writeBashChecked "consul-snapshot-${job}-script" ''
set -exuo pipefail
OWNER="${cfg.${job}.owner}"
BACKUP_DIR="${cfg.${job}.backupDirPrefix}/${job}"
BACKUP_SUFFIX="-${cfg.${job}.backupSuffix}";
INCLUDE_LEADER="${if cfg.${job}.includeLeader then "true" else "false"}"
SNAP_NAME="$BACKUP_DIR/consul-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap"
CONSUL_HTTP_ADDR="${cfg.${job}.consulAddress}"
applyPerms () {
TARGET="$1"
PERMS="$2"
chown "$OWNER" "$TARGET"
chmod "$PERMS" "$TARGET"
}
checkBackupDir () {
if [ ! -d "$BACKUP_DIR" ]; then
mkdir -p "$BACKUP_DIR"
applyPerms "$BACKUP_DIR" "0700"
fi
}

isNotLeader () {
if [ "$(consul info | grep 'leader =' | awk '{print $3}')" = "false" ]; then
return
environment = {
OWNER = cfg.${job}.owner;
BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}";
BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}";
INCLUDE_LEADER = lib.boolToString cfg.${job}.includeLeader;
CONSUL_HTTP_ADDR = cfg.${job}.consulAddress;
};

serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
Restart = "on-failure";
RestartSec = "30s";
ExecStart = pkgs.writeBashChecked "consul-snapshot-${job}-script" ''
set -exuo pipefail
SNAP_NAME="$BACKUP_DIR/consul-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap"
applyPerms () {
TARGET="$1"
PERMS="$2"
chown "$OWNER" "$TARGET"
chmod "$PERMS" "$TARGET"
}
checkBackupDir () {
if [ ! -d "$BACKUP_DIR" ]; then
mkdir -p "$BACKUP_DIR"
applyPerms "$BACKUP_DIR" "0700"
fi
}
isNotLeader () {
[ "$INCLUDE_LEADER" = "true" ] || \
consul info | egrep '^\s*leader\s+=\s+false$'
}
takeConsulSnapshot () {
consul snapshot save "$SNAP_NAME"
applyPerms "$SNAP_NAME" "0400"
}
export CONSUL_HTTP_ADDR
if isNotLeader; then
checkBackupDir
takeConsulSnapshot
fi
false
}
takeConsulSnapshot () {
consul snapshot save "$SNAP_NAME"
applyPerms "$SNAP_NAME" "0400"
}
export CONSUL_HTTP_ADDR
if [ "$INCLUDE_LEADER" = "true" ] || isNotLeader; then
checkBackupDir
takeConsulSnapshot
fi
find "$BACKUP_DIR" \
-type f \
-name "*''${BACKUP_SUFFIX}.snap" \
-printf "%T@ %p\n" \
| sort -r -n \
| tail -n +$((${toString cfg.${job}.backupCount} + 1)) \
| awk '{print $2}' \
| xargs -r rm
'')}";
find "$BACKUP_DIR" \
-type f \
-name "*''${BACKUP_SUFFIX}.snap" \
-printf "%T@ %p\n" \
| sort -r -n \
| tail -n +${toString (cfg.${job}.backupCount + 1)} \
| awk '{print $2}' \
| xargs -r rm
'';
};
};

in {
Expand Down Expand Up @@ -242,15 +249,21 @@ in {

config = mkIf cfg.enable {
# Hourly snapshot configuration
systemd.timers.consul-snapshots-hourly = mkIf cfg.hourly.enable (snapshotTimer "hourly");
systemd.services.consul-snapshots-hourly = mkIf cfg.hourly.enable (snapshotService "hourly");
systemd.timers.consul-snapshots-hourly =
mkIf cfg.hourly.enable (snapshotTimer "hourly");
systemd.services.consul-snapshots-hourly =
mkIf cfg.hourly.enable (snapshotService "hourly");

# Daily snapshot configuration
systemd.timers.consul-snapshots-daily = mkIf cfg.daily.enable (snapshotTimer "daily");
systemd.services.consul-snapshots-daily = mkIf cfg.daily.enable (snapshotService "daily");
systemd.timers.consul-snapshots-daily =
mkIf cfg.daily.enable (snapshotTimer "daily");
systemd.services.consul-snapshots-daily =
mkIf cfg.daily.enable (snapshotService "daily");

# Custom snapshot configuration
systemd.timers.consul-snapshots-custom = mkIf cfg.custom.enable (snapshotTimer "custom");
systemd.services.consul-snapshots-custom = mkIf cfg.custom.enable (snapshotService "custom");
systemd.timers.consul-snapshots-custom =
mkIf cfg.custom.enable (snapshotTimer "custom");
systemd.services.consul-snapshots-custom =
mkIf cfg.custom.enable (snapshotService "custom");
};
}
156 changes: 85 additions & 71 deletions modules/nomad-snapshots.nix
View file
Expand Up @@ -137,73 +137,81 @@ let
snapshotService = job: {
serviceConfig.Type = "oneshot";
path = with pkgs; [ coreutils curl findutils gawk hostname jq nomad ];
script = builtins.readFile "${(pkgs.writeBashChecked "nomad-snapshot-${job}-script" ''
set -exuo pipefail
OWNER="${cfg.${job}.owner}"
BACKUP_DIR="${cfg.${job}.backupDirPrefix}/${job}"
BACKUP_SUFFIX="-${cfg.${job}.backupSuffix}";
INCLUDE_LEADER="${if cfg.${job}.includeLeader then "true" else "false"}"
SNAP_NAME="$BACKUP_DIR/nomad-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap"
NOMAD_ADDR="${cfg.${job}.nomadAddress}"
applyPerms () {
TARGET="$1"
PERMS="$2"
chown "$OWNER" "$TARGET"
chmod "$PERMS" "$TARGET"
}
checkBackupDir () {
if [ ! -d "$BACKUP_DIR" ]; then
mkdir -p "$BACKUP_DIR"
applyPerms "$BACKUP_DIR" "0700"
fi
}
exportToken () {
if [ ! -f /var/lib/nomad/bootstrap.token ]; then
echo "Suitable nomad token for snapshotting not found."
echo "Ensure the appropriate token for snapshotting is available.";
exit 0;
else
set +x
NOMAD_TOKEN="$(< /var/lib/nomad/bootstrap.token)"
export NOMAD_TOKEN
set -x
fi
}

isNotLeader () {
if [ "$(nomad agent-info --json | jq -e -r '.stats.nomad.leader')" = "false" ]; then
return
environment = {
OWNER = cfg.${job}.owner;
BACKUP_DIR = "${cfg.${job}.backupDirPrefix}/${job}";
BACKUP_SUFFIX = "-${cfg.${job}.backupSuffix}";
INCLUDE_LEADER = lib.boolToString cfg.${job}.includeLeader;
NOMAD_ADDR = cfg.${job}.nomadAddress;
};

serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
Restart = "on-failure";
RestartSec = "30s";
ExecStart = pkgs.writeBashChecked "nomad-snapshot-${job}-script" ''
set -exuo pipefail
SNAP_NAME="$BACKUP_DIR/nomad-$(hostname)-$(date +"%Y-%m-%d_%H%M%SZ''${BACKUP_SUFFIX}").snap"
applyPerms () {
TARGET="$1"
PERMS="$2"
chown "$OWNER" "$TARGET"
chmod "$PERMS" "$TARGET"
}
checkBackupDir () {
if [ ! -d "$BACKUP_DIR" ]; then
mkdir -p "$BACKUP_DIR"
applyPerms "$BACKUP_DIR" "0700"
fi
}
exportToken () {
if [ ! -f /run/keys/nomad-snapshot-token ]; then
echo "Suitable nomad token for snapshotting not found."
echo "Ensure the appropriate token for snapshotting is available.";
exit 0;
else
set +x
NOMAD_TOKEN="$(< /run/keys/nomad-snapshot-token)"
export NOMAD_TOKEN
set -x
fi
}
isNotLeader () {
[ "$INCLUDE_LEADER" = "true" ] || \
nomad agent-info --json | jq -e '.stats.nomad.leader == "false"'
}
takeNomadSnapshot () {
nomad operator snapshot save "$SNAP_NAME"
applyPerms "$SNAP_NAME" "0400"
}
export NOMAD_ADDR
exportToken
if isNotLeader; then
checkBackupDir
takeNomadSnapshot
fi
false
}
takeNomadSnapshot () {
nomad operator snapshot save "$SNAP_NAME"
applyPerms "$SNAP_NAME" "0400"
}
export NOMAD_ADDR
exportToken
if [ "$INCLUDE_LEADER" = "true" ] || isNotLeader; then
checkBackupDir
takeNomadSnapshot
fi
find "$BACKUP_DIR" \
-type f \
-name "*''${BACKUP_SUFFIX}.snap" \
-printf "%T@ %p\n" \
| sort -r -n \
| tail -n +$((${toString cfg.${job}.backupCount} + 1)) \
| awk '{print $2}' \
| xargs -r rm
'')}";
find "$BACKUP_DIR" \
-type f \
-name "*''${BACKUP_SUFFIX}.snap" \
-printf "%T@ %p\n" \
| sort -r -n \
| tail -n +${toString (cfg.${job}.backupCount + 1)} \
| awk '{print $2}' \
| xargs -r rm
'';
};
};

in {
Expand Down Expand Up @@ -256,15 +264,21 @@ in {

config = mkIf cfg.enable {
# Hourly snapshot configuration
systemd.timers.nomad-snapshots-hourly = mkIf cfg.hourly.enable (snapshotTimer "hourly");
systemd.services.nomad-snapshots-hourly = mkIf cfg.hourly.enable (snapshotService "hourly");
systemd.timers.nomad-snapshots-hourly =
mkIf cfg.hourly.enable (snapshotTimer "hourly");
systemd.services.nomad-snapshots-hourly =
mkIf cfg.hourly.enable (snapshotService "hourly");

# Daily snapshot configuration
systemd.timers.nomad-snapshots-daily = mkIf cfg.daily.enable (snapshotTimer "daily");
systemd.services.nomad-snapshots-daily = mkIf cfg.daily.enable (snapshotService "daily");
systemd.timers.nomad-snapshots-daily =
mkIf cfg.daily.enable (snapshotTimer "daily");
systemd.services.nomad-snapshots-daily =
mkIf cfg.daily.enable (snapshotService "daily");

# Custom snapshot configuration
systemd.timers.nomad-snapshots-custom = mkIf cfg.custom.enable (snapshotTimer "custom");
systemd.services.nomad-snapshots-custom = mkIf cfg.custom.enable (snapshotService "custom");
systemd.timers.nomad-snapshots-custom =
mkIf cfg.custom.enable (snapshotTimer "custom");
systemd.services.nomad-snapshots-custom =
mkIf cfg.custom.enable (snapshotService "custom");
};
}
7 changes: 7 additions & 0 deletions modules/vault-agent-server.nix
View file
Expand Up @@ -88,6 +88,13 @@ in {
{{- with secret "nomad/creds/nomad-autoscaler" }}{{ .Data.secret_id }}{{ end -}}
'';
};

"/run/keys/nomad-snapshot-token" =
mkIf config.services.nomad-snapshot.enable {
contents = ''
{{- with secret "nomad/creds/management" }}{{ .Data.secret_id }}{{ end -}}
'';
};
};
};
};
Expand Down

0 comments on commit 2228603

Please sign in to comment.