From aca9bdb9490731953a9358f4022d0b93732b5cf6 Mon Sep 17 00:00:00 2001 From: Byron Ruth Date: Fri, 4 Nov 2022 11:51:15 -0400 Subject: [PATCH 1/3] Add example of leader failure Signed-off-by: Byron Ruth --- .../jetstream/leader-failure/cli/Dockerfile | 12 ++ examples/jetstream/leader-failure/cli/main.sh | 177 ++++++++++++++++++ .../leader-failure/docker-compose.yaml | 4 + examples/jetstream/leader-failure/meta.yaml | 2 + 4 files changed, 195 insertions(+) create mode 100644 examples/jetstream/leader-failure/cli/Dockerfile create mode 100644 examples/jetstream/leader-failure/cli/main.sh create mode 100644 examples/jetstream/leader-failure/docker-compose.yaml create mode 100644 examples/jetstream/leader-failure/meta.yaml diff --git a/examples/jetstream/leader-failure/cli/Dockerfile b/examples/jetstream/leader-failure/cli/Dockerfile new file mode 100644 index 00000000..ad27c8c3 --- /dev/null +++ b/examples/jetstream/leader-failure/cli/Dockerfile @@ -0,0 +1,12 @@ +FROM natsio/nats-box:0.13.2 + +RUN apk add bash curl + +COPY --from=nats:2.9.5 /nats-server /usr/local/bin/ + +COPY . . + +ENTRYPOINT ["bash"] + +CMD ["main.sh"] + diff --git a/examples/jetstream/leader-failure/cli/main.sh b/examples/jetstream/leader-failure/cli/main.sh new file mode 100644 index 00000000..a7a33ab3 --- /dev/null +++ b/examples/jetstream/leader-failure/cli/main.sh @@ -0,0 +1,177 @@ +#!/bin/sh + +set -euo pipefail + +# Define the system account to be included by all configurations. +cat <<- EOF > sys.conf +accounts: { + SYS: { + users: [{user: sys, password: sys}] + } + APP: { + jetstream: true, + users: [{user: app, password: app}] + } +} + +system_account: SYS +EOF + +# Create the *east* and *west* server configurations. +# A requirement of JetStream is to have a cluster block defined +# since this is how available storage resources are determined +# for placement of streams and consumers. +# +# In a production deployment, at least three nodes per cluster +# are recommended which supports creating R3 streams. In this +# test setup, only R1 streams can be created since streams replicas +# do not cross gateway connections. +cat <<- EOF > n1.conf +port: 4222 +http_port: 8222 +server_name: n1 + +include sys.conf + +jetstream: { + store_dir: "./n1" +} + +cluster: { + name: c1, + port: 6222, + routes: [ + "nats-route://0.0.0.0:6222", + "nats-route://0.0.0.0:6223", + "nats-route://0.0.0.0:6224", + ], +} +EOF + +cat <<- EOF > n2.conf +port: 4223 +http_port: 8223 +server_name: n2 + +include sys.conf + +jetstream: { + store_dir: "./n2" +} + +cluster: { + name: c1, + port: 6223, + routes: [ + "nats-route://0.0.0.0:6222", + "nats-route://0.0.0.0:6223", + "nats-route://0.0.0.0:6224", + ], +} +EOF + +cat <<- EOF > n3.conf +port: 4224 +http_port: 8224 +server_name: n3 + +include sys.conf + +jetstream: { + store_dir: "./n3" +} + +cluster: { + name: c1, + port: 6224, + routes: [ + "nats-route://0.0.0.0:6222", + "nats-route://0.0.0.0:6223", + "nats-route://0.0.0.0:6224", + ], +} +EOF + + +# Start the servers and sleep for a few seconds to startup. +nats-server -c n1.conf & +N1_PID=$! + +nats-server -c n2.conf & +N2_PID=$! + +nats-server -c n3.conf & +N3_PID=$! + +sleep 3 + +# Save and select the default context to use all seed servers. +nats context save \ + --server "nats://localhost:4222,nats://localhost:4223,nats://localhost:4224" \ + --user app \ + --password app \ + app + +nats context select app + + +# Show the server list which will indicate the clusters and +# gateway connections as well as the JetStream server report. +nats --user sys --password sys server list +nats --user sys --password sys server report jetstream + +# Add a stream. +nats stream add \ + --retention=limits \ + --storage=file \ + --replicas=3 \ + --discard=old \ + --dupe-window=2m \ + --max-age=-1 \ + --max-msgs=-1 \ + --max-bytes=-1 \ + --max-msg-size=-1 \ + --max-msgs-per-subject=-1 \ + --max-consumers=-1 \ + --allow-rollup \ + --no-deny-delete \ + --no-deny-purge \ + --subjects="orders" \ + ORDERS + +# Publish some messages. +nats req orders --count=100 'Message {{Count}}' + +# Force a step down of the leader. +nats stream cluster step-down ORDERS + +# Report the new leader. +nats --user sys --password sys server report jetstream + +# Publish more messages. +nats req orders --count=100 'Message {{Count}}' + +nats stream list + +# Now determine the leader and kill the server. +LEADER=$(nats stream info ORDERS --json | jq -r '.cluster.leader') +echo "$LEADER is the leader" + +case $LEADER in + "n1") + kill $N1_PID + ;; + "n2") + kill $N2_PID + ;; + "n3") + kill $N3_PID + ;; +esac + +# Publish more messages. +nats req orders --count=100 'Message {{Count}}' + +nats stream list + +nats --user sys --password sys server report jetstream diff --git a/examples/jetstream/leader-failure/docker-compose.yaml b/examples/jetstream/leader-failure/docker-compose.yaml new file mode 100644 index 00000000..b45dea38 --- /dev/null +++ b/examples/jetstream/leader-failure/docker-compose.yaml @@ -0,0 +1,4 @@ +version: '3.9' +services: + app: + image: ${IMAGE_TAG} diff --git a/examples/jetstream/leader-failure/meta.yaml b/examples/jetstream/leader-failure/meta.yaml new file mode 100644 index 00000000..3507bd1f --- /dev/null +++ b/examples/jetstream/leader-failure/meta.yaml @@ -0,0 +1,2 @@ +title: Leader Failure +description: |- From 5bf34c740d7a2fbd78e25e198728fa9307fda178 Mon Sep 17 00:00:00 2001 From: Byron Ruth Date: Fri, 4 Nov 2022 11:58:11 -0400 Subject: [PATCH 2/3] Update comments Signed-off-by: Byron Ruth --- examples/jetstream/leader-failure/cli/main.sh | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/examples/jetstream/leader-failure/cli/main.sh b/examples/jetstream/leader-failure/cli/main.sh index a7a33ab3..ce379738 100644 --- a/examples/jetstream/leader-failure/cli/main.sh +++ b/examples/jetstream/leader-failure/cli/main.sh @@ -2,8 +2,8 @@ set -euo pipefail -# Define the system account to be included by all configurations. -cat <<- EOF > sys.conf +# Share accounts configuration for all nodes. +cat <<- EOF > accounts.conf accounts: { SYS: { users: [{user: sys, password: sys}] @@ -17,21 +17,13 @@ accounts: { system_account: SYS EOF -# Create the *east* and *west* server configurations. -# A requirement of JetStream is to have a cluster block defined -# since this is how available storage resources are determined -# for placement of streams and consumers. -# -# In a production deployment, at least three nodes per cluster -# are recommended which supports creating R3 streams. In this -# test setup, only R1 streams can be created since streams replicas -# do not cross gateway connections. +# Configuration for each node in the cluster. cat <<- EOF > n1.conf port: 4222 http_port: 8222 server_name: n1 -include sys.conf +include accounts.conf jetstream: { store_dir: "./n1" @@ -53,7 +45,7 @@ port: 4223 http_port: 8223 server_name: n2 -include sys.conf +include accounts.conf jetstream: { store_dir: "./n2" @@ -75,7 +67,7 @@ port: 4224 http_port: 8224 server_name: n3 -include sys.conf +include accounts.conf jetstream: { store_dir: "./n3" @@ -105,22 +97,27 @@ N3_PID=$! sleep 3 -# Save and select the default context to use all seed servers. +# Create the app and sys contexts. nats context save \ --server "nats://localhost:4222,nats://localhost:4223,nats://localhost:4224" \ --user app \ --password app \ app -nats context select app +nats context save \ + --server "nats://localhost:4222,nats://localhost:4223,nats://localhost:4224" \ + --user sys \ + --password sys \ + sys +nats context select app # Show the server list which will indicate the clusters and # gateway connections as well as the JetStream server report. -nats --user sys --password sys server list -nats --user sys --password sys server report jetstream +nats --context sys server list +nats --context sys server report jetstream -# Add a stream. +# Add a basic R3 stream. nats stream add \ --retention=limits \ --storage=file \ @@ -142,6 +139,9 @@ nats stream add \ # Publish some messages. nats req orders --count=100 'Message {{Count}}' +# List the streams to show the written messages. +nats stream list + # Force a step down of the leader. nats stream cluster step-down ORDERS @@ -151,9 +151,11 @@ nats --user sys --password sys server report jetstream # Publish more messages. nats req orders --count=100 'Message {{Count}}' +# 200 messages... nats stream list -# Now determine the leader and kill the server. +# Now we actually kill the leader node completely. +# This will force a re-election among the two remaining nodes. LEADER=$(nats stream info ORDERS --json | jq -r '.cluster.leader') echo "$LEADER is the leader" @@ -172,6 +174,9 @@ esac # Publish more messages. nats req orders --count=100 'Message {{Count}}' +# Ensure we have 300 in the stream. nats stream list +# Report the new leader... nats --user sys --password sys server report jetstream + From 237970e5a4e604e1d6accbad0a924ac74041d341 Mon Sep 17 00:00:00 2001 From: Byron Ruth Date: Fri, 4 Nov 2022 12:21:17 -0400 Subject: [PATCH 3/3] Add sleeps Signed-off-by: Byron Ruth --- examples/jetstream/leader-failure/cli/main.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/jetstream/leader-failure/cli/main.sh b/examples/jetstream/leader-failure/cli/main.sh index ce379738..4b0c4315 100644 --- a/examples/jetstream/leader-failure/cli/main.sh +++ b/examples/jetstream/leader-failure/cli/main.sh @@ -145,6 +145,8 @@ nats stream list # Force a step down of the leader. nats stream cluster step-down ORDERS +sleep 2 + # Report the new leader. nats --user sys --password sys server report jetstream @@ -171,6 +173,8 @@ case $LEADER in ;; esac +sleep 2 + # Publish more messages. nats req orders --count=100 'Message {{Count}}'