From 73f218327627cf1b57105bafefb6e70550749132 Mon Sep 17 00:00:00 2001 From: Taylor Brennan Date: Fri, 4 Aug 2023 11:04:56 -0400 Subject: [PATCH 1/5] add s5cmd to image --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e1ab5ee..455001f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,10 @@ ARG VERSION RUN apk --update add gcc git curl alpine-sdk libc6-compat ca-certificates sqlite \ && curl -SsL https://github.com/segmentio/chamber/releases/download/v2.13.2/chamber-v2.13.2-linux-amd64 -o /bin/chamber \ - && chmod +x /bin/chamber + && curl -sL https://github.com/peak/s5cmd/releases/download/v2.1.0/s5cmd_2.1.0_Linux-64bit.tar.gz -o s5cmd.gz && tar -xzf s5cmd.gz -C /bin \ + && chmod +x /bin/chamber \ + && chmod +x /bin/s5cmd + COPY . /go/src/${SRC} WORKDIR /go/src/${SRC} @@ -15,11 +18,10 @@ RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/vers RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/version.version=$VERSION" ${SRC}/pkg/cmd/ctlstore-cli \ && cp ${GOPATH}/bin/ctlstore-cli /usr/local/bin -RUN apk del gcc git curl alpine-sdk libc6-compat - FROM alpine RUN apk --no-cache add sqlite COPY --from=0 /bin/chamber /bin/chamber +COPY --from=0 /bin/s5cmd /bin/s5cmd COPY --from=0 /usr/local/bin/ctlstore /usr/local/bin/ COPY --from=0 /usr/local/bin/ctlstore-cli /usr/local/bin/ From 232753002f26dacd9c5d903d2497b798fa971654 Mon Sep 17 00:00:00 2001 From: Taylor Brennan Date: Wed, 9 Aug 2023 12:28:39 -0400 Subject: [PATCH 2/5] add pigz to test faster decompress --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 455001f4..062c2489 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/vers && cp ${GOPATH}/bin/ctlstore-cli /usr/local/bin FROM alpine -RUN apk --no-cache add sqlite +RUN apk --no-cache add sqlite pigz COPY --from=0 /bin/chamber /bin/chamber COPY --from=0 /bin/s5cmd /bin/s5cmd From 3c37a765b8383f73091b82ffec409130336c687c Mon Sep 17 00:00:00 2001 From: Taylor Brennan Date: Wed, 9 Aug 2023 13:30:09 -0400 Subject: [PATCH 3/5] add downloading to script --- Dockerfile | 1 + scripts/download.sh | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100755 scripts/download.sh diff --git a/Dockerfile b/Dockerfile index 062c2489..72e9c3d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/vers FROM alpine RUN apk --no-cache add sqlite pigz +COPY --from=0 /go/src/github.com/segmentio/ctlstore/scripts/download.sh . COPY --from=0 /bin/chamber /bin/chamber COPY --from=0 /bin/s5cmd /bin/s5cmd COPY --from=0 /usr/local/bin/ctlstore /usr/local/bin/ diff --git a/scripts/download.sh b/scripts/download.sh new file mode 100755 index 00000000..9fd59050 --- /dev/null +++ b/scripts/download.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh + +set -eo pipefail + +CTLSTORE_BOOTSTRAP_URL=$1 + +if [ ! -f /var/spool/ctlstore/ldb.db ]; then + mkdir -p /var/spool/ctlstore + cd /var/spool/ctlstore + s5cmd -r 0 --log debug cp $CTLSTORE_BOOTSTRAP_URL . + + if [[ ${CTLSTORE_BOOTSTRAP_URL: -2} == gz ]]; then + echo "Decompressing" + pigz -d snapshot.db.gz + fi + + mv snapshot.db ldb.db + echo "ldb.db ready" +else + echo "Snapshot already present" +fi \ No newline at end of file From 6f1fbe52dbc363472c1e60001d986439a0bc4915 Mon Sep 17 00:00:00 2001 From: Taylor Brennan Date: Wed, 9 Aug 2023 15:44:13 -0400 Subject: [PATCH 4/5] add concurrency parameter --- scripts/download.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/download.sh b/scripts/download.sh index 9fd59050..664d0d89 100755 --- a/scripts/download.sh +++ b/scripts/download.sh @@ -3,11 +3,12 @@ set -eo pipefail CTLSTORE_BOOTSTRAP_URL=$1 +CONCURRENCY=${2:-20} if [ ! -f /var/spool/ctlstore/ldb.db ]; then mkdir -p /var/spool/ctlstore cd /var/spool/ctlstore - s5cmd -r 0 --log debug cp $CTLSTORE_BOOTSTRAP_URL . + s5cmd -r 0 --log debug cp --concurrency $CONCURRENCY $CTLSTORE_BOOTSTRAP_URL . if [[ ${CTLSTORE_BOOTSTRAP_URL: -2} == gz ]]; then echo "Decompressing" From 51c6b2e95ee2ac77be9bf93221f1c2b87e76d0cc Mon Sep 17 00:00:00 2001 From: Taylor Brennan Date: Thu, 10 Aug 2023 12:59:04 -0400 Subject: [PATCH 5/5] Add metrics to the download script --- scripts/download.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/download.sh b/scripts/download.sh index 664d0d89..675aa0fb 100755 --- a/scripts/download.sh +++ b/scripts/download.sh @@ -4,19 +4,36 @@ set -eo pipefail CTLSTORE_BOOTSTRAP_URL=$1 CONCURRENCY=${2:-20} +STATS_IP=$3 +STATS_PORT=${4:-8125} +TAGS="downloaded:false" +START=$(date +%s) +END=$(date +%s) if [ ! -f /var/spool/ctlstore/ldb.db ]; then + # busybox does not support sub-second resolution + START=$(date +%s) + mkdir -p /var/spool/ctlstore cd /var/spool/ctlstore s5cmd -r 0 --log debug cp --concurrency $CONCURRENCY $CTLSTORE_BOOTSTRAP_URL . + TAGS="downloaded:true" if [[ ${CTLSTORE_BOOTSTRAP_URL: -2} == gz ]]; then echo "Decompressing" pigz -d snapshot.db.gz + TAGS="$TAGS,compressed:true" fi + TAGS="$TAGS,concurrency:$CONCURRENCY" + mv snapshot.db ldb.db - echo "ldb.db ready" + END=$(date +%s) + echo "ldb.db ready in $(($END - $START)) seconds" else echo "Snapshot already present" +fi + +if [ ! -z "$STATS_IP" ]; then + echo -n "ctlstore.reflector.init_snapshot_download_time:$(($END - $START))|h|#$TAGS" | nc -u -w1 $NODE_IP $STATS_PORT fi \ No newline at end of file