diff --git a/Dockerfile b/Dockerfile index 0e1ab5ee..72e9c3d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,10 @@ ARG VERSION RUN apk --update add gcc git curl alpine-sdk libc6-compat ca-certificates sqlite \ && curl -SsL https://github.com/segmentio/chamber/releases/download/v2.13.2/chamber-v2.13.2-linux-amd64 -o /bin/chamber \ - && chmod +x /bin/chamber + && curl -sL https://github.com/peak/s5cmd/releases/download/v2.1.0/s5cmd_2.1.0_Linux-64bit.tar.gz -o s5cmd.gz && tar -xzf s5cmd.gz -C /bin \ + && chmod +x /bin/chamber \ + && chmod +x /bin/s5cmd + COPY . /go/src/${SRC} WORKDIR /go/src/${SRC} @@ -15,11 +18,11 @@ RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/vers RUN CGO_ENABLED=1 go install -ldflags="-X github.com/segmentio/ctlstore/pkg/version.version=$VERSION" ${SRC}/pkg/cmd/ctlstore-cli \ && cp ${GOPATH}/bin/ctlstore-cli /usr/local/bin -RUN apk del gcc git curl alpine-sdk libc6-compat - FROM alpine -RUN apk --no-cache add sqlite +RUN apk --no-cache add sqlite pigz +COPY --from=0 /go/src/github.com/segmentio/ctlstore/scripts/download.sh . COPY --from=0 /bin/chamber /bin/chamber +COPY --from=0 /bin/s5cmd /bin/s5cmd COPY --from=0 /usr/local/bin/ctlstore /usr/local/bin/ COPY --from=0 /usr/local/bin/ctlstore-cli /usr/local/bin/ diff --git a/scripts/download.sh b/scripts/download.sh new file mode 100755 index 00000000..675aa0fb --- /dev/null +++ b/scripts/download.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env sh + +set -eo pipefail + +CTLSTORE_BOOTSTRAP_URL=$1 +CONCURRENCY=${2:-20} +STATS_IP=$3 +STATS_PORT=${4:-8125} + +TAGS="downloaded:false" +START=$(date +%s) +END=$(date +%s) +if [ ! -f /var/spool/ctlstore/ldb.db ]; then + # busybox does not support sub-second resolution + START=$(date +%s) + + mkdir -p /var/spool/ctlstore + cd /var/spool/ctlstore + s5cmd -r 0 --log debug cp --concurrency $CONCURRENCY $CTLSTORE_BOOTSTRAP_URL . + + TAGS="downloaded:true" + if [[ ${CTLSTORE_BOOTSTRAP_URL: -2} == gz ]]; then + echo "Decompressing" + pigz -d snapshot.db.gz + TAGS="$TAGS,compressed:true" + fi + + TAGS="$TAGS,concurrency:$CONCURRENCY" + + mv snapshot.db ldb.db + END=$(date +%s) + echo "ldb.db ready in $(($END - $START)) seconds" +else + echo "Snapshot already present" +fi + +if [ ! -z "$STATS_IP" ]; then + echo -n "ctlstore.reflector.init_snapshot_download_time:$(($END - $START))|h|#$TAGS" | nc -u -w1 $NODE_IP $STATS_PORT +fi \ No newline at end of file