Skip to content

add db clients #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 17, 2022
Merged

add db clients #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7faaa5d
add db clients
haobibo Nov 17, 2022
6c75b52
Update Dockerfile
haobibo Nov 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions docker_bigdata/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,15 @@ ENV JAVA_HOME=/opt/jdk
ENV PATH=$PATH:${CONDA_PREFIX}/bin:${JAVA_HOME}/bin

RUN source /opt/utils/script-setup.sh \
&& source /opt/utils/script-setup-db-clients.sh \
&& install_apt /opt/utils/install_list_base.apt \
&& echo "Install tini" && setup_tini \
# pyflink does not yet support Python > 3.8
&& echo "Install Mamba, Python 3.8, and Conda:" && setup_mamba && setup_conda_with_mamba "3.8" \
&& echo "Install postgresql client:" && setup_postgresql_client \
&& echo "Install mysql client:" && setup_mysql_client \
&& echo "Install mongosh:" && setup_mongosh_client \
&& echo "Install redis-cli:" && setup_redis_client \
# pyflink does not yet support Python > 3.9
&& echo "Install Mamba, Python 3.9, and Conda:" && setup_mamba && setup_conda_with_mamba "3.9" \
&& echo "Install JDK:" && VERSION_JDK=11 setup_java_base \
&& echo "Install pyflink:" && install_pip /opt/utils/list_install_pip_pyflink.txt \
&& echo "Install pyspark:" && install_pip /opt/utils/list_install_pip_pyspark.txt \
Expand Down
13 changes: 0 additions & 13 deletions docker_bigdata/work/script-db-clients.sh
View file
Open in desktop

This file was deleted.

42 changes: 42 additions & 0 deletions docker_bigdata/work/script-setup-db-clients.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
source /opt/utils/script-utils.sh


setup_postgresql_client() {
local VER_PG=${VERSION_PG:-"14"}
# from: https://www.postgresql.org/download/linux/ubuntu/
echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
# will download ~9MB files and use ~55MB disk after installation
sudo apt-get -y install "postgresql-client-${VER_PG}"
echo "@ Version of psql client: $(psql --version)"
}


setup_mysql_client() {
sudo apt-get update
# will download ~5MB files and use ~76MB disk after installation
sudo apt-get -y install mysql-client
echo "@ Version of mysql client: $(mysql --version)"
}


setup_mongosh_client() {
# from: https://www.mongodb.com/docs/mongodb-shell/install/
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" > /etc/apt/sources.list.d/mongodb-org-6.0.list
curl -sL https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
sudo apt-get update
# will download ~38MB files and use ~218MB disk after installation
sudo apt-get -y install mongodb-mongosh
echo "@ Version of mongosh client: $(mongosh --version)"
}


setup_redis_client() {
# from https://redis.io/docs/getting-started/installation/install-redis-on-linux/
curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
sudo apt-get update
sudo apt-get -y install redis-tools
echo "@ Version of redis-cli: $(redis-cli --version)"
}
40 changes: 22 additions & 18 deletions tool.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,20 @@
#!/bin/bash
# set -exu
set -xu

export REGISTRY_URL="docker.io" # docker.io or other registry URL, DOCKER_REGISTRY_USER/DOCKER_REGISTRY_PASSWORD to be set in CI env.
export BUILDKIT_PROGRESS="plain" # Full logs for CI build.
# DOCKER_REGISTRY_USER and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets.

CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/data-lab"}
CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"}
CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/')

if [ "${CI_PROJECT_BRANCH}" = "main" ] ; then
export CI_PROJECT_NAMESPACE=$(echo "$(dirname ${CI_PROJECT_NAME})") ;
# If on the main branch, docker images namespace will be same as CI_PROJECT_NAME's name space
export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})" ;
else
export CI_PROJECT_NAMESPACE=$(echo "$(dirname ${CI_PROJECT_NAME})")0${CI_PROJECT_BRANCH} ;
# not main branch, docker namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}
export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})0${CI_PROJECT_SPACE}" ;
fi

export NAMESPACE=$(echo "${REGISTRY_URL:-"docker.io"}/${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}')
Expand All @@ -22,36 +25,37 @@ jq '.experimental=true' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /
sudo service docker restart

build_image() {
echo $@ ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=`date +%Y.%m%d`;
docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "$(dirname $FILE)" ;
echo "$@" ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)";
docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ;
docker tag "${NAMESPACE}/${IMG}:${TAG}" "${NAMESPACE}/${IMG}:${VER}" ;
}

build_image_no_tag() {
echo $@ ;
IMG=$1; TAG=$2; FILE=$3; shift 3;
docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "$(dirname $FILE)" ;
echo "$@" ;
IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)";
docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ;
}

build_image_common() {
echo $@ ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=`date +%Y.%m%d`;
docker build --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "$(dirname $FILE)" ;
echo "$@" ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)";
docker build --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ;
docker tag "${NAMESPACE}/${IMG}:${TAG}" "${NAMESPACE}/${IMG}:${VER}" ;
}

alias_image() {
IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=`date +%Y.%m%d`;
IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M);
docker tag "${NAMESPACE}/${IMG_1}:${TAG_1}" "${NAMESPACE}/${IMG_2}:${TAG_2}" ;
docker tag "${NAMESPACE}/${IMG_2}:${TAG_2}" "${NAMESPACE}/${IMG_2}:${VER}" ;
}

push_image() {
docker image prune --force && docker images ;
IMGS=$(docker images | grep "second" | awk '{print $1 ":" $2}') ;
KEYWORD="${1:-second}";
docker image prune --force && docker images | sort;
IMAGES=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ;
echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_URL}" -u "$DOCKER_REGISTRY_USER" --password-stdin ;
for IMG in $(echo $IMGS | tr " " "\n") ;
for IMG in $(echo "${IMAGES}" | tr " " "\n") ;
do
docker push "${IMG}";
status=$?;
Expand All @@ -60,8 +64,8 @@ push_image() {
}

remove_folder() {
sudo du -h -d1 $1 || true ;
sudo rm -rf $1 || true ;
sudo du -h -d1 "$1" || true ;
sudo rm -rf "$1" || true ;
}

free_diskspace() {
Expand Down