Permalink
Browse files

cleaned docker scripts

  • Loading branch information...
ivangiangreco committed Nov 1, 2017
1 parent c7f50f9 commit 4107b66acce84a1d3447bc7007eae7b33350d30d
@@ -10,8 +10,7 @@ build
src/generated
*.csv
lib/grpc-assembly-0.1-SNAPSHOT.jar
scripts/docker/data/internals
scripts/docker/data/data
scripts/docker-min/shared/
scripts/docker/shared/
scripts/docker-selfcontained/data/
tmp
chronos
@@ -1,4 +1,4 @@
FROM vitrivr/adampro:latest
FROM vitrivr/adampro:latest-selfcontained
# ssh, dnsmasq
RUN apt-get -y install ssh
@@ -7,7 +7,7 @@ adampro {
port = "5890"
}
engines = ["index", "parquet", "postgres", "leveldb", "solr"]
engines = ["index", "parquet", "postgres", "solr"]
}
storage {
@@ -36,9 +36,4 @@ storage {
engine = "SolrEngine"
url = "http://MASTER_HOSTNAME:8983/solr"
}
leveldb {
engine = "LevelDbEngine"
path = "/adampro/data/data/leveldb/"
}
}
22 scripts/docker-hdfs/bootstrap.sh 100644 → 100755
@@ -35,17 +35,31 @@ if [[ $1 = "--masternode" || $2 = "--masternode" ]]; then
service dnsmasq start
$HADOOP_PREFIX/sbin/start-yarn.sh
$HADOOP_PREFIX/sbin/start-dfs.sh
# storage engines
service postgresql stop
su --login - postgres --command "$POSTGRES_HOME/bin/pg_ctl -w start -D $PGDATA"
if [[( -z "$ADAMPRO_START_POSTGRES" ) || ( "$ADAMPRO_START_POSTGRES" == "true")]]; then
service postgresql stop
su --login - postgres --command "$POSTGRES_HOME/bin/pg_ctl -w start -D $PGDATA"
fi
# start solr
if [[ (-z "$ADAMPRO_START_SOLR" ) || ( "$ADAMPRO_START_SOLR" == "true")]]; then
solr start -noprompt &
fi
# run ADAMpro
$SPARK_HOME/sbin/start-master.sh
$SPARK_HOME/bin/spark-submit --master "$ADAMPRO_MASTER" --driver-memory $ADAMPRO_DRIVER_MEMORY --executor-memory $ADAMPRO_EXECUTOR_MEMORY --deploy-mode client --driver-java-options "-Dlog4j.configuration=file:$ADAM_HOME/log4j.xml" --conf "spark.executor.extraJavaOptions=-Dlog4j.configuration=file:$ADAM_HOME/log4j.xml" --driver-java-options "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder" --conf "spark.executor.extraJavaOptions=-XX:+UnlockCommercialFeatures -XX:+FlightRecorder" --class org.vitrivr.adampro.main.Startup $ADAM_HOME/ADAMpro-assembly-0.1.0.jar &
# start web UI
java -jar $ADAM_HOME/ADAMpro-web-assembly-0.1.0.jar &
if [[ ( -z "$ADAMPRO_START_WEBUI" ) || ( "$ADAMPRO_START_WEBUI" == "true")]]; then
java -jar $ADAMPRO_HOME/ADAMpro-web-assembly-0.1.0.jar &
fi
# start notebook
if [[ ( -z "$ADAMPRO_START_NOTEBOOK" ) || ( "$ADAMPRO_START_NOTEBOOK" == "true")]]; then
$SPARK_NOTEBOOK_HOME/bin/spark-notebook -Dhttp.port=10088 &
fi
fi
@@ -2,9 +2,7 @@ version: '3.3'
services:
master:
build:
context: .
image: vitrivr/adampro-dist-hdfs
image: vitrivr/adampro:latest-hdfs
entrypoint: "/adampro/bootstrap.sh -d --masternode"
hostname: master
environment:
@@ -39,11 +37,7 @@ services:
worker:
build:
context: .
args:
MASTER_NODE: master
image: vitrivr/adampro-dist-hdfs
image: vitrivr/adampro:latest-hdfs
entrypoint: "/adampro/bootstrap.sh -d --workernode"
environment:
ADAMPRO_MASTER: spark://master:7077

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.

This file was deleted.

Oops, something went wrong.
@@ -0,0 +1,54 @@
FROM vitrivr/adampro:latest
# env variables
ENV PGDATA ${ADAMPRO_DATA}/data/postgres
# spark notebook
RUN curl https://s3.eu-central-1.amazonaws.com/spark-notebook/tgz/spark-notebook-0.7.0-scala-2.11.8-spark-2.1.0-hadoop-2.7.3-with-hive.tgz?max-keys=100000 | tar -xz -C /usr/lib/; \
cd /usr/lib && ln -s spark-notebook-0.7.0-scala-2.11.8-spark-2.1.0-hadoop-2.7.3-with-hive spark-notebook
ENV SPARK_NOTEBOOK_HOME /usr/lib/spark-notebook/
# postgresql
RUN echo deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main >> /etc/apt/sources.list.d/pgdg.list && curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && apt-get update; \
apt-get -y install postgresql-client-9.4 postgresql-9.4 postgresql-contrib-9.4 libpq-dev postgresql-server-dev-9.4
ENV POSTGRES_HOME /usr/lib/postgresql/9.4/
# solr
RUN mkdir -p /usr/lib/solr-6.1.0 && curl http://archive.apache.org/dist/lucene/solr/6.1.0/solr-6.1.0.tgz | tar -xz -C /usr/lib/; \
apt-get -y install lsof; \
ln -s /usr/lib/solr-6.1.0 /usr/lib/solr
ARG SOLR_HOME=/usr/lib/solr
ENV PATH "$PATH:/usr/lib/solr/bin"
# ADAMpro data
RUN mkdir -p ${ADAMPRO_HOME}/data; mkdir -p ${ADAMPRO_HOME}/logs/
COPY data/ ${ADAMPRO_HOME}/data/
RUN if [ -d $PGDATA ]; then chown -R postgres:postgres $PGDATA && chmod -R 700 $PGDATA; else mkdir -p $PGDATA; chown -R postgres:postgres $PGDATA; su --login - postgres --command "$POSTGRES_HOME/bin/initdb -D $PGDATA; $POSTGRES_HOME/bin/pg_ctl -w start -D $PGDATA; $POSTGRES_HOME/bin/createuser -s adampro; $POSTGRES_HOME/bin/createdb adampro; $POSTGRES_HOME/bin/pg_ctl -w stop -D $PGDATA"; fi; \
echo "host all all 0.0.0.0/0 trust" >> $PGDATA/pg_hba.conf && echo "listen_addresses='*'" >> $PGDATA/postgresql.conf; \
if [ -d $ADAMPRO_HOME/data/data/solr ]; then rm -r $SOLR_HOME/server/solr && ln -s $ADAMPRO_HOME/data/data/solr $SOLR_HOME/server/solr; else mv $SOLR_HOME/server/solr $ADAMPRO_HOME/data/data/ && ln -s $ADAMPRO_HOME/data/data/solr $SOLR_HOME/server/solr; fi
# port for postgresql
EXPOSE 5432
# port for solr
EXPOSE 8983
# port for spark UI
EXPOSE 4040
# port for grpc
EXPOSE 5890
# port for ADAMpro web UI
EXPOSE 9099
# port for spark notebook
EXPOSE 10088
# bootstrap
ENV ADAMPRO_START_POSTGRES true
ENV ADAMPRO_START_SOLR true
ENV ADAMPRO_START_WEBUI true
ENV ADAMPRO_START_NOTEBOOK true
COPY adampro.conf ${ADAMPRO_HOME}/adampro.conf.template
RUN chown root.root ${ADAMPRO_HOME}/adampro.conf; chmod 700 ${ADAMPRO_HOME}/adampro.conf
COPY bootstrap.sh ${ADAMPRO_HOME}/bootstrap.sh
RUN chown root.root ${ADAMPRO_HOME}/bootstrap.sh; chmod 700 ${ADAMPRO_HOME}/bootstrap.sh
ENTRYPOINT ["/adampro/bootstrap.sh"]
@@ -23,14 +23,14 @@ storage {
postgres {
engine = "PostgresqlEngine"
url = "jdbc:postgresql://HOSTNAME:5432/adampro"
url = "jdbc:postgresql://localhost:5432/adampro"
user = "adampro"
password = "adampro"
}
solr {
engine = "SolrEngine"
url = "http://HOSTNAME:8983/solr"
url = "http://localhost:8983/solr"
}
leveldb {
@@ -0,0 +1,33 @@
#!/bin/bash
# storage engines
if [[( -z "$ADAMPRO_START_POSTGRES" ) || ( "$ADAMPRO_START_POSTGRES" == "true")]]; then
service postgresql stop
su --login - postgres --command "$POSTGRES_HOME/bin/pg_ctl -w start -D $PGDATA"
fi
# start solr
if [[ (-z "$ADAMPRO_START_SOLR" ) || ( "$ADAMPRO_START_SOLR" == "true")]]; then
solr start -noprompt &
fi
# run ADAMpro
export SPARK_SUBMIT_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005
$SPARK_HOME/bin/spark-submit --master "$ADAMPRO_MASTER" --driver-memory "$ADAMPRO_MEMORY" --executor-memory "$ADAMPRO_MEMORY" --deploy-mode client --driver-java-options "-Dlog4j.configuration=file:$ADAMPRO_HOME/log4j.xml" --conf "spark.executor.extraJavaOptions=-Dlog4j.configuration=file:$ADAMPRO_HOME/log4j.xml" --driver-java-options "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder" --conf "spark.executor.extraJavaOptions=-XX:+UnlockCommercialFeatures -XX:+FlightRecorder" --conf "spark.sql.broadcastTimeout=3600" --class org.vitrivr.adampro.main.Startup $ADAMPRO_HOME/ADAMpro-assembly-0.1.0.jar &
# start web UI
if [[ ( -z "$ADAMPRO_START_WEBUI" ) || ( "$ADAMPRO_START_WEBUI" == "true")]]; then
java -jar $ADAMPRO_HOME/ADAMpro-web-assembly-0.1.0.jar &
fi
# start notebook
if [[ ( -z "$ADAMPRO_START_NOTEBOOK" ) || ( "$ADAMPRO_START_NOTEBOOK" == "true")]]; then
$SPARK_NOTEBOOK_HOME/bin/spark-notebook -Dhttp.port=10088 &
fi
# startup
if [[ $1 == "-bash" ]]; then
/bin/bash
fi
while true; do sleep 60 ; done
Oops, something went wrong.

0 comments on commit 4107b66

Please sign in to comment.