-
Notifications
You must be signed in to change notification settings - Fork 35
/
uno.conf
207 lines (185 loc) · 8.25 KB
/
uno.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
#! /usr/bin/env bash
# From load-env.sh, variable UNO_HOME defaults to root directory of this Uno repo.
# Versions
# --------
export HADOOP_VERSION=${HADOOP_VERSION:-3.2.0}
export ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION:-3.4.14}
export ACCUMULO_VERSION=${ACCUMULO_VERSION:-2.0.0}
export FLUO_VERSION=${FLUO_VERSION:-1.2.0}
export FLUO_YARN_VERSION=${FLUO_YARN_VERSION:-1.0.0}
# Hashes (supports MD5, SHA-1, SHA-256, and SHA-512; MD5 requires md5sum)
# --------------
# Hashes below match default versions above. If you change a version above,
# you must also change the hash below.
export HADOOP_HASH=$(grep -F hadoop:${HADOOP_VERSION}: $UNO_HOME/conf/checksums | cut -d : -f 3)
export ZOOKEEPER_HASH=$(grep -F zookeeper:${ZOOKEEPER_VERSION}: $UNO_HOME/conf/checksums | cut -d : -f 3)
export ACCUMULO_HASH=$(grep -F accumulo:${ACCUMULO_VERSION}: $UNO_HOME/conf/checksums | cut -d : -f 3)
export FLUO_HASH=037f89cd2bfdaf76a1368256c52de46d6b9a85c9c1bfc776ec4447d02c813fb2
export FLUO_YARN_HASH=c6220d35cf23127272f3b5638c44586504dc17a46f5beecdfee5027b5ff874b0
# Network configuration
# ---------------------
# Hostname to use in configuration. Set to $(hostname) to use your hostname
export UNO_HOST=${UNO_HOST:-localhost}
# Download configuration
# ----------------------
# Directory where all depedency packages are downloaded
export DOWNLOADS=$UNO_HOME/downloads
# Tarball file names
export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
export HADOOP_TARBALL=hadoop-"$HADOOP_VERSION".tar.gz
export ZOOKEEPER_TARBALL=zookeeper-"$ZOOKEEPER_VERSION".tar.gz
export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz
# Building Accumulo
#------------------
# If set, 'uno fetch' will build (instead of downloading) an Accumulo tarball
# from that directory and copy it to the downloads directory.
#export ACCUMULO_REPO=/path/to/accumulo
# Comment out the following if block if you don't want to automatically detect
# version from the pom.xml. This could be useful if you want to switch branches
# in your workspace and don't want the detected version to change.
if [[ -n "$ACCUMULO_REPO" ]]; then
# Detect the version from the accumulo pom.xml in the workspace
ACCUMULO_VERSION=$(xmllint --shell "$ACCUMULO_REPO"/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=)
export ACCUMULO_VERSION
export ACCUMULO_TARBALL=accumulo-$ACCUMULO_VERSION-bin.tar.gz
fi
# Building Fluo
# -------------
# If set, 'uno fetch' will build (rather than download) a Fluo tarball
# from that directory and copy it to the downloads directory.
#export FLUO_REPO=/path/to/fluo
# Comment out the following if block if you don't want to automatically detect
# version from the pom.xml. This could be useful if you want to switch branches
# in your workspace and don't want the detected version to change.
if [[ -n "$FLUO_REPO" ]]; then
# Detect the version from the accumulo pom.xml in the workspace
FLUO_VERSION=$(xmllint --shell "$FLUO_REPO"/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=)
export FLUO_VERSION
export FLUO_TARBALL=fluo-$FLUO_VERSION-bin.tar.gz
fi
# Building Fluo YARN
# ------------------
# If set, 'uno fetch' will build (rather than download) a Fluo YARN tarball
# from that directory and copy it to the downloads directory.
#export FLUO_YARN_REPO=/path/to/fluo-yarn
# Comment out the following if block if you don't want to automatically detect
# version from the pom.xml. This could be useful if you want to switch branches
# in your workspace and don't want the detected version to change.
if [[ -n "$FLUO_YARN_REPO" ]]; then
# Detect the version from the accumulo pom.xml in the workspace
FLUO_YARN_VERSION=$(xmllint --shell "$FLUO_YARN_REPO"/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="version"]/text()' | grep content= | cut -f2 -d=)
export FLUO_YARN_VERSION
export FLUO_YARN_TARBALL=fluo-yarn-$FLUO_YARN_VERSION-bin.tar.gz
fi
# Version compatability checks
# ----------------------------
# Apache built Accumuo 1.x only works with Hadoop 2.x. Apache built Accumulo
# 2.x only works with Hadoop 3.x. However if building Accumulo 1.x, it can
# work with Hadoop 3.x. The following sanity checks fail fast when versions
# are incompatible. If the checks cause a problem, comment them out and
# consider submitting an issue or PR.
if [[ "${HADOOP_VERSION}" = 1.* ]]; then
echo "ERROR from uno.conf : Hadoop 1.x is not supported" 1>&2
exit 1
fi
if [[ -z "$ACCUMULO_REPO" && "${ACCUMULO_VERSION}" = 1.* && ! "${HADOOP_VERSION}" = 2.* ]]; then
echo "ERROR from uno.conf : When using Accumulo 1.x, expect Hadoop 2.x not $HADOOP_VERSION" 1>&2
exit 1
fi
if [[ "${ACCUMULO_VERSION}" = 2.* && ! "${HADOOP_VERSION}" = 3.* ]]; then
echo "ERROR from uno.conf : When using Accumulo 2.x, expect Hadoop 3.x not $HADOOP_VERSION" 1>&2
exit 1
fi
# Installation directories
# ------------------------
# Software installation directory
export INSTALL=$UNO_HOME/install
# Directory where all Fluo dependencies store their data
export DATA_DIR=$INSTALL/data
# Home directories
export ZOOKEEPER_HOME=$INSTALL/zookeeper-$ZOOKEEPER_VERSION
export HADOOP_HOME=$INSTALL/hadoop-$HADOOP_VERSION
export ACCUMULO_HOME=$INSTALL/accumulo-$ACCUMULO_VERSION
export FLUO_HOME=$INSTALL/fluo-$FLUO_VERSION
export FLUO_YARN_HOME=$INSTALL/fluo-yarn-$FLUO_YARN_VERSION
# Config directories
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
# Log directories
export LOGS_DIR=$INSTALL/logs
export ACCUMULO_LOG_DIR=$LOGS_DIR/accumulo
export HADOOP_LOG_DIR=$LOGS_DIR/hadoop
export ZOO_LOG_DIR=$LOGS_DIR/zookeeper
# Accumulo configuration
# ----------------------
# Accumulo instance name
export ACCUMULO_INSTANCE=uno
# Accumulo user
export ACCUMULO_USER=root
# Accumulo password
export ACCUMULO_PASSWORD=secret
# Plugin configuration
# --------------------
# Post-install plugins. Example: "influx-metrics accumulo-encryption"
export POST_INSTALL_PLUGINS=""
# Post-run plugins. Example: "spark"
export POST_RUN_PLUGINS=""
# Configuration for 'spark' plugin
export SPARK_VERSION=${SPARK_VERSION:-2.3.2}
export SPARK_HOME=$INSTALL/spark-${SPARK_VERSION}-bin-without-hadoop
export SPARK_TARBALL=spark-${SPARK_VERSION}-bin-without-hadoop.tgz
export SPARK_HASH=$(grep -F spark:${SPARK_VERSION}: $UNO_HOME/conf/checksums | cut -d : -f 3)
# Configuration for 'influxdb-metrics' plugin
# InfluxDB metrics can only be set up on Linux. Mac OS X is not supported.
export INFLUXDB_VERSION=0.9.4.2
export INFLUXDB_HOME=$INSTALL/influxdb-"$INFLUXDB_VERSION"
export INFLUXDB_TARBALL=influxdb-"$INFLUXDB_VERSION".tar.gz
export INFLUXDB_HASH=fe4269500ae4d3d936b1ccdd9106c5e82c56751bcf0625ed36131a51a20a1c0c
export GRAFANA_VERSION=2.5.0
export GRAFANA_HOME=$INSTALL/grafana-"$GRAFANA_VERSION"
export GRAFANA_TARBALL=grafana-"$GRAFANA_VERSION".tar.gz
export GRAFANA_HASH=d3eaa2c45ae9f8e7424a7b0b74fa8c8360bd25a1f49545d8fb5a874ebf0530fe
# Performance Profiles
# --------------------
PERFORMACE_PROFILE=8GX2
case "$PERFORMACE_PROFILE" in
8GX2)
# Enable Accumulo native map
export ACCUMULO_USE_NATIVE_MAP=false
# Accumulo tserver jvm heap size
export ACCUMULO_TSERV_MEM=768m
# Accumulo data cache size. Only applies when using Accumulo 1.x
export ACCUMULO_DCACHE_SIZE=256M
# Accumulo index cache size. Only applies when using Accumulo 1.x
export ACCUMULO_ICACHE_SIZE=128M
# Accumulo in mem map size. Only applies when using Accumulo 1.x
export ACCUMULO_IMAP_SIZE=128M
# Yarn Node Manager max memory (in MB)
export YARN_NM_MEM_MB=8192
# Yarn Node Manager max cpu vcores
export YARN_NM_CPU_VCORES=8
# Number of threads per worker process
export FLUO_WORKER_THREADS=10
# Worker process memory
export FLUO_WORKER_MEM_MB=1024
# Number worker processes
export FLUO_WORKER_INSTANCES=1
;;
32GX4)
export ACCUMULO_USE_NATIVE_MAP=false
export ACCUMULO_TSERV_MEM=6G
# Accumulo configs below only apply when using Accumulo 1.x
export ACCUMULO_DCACHE_SIZE=2G
export ACCUMULO_ICACHE_SIZE=1G
export ACCUMULO_IMAP_SIZE=1G
export YARN_NM_MEM_MB=16384
export YARN_NM_CPU_VCORES=8
export FLUO_WORKER_THREADS=64
export FLUO_WORKER_MEM_MB=4096
export FLUO_WORKER_INSTANCES=2
;;
*)
echo "Unknown performance profile"
exit 1;
;;
esac