Skip to content
Permalink
Browse files

Create a dockerfile for tools/kafka_repeater; Add a configuration fil…

…e generator for kafka_repeater.
  • Loading branch information...
SwimmingTiger committed Jul 29, 2019
1 parent c78df43 commit 71ba00275d610ea9592280c4a3f404d41b9ca41b
@@ -29,7 +29,6 @@ install/run_*/
/docker/
!/docker/btcpool/deploy/
/docs/
/tools/
/Dockerfile
/README.md
/.dockerignore
@@ -0,0 +1,75 @@
<?php
// Function library for generating configuration files
function errPuts($msg) {
fputs(STDERR, $msg);
}
function fatal($msg) {
errPuts("\n\n============== FATAL ==============\n".$msg."\n\n");
exit(1);
}
function finish($msg) {
errPuts("\n\n============== FINISH ==============\n".$msg."\n\n");
exit(0);
}
function notNull($envName) {
if (!isset($_ENV[$envName])) {
fatal("Environmental variable '$envName' must be defined");
}
if (empty($_ENV[$envName])) {
fatal("Environmental variable '$envName' should not be empty");
}
return $_ENV[$envName];
}
function notNullTrim($envName) {
if (!isset($_ENV[$envName])) {
fatal("Environmental variable '$envName' must be defined");
}
$value = trim($_ENV[$envName]);
if (empty($value)) {
fatal("Environmental variable '$envName' should not be empty");
}
return $value;
}
function optional($envName, $defaultValue = '') {
if (!isset($_ENV[$envName])) {
return $defaultValue;
}
if (empty($_ENV[$envName])) {
return $defaultValue;
}
return $_ENV[$envName];
}
function optionalTrim($envName, $defaultValue = '') {
if (!isset($_ENV[$envName])) {
return $defaultValue;
}
$value = trim($_ENV[$envName]);
if (empty($value)) {
return $defaultValue;
}
return $value;
}
function isTrue($envName) {
$falseTags = ['false', 'null', 'nil', 'undefined'];
if (!isset($_ENV[$envName])) {
return false;
}
$value = strtolower(trim($_ENV[$envName]));
if (in_array($value, $falseTags)) {
return false;
}
return (bool)$value;
}
function toString($obj) {
return json_encode($obj, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
}
@@ -0,0 +1,22 @@
<?php
require_once __DIR__.'/functions.php';
// A class to automatically load environment variable,
// fix $_ENV is empty due to php.ini
class EnvAutoLoader implements ArrayAccess {
/* interface ArrayAccess */
public function offsetExists($name) {
return getenv($name) !== FALSE;
}
public function offsetGet($name) {
return getenv($name);
}
public function offsetSet($name, $value) {
throw new Exception('Not implemented');
}
public function offsetUnset($name) {
throw new Exception('Not implemented');
}
}
$_ENV = new EnvAutoLoader();
@@ -0,0 +1,127 @@
#!/usr/bin/env php
<?php
require_once __DIR__.'/../lib/init.php';
?>
#
# kafka repeater cfg
#
# @since 2018-17
# @copyright btc.com
#

kafka = {
in_brokers = "<?=notNullTrim('kafka_in_brokers')?>";
in_topic = "<?=notNullTrim('kafka_in_topic')?>";
# Used to record progress / offsets.
# Change it to reset the progress (will forward from the beginning).
# The two repeater cannot have the same group id, otherwise the result is undefined.
in_group_id = "<?=notNullTrim('kafka_in_group_id')?>";

out_brokers = "<?=notNullTrim('kafka_out_brokers')?>";
out_topic = "<?=notNullTrim('kafka_out_topic')?>";

<?php
if (isTrue('kafka_out_use_ssl')) {
$ca = notNull('kafka_ssl_ca_content');
$certificate = notNull('kafka_ssl_certificate_content');
$key = notNull('kafka_ssl_key_content');
file_put_contents('/tmp/ca.crt', $ca);
file_put_contents('/tmp/client.crt', $certificate);
file_put_contents('/tmp/client.key', $key);
?>
# authorize settings (only for out_brokers)
#
# To generate a key and a self-signed certificate, run:
# openssl genrsa -out client.key 2048
# openssl req -new -key client.key -out client.crt.req
# openssl x509 -req -days 365 -in client.crt.req -signkey client.key -out client.crt
# cp client.crt ca.crt
#
security = {
protocol = "ssl";
};
ssl = {
ca = {
location = "/tmp/ca.crt";
};
certificate = {
location = "/tmp/client.crt";
};
key = {
location = "/tmp/client.key";
password = "<?=optional('kafka_ssl_key_password')?>";
};
};
<?php
} // kafka_out_use_ssl
?>
};

###
# Choose a message converter you want to use. Set it to true to enable it.
# If no converter is enabled, kafka messages will be forwarded without any conversion.
# Currently, only one converter can be enabled at a time.
# TODO: Support for multiple converters (converter chain).
###

<?php
$convertor = strtolower(optionalTrim('message_convertor'));
?>

# Convert the Share version
# TODO: add bitcoin_v1_to_v2
share_convertor = {
bitcoin_v2_to_v1 = <?=toString($convertor == 'share_convertor_bitcoin_v2_to_v1')?>;
};

# Change the difficulty of shares according to stratum jobs from a topic.
# TODO: add bitcoin_v2
<?php
if (in_array($convertor, [
'share_diff_changer_bitcoin_v1',
'share_diff_changer_bitcoin_v2_to_v1'
])) {
?>
share_diff_changer = {
job_brokers = "<?=notNullTrim('share_diff_changer_job_brokers')?>";
job_topic = "<?=notNullTrim('share_diff_changer_job_topic')?>";
job_group_id = "<?=notNullTrim('share_diff_changer_job_group_id')?>";

bitcoin_v1 = <?=toString($convertor == 'share_diff_changer_bitcoin_v1')?>;
bitcoin_v2_to_v1 = <?=toString($convertor == 'share_diff_changer_bitcoin_v2_to_v1')?>;

# The program uses a deterministic difficulty copy logic.
# This means that no matter when you run the program, the result is the same.
#
# However, if you use the program to copy instant-generated shares, you will meet a problem:
# Shares in a time period will be modified and forwarded only after got the stratum job within this period of time,
# so current shares will always be stacked until a new job got later.
#
# Setting a fixed time offset will make the current job available for the current and subsequent shares,
# then shares will no longer be stacked.
# When the time reaches the end of the offset, usually a new stratum job is ready.
# So the copying process will become continuous rather than "pulsed".
#
# Since the difficulty changing of bitcoin (including BCH) is not very fast,
# setting a time offset within 1 minute will not have a significant effect on users' earning.
job_time_offset = 30;
};
<?php
} // share_diff_changer
?>

# Print shares only, no forwarding
share_printer = {
bitcoin_v1 = <?=toString($convertor == 'share_printer_bitcoin_v1')?>;
};

# Print kafka messages, no forwarding
message_printer = {
print_hex = <?=toString($convertor == 'message_printer_print_hex')?>;
};

# Logging options
log = {
repeated_number_display_interval = <?=optionalTrim('log_repeated_number_display_interval', 10)?>; # seconds
};
@@ -80,3 +80,22 @@ file(GLOB SOURCES

add_executable(kafka_repeater ${SOURCES})
target_link_libraries(kafka_repeater ${THIRD_LIBRARIES})


###################################### Install & Package ######################################

install(TARGETS kafka_repeater DESTINATION bin)

# generate deb package by CPack
if(POOL__GENERATE_DEB_PACKAGE)
set(CPACK_GENERATOR "DEB")
set(CPACK_PACKAGE_NAME "btcpool-kafka-repeater")
set(CPACK_DEBIAN_PACKAGE_MAINTAINER "YihaoPeng yihao.peng@bitmain.com")

message("-- Package Name: ${CPACK_PACKAGE_NAME}")

# dependencies
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)

include(CPack)
endif()
@@ -0,0 +1,74 @@
#
# Dockerfile
#
# @author hanjiang.yu@bitmain.com
# @copyright btc.com
# @since 2018-12-01
#
#
FROM ubuntu:18.04 as build
LABEL maintainer="Yihao Peng <yihao.peng@bitmain.com>"

ARG APT_MIRROR_URL
ARG BUILD_JOBS=1

COPY docker/btcpool/deploy/update_apt_sources.sh /tmp/
RUN /tmp/update_apt_sources.sh

# Install build dependencies
RUN apt-get update && apt-get install -y \
autoconf \
automake \
build-essential \
cmake \
libboost-all-dev \
libconfig++-dev \
libgoogle-glog-dev \
libssl-dev \
libtool \
libzookeeper-mt-dev \
pkg-config \
python \
wget \
zlib1g-dev \
&& apt-get autoremove && apt-get clean q&& rm -rf /var/lib/apt/lists/*

# Build librdkafka static library
# Remove dynamic libraries of librdkafka
# In this way, the constructed deb package will
# not have dependencies that not from software sources.
RUN cd /tmp && wget https://github.com/edenhill/librdkafka/archive/0.9.1.tar.gz && \
[ $(sha256sum 0.9.1.tar.gz | cut -d " " -f 1) = "5ad57e0c9a4ec8121e19f13f05bacc41556489dfe8f46ff509af567fdee98d82" ] && \
tar zxf 0.9.1.tar.gz && cd librdkafka-0.9.1 && \
./configure && make && make install && rm -rf /tmp/* && \
cd /usr/local/lib && \
find . | grep 'rdkafka' | grep '.so' | xargs rm

COPY . /work/btcpool

RUN mkdir /work/build && cd /work/build && \
cmake -DPOOL__GENERATE_DEB_PACKAGE=ON /work/btcpool/tools/kafka_repeater && \
make package -j$BUILD_JOBS

######### Release image #########
FROM ubuntu:18.04
LABEL maintainer="Yihao Peng <yihao.peng@bitmain.com>"

ARG APT_MIRROR_URL

COPY docker/btcpool/deploy/update_apt_sources.sh /tmp/
RUN /tmp/update_apt_sources.sh

# Copy deb packages and scripts
COPY --from=build /work/build/*.deb /work/package/
COPY --from=build /work/btcpool/install/cfg-generator /work/cfg-generator
COPY tools/kafka_repeater/entrypoint.sh /

# Install utilities & btcpool w/ debug symbols
RUN apt-get update && \
apt-get install -y php-cli /work/package/*.deb && \
apt-get autoremove && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

ENTRYPOINT ["/entrypoint.sh"]

0 comments on commit 71ba002

Please sign in to comment.
You can’t perform that action at this time.