diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..fe27d31 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,86 @@ +cmake_minimum_required(VERSION 2.4.0) + +# Set the plugin name to build +project(Kafka) + +# Supported options: +# -DFOGLAMP_INCLUDE +# -DFOGLAMP_LIB +# -DFOGLAMP_SRC +# -DFOGLAMP_INSTALL +# +# If no -D options are given and FOGLAMP_ROOT environment variable is set +# then FogLAMP libraries and header files are pulled from FOGLAMP_ROOT path. + +set(CMAKE_CXX_FLAGS "-std=c++11 -O3") + +# Generation version header file +set_source_files_properties(version.h PROPERTIES GENERATED TRUE) +add_custom_command( + OUTPUT version.h + DEPENDS ${CMAKE_SOURCE_DIR}/VERSION + COMMAND ${CMAKE_SOURCE_DIR}/mkversion ${CMAKE_SOURCE_DIR} + COMMENT "Generating version header" + VERBATIM +) +include_directories(${CMAKE_BINARY_DIR}) + +# Set plugin type (south, north, filter) +set(PLUGIN_TYPE "north") + +# Add here all needed FogLAMP libraries as list +set(NEEDED_FOGLAMP_LIBS common-lib plugins-common-lib) + +# Find source files +file(GLOB SOURCES *.cpp) + +# Find FogLAMP includes and libs, by including FindFogLAMP.cmak file +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}) +find_package(FogLAMP) +# If errors: make clean and remove Makefile +if (NOT FOGLAMP_FOUND) + if (EXISTS "${CMAKE_BINARY_DIR}/Makefile") + execute_process(COMMAND make clean WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + file(REMOVE "${CMAKE_BINARY_DIR}/Makefile") + endif() + # Stop the build process + message(FATAL_ERROR "FogLAMP plugin '${PROJECT_NAME}' build error.") +endif() +# On success, FOGLAMP_INCLUDE_DIRS and FOGLAMP_LIB_DIRS variables are set + +# Add ./include +include_directories(include) + +# Add /usr/local/include/librdkafka +include_directories(/usr/local/include/librdkafka) + +# Add FogLAMP include dir(s) +include_directories(${FOGLAMP_INCLUDE_DIRS}) + +# Add other include paths this plugin needs +if (FOGLAMP_SRC) + include_directories(${FOGLAMP_SRC}/C/plugins/common/include) +endif() + +# Add FogLAMP lib path +link_directories(${FOGLAMP_LIB_DIRS} /usr/local/lib) + +# Create shared library +add_library(${PROJECT_NAME} SHARED ${SOURCES} version.h) + +target_link_libraries(${PROJECT_NAME} librdkafka.a) +# Add FogLAMP library names +target_link_libraries(${PROJECT_NAME} ${NEEDED_FOGLAMP_LIBS}) + +# Add additional libraries +target_link_libraries(${PROJECT_NAME} -lssl -lm -lcrypto -lz -ldl -lpthread -lrt) + +# Set the build version +set_target_properties(${PROJECT_NAME} PROPERTIES SOVERSION 1) + +set(FOGLAMP_INSTALL "" CACHE INTERNAL "") +# Install library +if (FOGLAMP_INSTALL) + message(STATUS "Installing ${PROJECT_NAME} in ${FOGLAMP_INSTALL}/plugins/${PLUGIN_TYPE}/${PROJECT_NAME}") + install(TARGETS ${PROJECT_NAME} DESTINATION ${FOGLAMP_INSTALL}/plugins/${PLUGIN_TYPE}/${PROJECT_NAME}) +endif() diff --git a/FindFogLAMP.cmake b/FindFogLAMP.cmake new file mode 100644 index 0000000..28e6a86 --- /dev/null +++ b/FindFogLAMP.cmake @@ -0,0 +1,142 @@ +# This CMake file locates the FogLAMP header files and libraries +# +# The following variables are set: +# FOGLAMP_INCLUDE_DIRS - Path(s) to FogLAMP headers files found +# FOGLAMP_LIB_DIRS - Path to FogLAMP shared libraries +# FOGLAMP_SUCCESS - Set on succes +# +# In case of error use SEND_ERROR and return() +# + +# Set defaults paths of installed FogLAMP SDK package +set(FOGLAMP_DEFAULT_INCLUDE_DIR "/usr/include/foglamp" CACHE INTERNAL "") +set(FOGLAMP_DEFAULT_LIB_DIR "/usr/lib/foglamp" CACHE INTERNAL "") + +# CMakeLists.txt options +set(FOGLAMP_SRC "" CACHE INTERNAL "") +set(FOGLAMP_INCLUDE "" CACHE INTERNAL "") +set(FOGLAMP_LIB "" CACHE INTERNAL "") + +# Return variables +set(FOGLAMP_INCLUDE_DIRS "" CACHE INTERNAL "") +set(FOGLAMP_LIB_DIRS "" CACHE INTERNAL "") +set(FOGLAMP_FOUND "" CACHE INTERNAL "") + +# No options set +# If FOGLAMP_ROOT env var is set, use it +if (NOT FOGLAMP_SRC AND NOT FOGLAMP_INCLUDE AND NOT FOGLAMP_LIB) + if (DEFINED ENV{FOGLAMP_ROOT}) + message(STATUS "No options set.\n" + " +Using found FOGLAMP_ROOT $ENV{FOGLAMP_ROOT}") + set(FOGLAMP_SRC $ENV{FOGLAMP_ROOT}) + endif() +endif() + +# -DFOGLAMP_SRC=/some_path or FOGLAMP_ROOT path +# Set return variable FOGLAMP_INCLUDE_DIRS +if (FOGLAMP_SRC) + unset(_INCLUDE_LIST CACHE) + file(GLOB_RECURSE _INCLUDE_COMMON "${FOGLAMP_SRC}/C/common/*.h") + file(GLOB_RECURSE _INCLUDE_SERVICES "${FOGLAMP_SRC}/C/services/common/*.h") + file(GLOB_RECURSE _INCLUDE_PLUGINS_FILTER_COMMON "${FOGLAMP_SRC}/C/plugins/filter/common/*.h") + list(APPEND _INCLUDE_LIST ${_INCLUDE_COMMON} ${_INCLUDE_SERVICES} ${_INCLUDE_PLUGINS_FILTER_COMMON}) + foreach(_ITEM ${_INCLUDE_LIST}) + get_filename_component(_ITEM_PATH ${_ITEM} DIRECTORY) + list(APPEND FOGLAMP_INCLUDE_DIRS ${_ITEM_PATH}) + endforeach() + # Add rapidjson header files + list(APPEND FOGLAMP_INCLUDE_DIRS "${FOGLAMP_SRC}/C/thirdparty/rapidjson/include") + unset(INCLUDE_LIST CACHE) + + list(REMOVE_DUPLICATES FOGLAMP_INCLUDE_DIRS) + + string (REPLACE ";" "\n +" DISPLAY_PATHS "${FOGLAMP_INCLUDE_DIRS}") + if (NOT DEFINED ENV{FOGLAMP_ROOT}) + message(STATUS "Using -DFOGLAMP_SRC option for includes\n +" "${DISPLAY_PATHS}") + else() + message(STATUS "Using FOGLAMP_ROOT for includes\n +" "${DISPLAY_PATHS}") + endif() + + if (NOT FOGLAMP_INCLUDE_DIRS) + message(SEND_ERROR "Needed FogLAMP header files not found in path ${FOGLAMP_SRC}/C") + return() + endif() +else() + # -DFOGLAMP_INCLUDE=/some_path + if (NOT FOGLAMP_INCLUDE) + set(FOGLAMP_INCLUDE ${FOGLAMP_DEFAULT_INCLUDE_DIR}) + message(STATUS "Using FogLAMP dev package includes " ${FOGLAMP_INCLUDE}) + else() + message(STATUS "Using -DFOGLAMP_INCLUDE option " ${FOGLAMP_INCLUDE}) + endif() + # Remove current value from cache + unset(_FIND_INCLUDES CACHE) + # Get up to date var from find_path + find_path(_FIND_INCLUDES NAMES plugin_api.h PATHS ${FOGLAMP_INCLUDE}) + if (_FIND_INCLUDES) + list(APPEND FOGLAMP_INCLUDE_DIRS ${_FIND_INCLUDES}) + endif() + # Remove current value from cache + unset(_FIND_INCLUDES CACHE) + + if (NOT FOGLAMP_INCLUDE_DIRS) + message(SEND_ERROR "Needed FogLAMP header files not found in path ${FOGLAMP_INCLUDE}") + return() + endif() +endif() + +# +# FogLAMP Libraries +# +# Check -DFOGLAMP_LIB=/some path is valid +# or use FOGLAMP_SRC/cmake_build/C/lib +# FOGLAMP_SRC might have been set to FOGLAMP_ROOT above +# +if (FOGLAMP_SRC) + # Set return variable FOGLAMP_LIB_DIRS + set(FOGLAMP_LIB "${FOGLAMP_SRC}/cmake_build/C/lib") + + if (NOT DEFINED ENV{FOGLAMP_ROOT}) + message(STATUS "Using -DFOGLAMP_SRC option for libs \n +" "${FOGLAMP_SRC}/cmake_build/C/lib") + else() + message(STATUS "Using FOGLAMP_ROOT for libs \n +" "${FOGLAMP_SRC}/cmake_build/C/lib") + endif() + + if (NOT EXISTS "${FOGLAMP_SRC}/cmake_build") + message(SEND_ERROR "FogLAMP has not been built yet in ${FOGLAMP_SRC} Compile it first.") + return() + endif() + + # Set return variable FOGLAMP_LIB_DIRS + set(FOGLAMP_LIB_DIRS "${FOGLAMP_SRC}/cmake_build/C/lib") +else() + if (NOT FOGLAMP_LIB) + set(FOGLAMP_LIB ${FOGLAMP_DEFAULT_LIB_DIR}) + message(STATUS "Using FogLAMP dev package libs " ${FOGLAMP_LIB}) + else() + message(STATUS "Using -DFOGLAMP_LIB option " ${FOGLAMP_LIB}) + endif() + # Set return variable FOGLAMP_LIB_DIRS + set(FOGLAMP_LIB_DIRS ${FOGLAMP_LIB}) +endif() + +# Check NEEDED_FOGLAMP_LIBS in libraries in FOGLAMP_LIB_DIRS +# NEEDED_FOGLAMP_LIBS variables comes from CMakeLists.txt +foreach(_LIB ${NEEDED_FOGLAMP_LIBS}) + # Remove current value from cache + unset(_FOUND_LIB CACHE) + # Get up to date var from find_library + find_library(_FOUND_LIB NAME ${_LIB} PATHS ${FOGLAMP_LIB_DIRS}) + if (_FOUND_LIB) + # Extract path form founf library file + get_filename_component(_DIR_LIB ${_FOUND_LIB} DIRECTORY) + else() + message(SEND_ERROR "Needed FogLAMP library ${_LIB} not found in ${FOGLAMP_LIB_DIRS}") + return() + endif() + # Remove current value from cache + unset(_FOUND_LIB CACHE) +endforeach() + +# Set return variable FOGLAMP_FOUND +set(FOGLAMP_FOUND "true") diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fb1fb5d --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Dianomic Systems + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE.librdkafka b/LICENSE.librdkafka new file mode 100644 index 0000000..1614926 --- /dev/null +++ b/LICENSE.librdkafka @@ -0,0 +1,25 @@ +librdkafka - Apache Kafka C driver library + +Copyright (c) 2012-2018, Magnus Edenhill +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md deleted file mode 100644 index 1370275..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# foglamp-north-kafka -A simple Kafka producer north plugin for FogLAMP diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..a9ad55b --- /dev/null +++ b/README.rst @@ -0,0 +1,142 @@ +================================ +FogLAMP "Kafka" C++ North Plugin +================================ + +This is a simple FogLAMP north plugin for sending data to Apache Kafka. + +FogLAMP acts as a Kafka producer, sending reading data to Kafka. This +implementation is a simplified producer that sends all data on a single +Kafka topic. Each message contains an asset name, timestamp and set of +readings values as a JSON document. + +Copyright +--------- + +This plugin buils upon the open source librdkafka library and as such portions +of ths work are copyright Magnbus Edenhill. + +librdkafka - Apache Kafka C driver library + +Copyright (c) 2012-2018, Magnus Edenhill +All rights reserved. + +Build +----- + +The plugin is built on top of the librdkafka library. This must be downloaded, built +and installed before the plugin can be built. + +.. code-block:: console + + $ git clone https://github.com/edenhill/librdkafka.git + $ cd librdkafka + $ ./configure + $ make + $ sudo make isntall + +To build FogLAMP "Kafka" C++ north plugin: + +.. code-block:: console + + $ mkdir build + $ cd build + $ cmake .. + +- By default the FogLAMP develop package header files and libraries + are expected to be located in /usr/include/foglamp and /usr/lib/foglamp +- If **FOGLAMP_ROOT** env var is set and no -D options are set, + the header files and libraries paths are pulled from the ones under the + FOGLAMP_ROOT directory. + Please note that you must first run 'make' in the FOGLAMP_ROOT directory. + +You may also pass one or more of the following options to cmake to override +this default behaviour: + +- **FOGLAMP_SRC** sets the path of a FogLAMP source tree +- **FOGLAMP_INCLUDE** sets the path to FogLAMP header files +- **FOGLAMP_LIB sets** the path to FogLAMP libraries +- **FOGLAMP_INSTALL** sets the installation path of Random plugin + +NOTE: + - The **FOGLAMP_INCLUDE** option should point to a location where all the FogLAMP + header files have been installed in a single directory. + - The **FOGLAMP_LIB** option should point to a location where all the FogLAMP + libraries have been installed in a single directory. + - 'make install' target is defined only when **FOGLAMP_INSTALL** is set + +Examples: + +- no options + + $ cmake .. + +- no options and FOGLAMP_ROOT set + + $ export FOGLAMP_ROOT=/some_foglamp_setup + + $ cmake .. + +- set FOGLAMP_SRC + + $ cmake -DFOGLAMP_SRC=/home/source/develop/FogLAMP .. + +- set FOGLAMP_INCLUDE + + $ cmake -DFOGLAMP_INCLUDE=/dev-package/include .. +- set FOGLAMP_LIB + + $ cmake -DFOGLAMP_LIB=/home/dev/package/lib .. +- set FOGLAMP_INSTALL + + $ cmake -DFOGLAMP_INSTALL=/home/source/develop/FogLAMP .. + + $ cmake -DFOGLAMP_INSTALL=/usr/local/foglamp .. + +********************************* +Packaging for 'kafka' north +********************************* + +This repo contains the scripts used to create a foglamp-north-kafka Debian package. + +The make_deb script +=================== + +Run the make_deb command after compiling the plugin: + +.. code-block:: console + + $ ./make_deb help + make_deb [help|clean|cleanall] + This script is used to create the Debian package of FogLAMP C++ 'kafka' north plugin + Arguments: + help - Display this help text + clean - Remove all the old versions saved in format .XXXX + cleanall - Remove all the versions, including the last one + $ + +Building a Package +================== + +Finally, run the ``make_deb`` command: + +.. code-block:: console + + $ ./make_deb + The package root directory is : /home/ubuntu/source/foglamp-north-kafka + The FogLAMP required version : >=1.4 + The package will be built in : /home/ubuntu/source/foglamp-north-http-kafka/packages/build + The architecture is set as : x86_64 + The package name is : foglamp-north-kafka-1.0.0-x86_64 + + Populating the package and updating version file...Done. + Building the new package... + dpkg-deb: building package 'foglamp-north-kafka' in 'foglamp-north-kafka-1.0.0-x86_64.deb'. + Building Complete. + $ + +Cleaning the Package Folder +=========================== + +Use the ``clean`` option to remove all the old packages and the files used to make the package. + +Use the ``cleanall`` option to remove all the packages and the files used to make the package. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..bc80560 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.5.0 diff --git a/foglamp.version b/foglamp.version new file mode 100644 index 0000000..430e744 --- /dev/null +++ b/foglamp.version @@ -0,0 +1 @@ +foglamp_version>=1.5 diff --git a/include/kafka.h b/include/kafka.h new file mode 100644 index 0000000..e53b6ae --- /dev/null +++ b/include/kafka.h @@ -0,0 +1,36 @@ +#ifndef _KAFKA_H +#define _KAFKA_H +/* + * FogLAMP Kafka north plugin. + * + * Copyright (c) 2018 Dianomic Systems + * + * Released under the Apache 2.0 Licence + * + * Author: Mark Riddoch + */ +#include +#include +#include +#include +#include + +/** + * A wrapper class for a simple producer model for Kafka using the librdkafka library + */ +class Kafka +{ + public: + Kafka(const std::string& brokers, const std::string& topic); + ~Kafka(); + uint32_t send(const std::vector readings); + void pollThread(); + private: + volatile bool m_running; + std::string m_topic; + std::thread *m_thread; + rd_kafka_t *m_rk; + rd_kafka_topic_t *m_rkt; + rd_kafka_conf_t *m_conf; +}; +#endif diff --git a/kafka.cpp b/kafka.cpp new file mode 100644 index 0000000..e33014c --- /dev/null +++ b/kafka.cpp @@ -0,0 +1,137 @@ +/* + * FogLAMP Kafka north plugin. + * + * Copyright (c) 2018 Dianomic Systems + * + * Released under the Apache 2.0 Licence + * + * Author: Mark Riddoch + */ +#include +#include +#include + +using namespace std; + +/** + * Callback for asynchronous producer results. + */ +static void dr_msg_cb(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, void *opaque) +{ + if (rkmessage->err) + { + Logger::getLogger()->error("Kafka message delivery failed: %s\n", + rd_kafka_err2str(rkmessage->err)); + } +} + +/** + * C Wrapper for the polling thread that collects prodcer feedback + */ +static void pollThreadWrapper(Kafka *kafka) +{ + kafka->pollThread(); +} + +/** + * Kafka constructor + * + * Setup the underlying C library elements and wrap them in + * this C++ class. + * + * @param brokers List of bootstrap brokers to contact + * @param topic THe Kafka topic to publish on + */ +Kafka::Kafka(const string& brokers, const string& topic) : + m_topic(topic), m_running(true) +{ +char errstr[512]; + + m_conf = rd_kafka_conf_new(); + if (rd_kafka_conf_set(m_conf, "bootstrap.servers", brokers.c_str(), + errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) + { + Logger::getLogger()->fatal(errstr); + throw exception(); + } + + rd_kafka_conf_set_dr_msg_cb(m_conf, dr_msg_cb); + m_rk = rd_kafka_new(RD_KAFKA_PRODUCER, m_conf, errstr, sizeof(errstr)); + if (!m_rk) + { + Logger::getLogger()->fatal(errstr); + throw exception(); + } + m_rkt = rd_kafka_topic_new(m_rk, topic.c_str(), NULL); + if (!m_rkt) { + Logger::getLogger()->fatal("Failed to create topic object: %s\n", + rd_kafka_err2str(rd_kafka_last_error())); + rd_kafka_destroy(m_rk); + throw exception(); + } + m_thread = new thread(pollThreadWrapper, this); +} + +/** + * Kafka destructor. + * Terminate the poll thread, close the Kafka connections/topic etc and do cleanup + */ +Kafka::~Kafka() +{ + rd_kafka_flush(m_rk, 1000); + rd_kafka_topic_destroy(m_rkt); + m_running = false; + rd_kafka_destroy(m_rk); + m_thread->join(); + delete m_thread; +} + +/** + * Polling thread used to collect delivery status + */ +void +Kafka::pollThread() +{ + while (m_running) + { + rd_kafka_poll(m_rk, 0); + usleep(100); + } +} + +/** + * Send the readings to the kafka topic + * + * @param readings The Readings to send + * @return The number of readings sent + */ +uint32_t +Kafka::send(const vector readings) +{ +ostringstream payload; +uint32_t sent = 0; + + for (auto it = readings.cbegin(); it != readings.cend(); ++it) + { + string assetName = (*it)->getAssetName(); + payload << "{ \"asset\" : \"" << assetName << "\", "; + payload << "\"timestamp\" : \"" << (*it)->getAssetDateTime(Reading::FMT_ISO8601) << "\", "; + vector datapoints = (*it)->getReadingData(); + for (auto dit = datapoints.cbegin(); dit != datapoints.cend(); + ++dit) + { + if (dit != datapoints.cbegin()) + { + payload << ","; + } + payload << "\"" << (*dit)->getName(); + payload << "\" : \"" << (*dit)->getData().toString() << "\""; + + } + payload << "}"; + rd_kafka_produce(m_rkt, RD_KAFKA_PARTITION_UA, RD_KAFKA_MSG_F_COPY, + (char *)payload.str().c_str(), payload.str().length(), NULL, 0, NULL); + sent++; + } + return sent; +} diff --git a/make_deb b/make_deb new file mode 100755 index 0000000..8fb9ebd --- /dev/null +++ b/make_deb @@ -0,0 +1,162 @@ +#!/bin/bash + +##-------------------------------------------------------------------- +## Copyright (c) 2018 Dianomic Systems +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +##-------------------------------------------------------------------- +## +## Author: Mark Riddoch +## + +set -e + +plugin_type="north" +plugin_name="kafka" +plugin_install_dirname=${plugin_name} +usage="$(basename "$0") [help|clean|cleanall] +This script is used to create the Debian package of FogLAMP C++ '${plugin_name}' ${plugin_type} plugin +Arguments: + help - Display this help text + clean - Remove all the old versions saved in format .XXXX + cleanall - Remove all the versions, including the last one" + +GIT_ROOT=`pwd` # The script must be executed from the root git directory +for i in "$@" +do + case "$i" in + clean) + echo -n "Cleaning the build folder from older versions..." + find "${GIT_ROOT}/packages/build" -maxdepth 1 | grep '.*\.[0-9][0-9][0-9][0-9]' | xargs rm -rf + echo "Done." + exit 0 + ;; + cleanall) + if [ -d "${GIT_ROOT}/packages/build" ]; then + echo -n "Cleaning the build folder..." + rm -rf ${GIT_ROOT}/packages/build/* + echo "Done." + else + echo "No build folder, skipping cleanall" + fi + exit 0 + ;; + help) + echo "${usage}" + exit 0 + ;; + *) + echo "Unrecognized option: $i" + exit 1 + ;; + esac +done + +# If the architecture has not been defined, then the script is complete +if [ "$(dpkg --print-architecture)" == "armhf" ]; then + echo "Building on arm architecture..." + architecture="armhf" +elif [ "$(dpkg --print-architecture)" == "amd64" ]; then + echo "Building on amd64 architecture..." + architecture="x86_64" +else + echo "The host architecture is not supported for this plugin!!" + exit 1 +fi + +# Get plugin version from VERSION +plugin_version=`cat ${GIT_ROOT}/VERSION` +# Get FogLAMP version dependency from foglamp_version file +foglamp_version=`cat ${GIT_ROOT}/foglamp.version | tr -d ' ' | grep 'foglamp_version' | head -1 | sed -e 's/\(.*\)version\(.*\)/\2/g'` +BUILD_ROOT="${GIT_ROOT}/packages/build" + +# Final package name +package_name="foglamp-${plugin_type}-${plugin_name}-${plugin_version}-${architecture}" + +# Print the summary of findings +echo "The package root directory is : ${GIT_ROOT}" +echo "The FogLAMP required version : ${foglamp_version}" +echo "The package will be built in : ${BUILD_ROOT}" +echo "The architecture is set as : ${architecture}" +echo "The package name is : ${package_name}" +echo + +# Create the package directory. If a directory with the same name exists, +# it is copied with a version number. + +# First, create the BUILD_ROOT folder, if necessary +if [ ! -L "${BUILD_ROOT}" -a ! -d "${BUILD_ROOT}" ]; then + mkdir -p "${BUILD_ROOT}" +fi + +# Check if the default directory exists +if [[ ! -d "${FOGLAMP_ROOT}" ]]; then + logger -p local0.err -t "foglamp.script.foglamp" "FogLAMP cannot be executed: ${FOGLAMP_ROOT} is not a valid directory." + echo "FogLAMP cannot be executed: ${FOGLAMP_ROOT} is not a valid directory." + echo "Create the enviroment variable FOGLAMP_ROOT before using FogLAMP." + echo "Specify the base directory for FogLAMP and set the variable with:" + echo "export FOGLAMP_ROOT=" + exit 1 +fi + +# Check/set LD_LIBRARY_PATH +libPathSet=0 +libdir=${FOGLAMP_ROOT}/lib; [ -d ${libdir} ] && LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | sed "s|${libdir}||g") && export LD_LIBRARY_PATH=${libdir}:${LD_LIBRARY_PATH} && libPathSet=1 +libdir=${FOGLAMP_ROOT}/cmake_build/C/lib; [ -d ${libdir} ] && LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | sed "s|${libdir}||g") && export LD_LIBRARY_PATH=${libdir}:${LD_LIBRARY_PATH} && libPathSet=1 +[ "$libPathSet" -eq "0" ] && echo "Unable to set/update LD_LIBRARY_PATH to include path of Foglamp shared libraries: check whether ${FOGLAMP_ROOT}/lib or ${FOGLAMP_ROOT}/cmake_build/C/lib exists" && exit 1 + +rm -rf build; mkdir ./build; cd ./build; cmake ..; make; cd .. + +cd "${BUILD_ROOT}" +existing_pkgs=`find . -maxdepth 1 -name "${package_name}.????" | wc -l` +existing_pkgs=$((existing_pkgs+1)) +new_stored_pkg=$(printf "${package_name}.%04d" "${existing_pkgs}") +if [ -d "${package_name}" ]; then + echo "Saving the old working environment as ${new_stored_pkg}" + mv "${package_name}" "${new_stored_pkg}" +fi +mkdir "${package_name}" + +# Populate the package directory with Debian files +# First with files common to all pla +echo -n "Populating the package and updating version file..." +cd "${package_name}" +cp -R ${GIT_ROOT}/packages/Debian/common/* . +cp -R ${GIT_ROOT}/packages/Debian/${architecture}/* . +sed -i "s/Version: 1.0.0/Version: ${plugin_version}/g" DEBIAN/control +sed -i "s/Depends: foglamp/Depends: foglamp (${foglamp_version})/g" DEBIAN/control + +mkdir -p usr/local/foglamp +cd usr/local/foglamp +mkdir -p "plugins/${plugin_type}/${plugin_install_dirname}" +cp -R --preserve=links ${GIT_ROOT}/build/lib*.so* "plugins/${plugin_type}/${plugin_install_dirname}" +echo "Done." + +# Build the package +cd "${BUILD_ROOT}" + +# Save the old versions +existing_pkgs=`find . -maxdepth 1 -name "${package_name}.deb.????" | wc -l` +existing_pkgs=$((existing_pkgs+1)) +new_stored_pkg=$(printf "${package_name}.deb.%04d" "${existing_pkgs}") + +if [ -e "${package_name}.deb" ]; then + echo "Saving the old package as ${new_stored_pkg}" + mv "${package_name}.deb" "${new_stored_pkg}" +fi + +echo "Building the new package..." +dpkg-deb --build ${package_name} +echo "Building Complete." + +exit 0 diff --git a/mkversion b/mkversion new file mode 100755 index 0000000..6f71d5b --- /dev/null +++ b/mkversion @@ -0,0 +1,11 @@ +#!/bin/sh +cat > version.h << END_WARNING + +/* + * WARNING: This is an automatically generated file. + * Do not edit this file. + * To change the version edit the file VERSION + */ + +END_WARNING +/bin/echo '#define VERSION "'`cat $1/VERSION`'"' >> version.h diff --git a/packages/Debian/armhf/DEBIAN/control b/packages/Debian/armhf/DEBIAN/control new file mode 100644 index 0000000..acce333 --- /dev/null +++ b/packages/Debian/armhf/DEBIAN/control @@ -0,0 +1,10 @@ +Package: foglamp-north-kafka +Version: 1.0.0 +Section: devel +Priority: optional +Architecture: armhf +Depends: foglamp +Conflicts: +Maintainer: Dianomic Systems, Inc. +Homepage: http://www.dianomic.com +Description: FogLAMP C++ Kafka north plugin diff --git a/packages/Debian/common/DEBIAN/postint b/packages/Debian/common/DEBIAN/postint new file mode 100755 index 0000000..add7afe --- /dev/null +++ b/packages/Debian/common/DEBIAN/postint @@ -0,0 +1,34 @@ +#!/bin/sh + +##-------------------------------------------------------------------- +## Copyright (c) 2018 Dianomic Systems +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +##-------------------------------------------------------------------- + +##-------------------------------------------------------------------- +## +## @postinst DEBIAN/postinst +## This script is used to execute post installation tasks. +## +## Author: Massimiliano Pinto +## +##-------------------------------------------------------------------- + +set -e + +set_files_ownership () { + chown -R root:root /usr/local/foglamp/plugins/north/Kafka +} + +set_files_ownership diff --git a/packages/Debian/common/DEBIAN/preinst b/packages/Debian/common/DEBIAN/preinst new file mode 100755 index 0000000..c768de4 --- /dev/null +++ b/packages/Debian/common/DEBIAN/preinst @@ -0,0 +1,65 @@ +#!/bin/sh + +##-------------------------------------------------------------------- +## Copyright (c) 2018 Dianomic Systems +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +##-------------------------------------------------------------------- + +##-------------------------------------------------------------------- +## +## @preinst DEBIAN/preinst +## This script is used to execute pre installation tasks. +## +## Author: Massimiliano Pinto +## +##-------------------------------------------------------------------- + +set -e + +PKG_NAME="foglamp-north-kafka" + +get_foglamp_script () { + foglamp_script=$(dpkg -L foglamp | grep 'foglamp/bin/foglamp$') + echo $foglamp_script +} + +is_foglamp_running () { + set +e + foglamp_script=$(get_foglamp_script) + foglamp_status_output=$($foglamp_script status 2>&1 | grep 'FogLAMP Uptime') + rc=$((!$?)) + echo $rc + set -e +} + +is_plugin_installed () { + set +e + current_files_all=$(dpkg -L ${PKG_NAME} | grep 'foglamp/plugins/north/Kafka/lib*') + rc=$((!$?)) + echo $rc + set -e +} + +# main +IS_PLUGIN_INSTALLED=$(is_plugin_installed) +if [ "$IS_PLUGIN_INSTALLED" -eq "1" ]; then + echo "FogLAMP C++ plugin ${PKG_NAME} is already installed: this is an upgrade/downgrade." + # exit if foglamp is running + IS_FOGLAMP_RUNNING=$(is_foglamp_running) + if [ "$IS_FOGLAMP_RUNNING" -eq "1" ] + then + echo "*** ERROR. FogLAMP is currently running. Stop FogLAMP and try again. ***" + exit 1 + fi +fi diff --git a/packages/Debian/x86_64/DEBIAN/control b/packages/Debian/x86_64/DEBIAN/control new file mode 100644 index 0000000..91db11d --- /dev/null +++ b/packages/Debian/x86_64/DEBIAN/control @@ -0,0 +1,10 @@ +Package: foglamp-north-kafka +Version: 1.0.0 +Section: devel +Priority: optional +Architecture: amd64 +Depends: foglamp +Conflicts: +Maintainer: Dianomic Systems, Inc. +Homepage: http://www.dianomic.com +Description: FogLAMP C++ Kafka north plugin diff --git a/plugin.cpp b/plugin.cpp new file mode 100644 index 0000000..7f72075 --- /dev/null +++ b/plugin.cpp @@ -0,0 +1,127 @@ +/* + * FogLAMP Kafka north plugin. + * + * Copyright (c) 2018 Dianomic Systems + * + * Released under the Apache 2.0 Licence + * + * Author: Mark Riddoch + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace rapidjson; + +#define PLUGIN_NAME "Kafka" +/** + * Plugin specific default configuration + */ +#define PLUGIN_DEFAULT_CONFIG "\"brokers\": { " \ + "\"description\": \"The bootstrap broker list to retrieve full Kafka brokers\", " \ + "\"type\": \"string\", " \ + "\"order\": \"1\", " \ + "\"displayName\": \"Bootstrap Brokers\", " \ + "\"default\": \"localhost:9092,kafka.local:9092\" }, " \ + "\"topic\": { " \ + "\"description\": \"The topic to send reading data on\", " \ + "\"order\": \"2\", " \ + "\"displayName\": \"Kafka Topic\", " \ + "\"type\": \"string\", \"default\": \"FogLAMP\" }, " \ + "\"source\": { " \ + "\"description\": \"The source of data to send\", " \ + "\"type\": \"enumeration\", \"default\": \"readings\", " \ + "\"order\": \"3\", " \ + "\"displayName\": \"Data Source\", " \ + "\"options\" : [\"readings\",\"statistics\"] }" + +#define KAFKA_PLUGIN_DESC "\"plugin\": {\"description\": \"Simple plugin to send data to a Kafka topic\", \"type\": \"string\", \"default\": \"" PLUGIN_NAME "\", \"readonly\": \"true\"}" + +#define PLUGIN_DEFAULT_CONFIG_INFO "{" KAFKA_PLUGIN_DESC ", " PLUGIN_DEFAULT_CONFIG "}" + +/** + * The Kafka plugin interface + */ +extern "C" { + +/** + * The C API plugin information structure + */ +static PLUGIN_INFORMATION info = { + PLUGIN_NAME, // Name + VERSION, // Version + 0, // Flags + PLUGIN_TYPE_NORTH, // Type + "1.0.0", // Interface version + PLUGIN_DEFAULT_CONFIG_INFO // Configuration +}; + +/** + * Return the information about this plugin + */ +PLUGIN_INFORMATION *plugin_info() +{ + return &info; +} + +/** + * Initialise the plugin with configuration. + * + * This function is called to get the plugin handle. + */ +PLUGIN_HANDLE plugin_init(ConfigCategory* configData) +{ + if (!configData->itemExists("brokers")) + { + Logger::getLogger()->fatal("Kafka plugin must have a bootstrap broker list defined"); + throw exception(); + } + string brokers = configData->getValue("brokers"); + if (!configData->itemExists("topic")) + { + Logger::getLogger()->fatal("Kafka plugin must define a topic"); + throw exception(); + } + string topic = configData->getValue("topic"); + + Kafka *kafka = new Kafka(brokers, topic); + + return (PLUGIN_HANDLE)kafka; +} + +/** + * Send Readings data to historian server + */ +uint32_t plugin_send(const PLUGIN_HANDLE handle, + const vector& readings) +{ +Kafka *kafka = (Kafka *)handle; + + return kafka->send(readings); +} + +/** + * Shutdown the plugin + * + * Delete allocated data + * + * @param handle The plugin handle + */ +void plugin_shutdown(PLUGIN_HANDLE handle) +{ +Kafka *kafka = (Kafka *)handle; + + delete kafka; +} + +// End of extern "C" +};