Merge branch 'main' into gzstream

Signed-off-by: Yifan Yuan <tuji.yyf@alibaba-inc.com>

.github/workflows/release/build.sh

@@ -50,11 +50,11 @@ elif [[ ${OS} =~ "centos" ]]; then
     fi
 
     yum install -y epel-release libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel
-    yum install -y rpm-build make git wget sudo
-    yum install --skip-broken -y libzstd-static libzstd-devel
+    yum install -y rpm-build make git wget sudo autoconf automake libtool
+    yum install --skip-broken -y libzstd-static gcc gcc-c++ binutils libzstd-devel
 elif [[ ${OS} =~ "mariner" ]]; then
     yum install -y libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel glibc-devel libzstd-devel binutils ca-certificates-microsoft build-essential
-    yum install -y rpm-build make git wget sudo tar gcc gcc-c++
+    yum install -y rpm-build make git wget sudo tar gcc gcc-c++ autoconf automake libtool
 
     DISTRO=${OS/:/.}
     PACKAGE_RELEASE="-DPACKAGE_RELEASE=1.${DISTRO}"

CMake/Findcurl.cmake

@@ -0,0 +1,67 @@
+include(FetchContent)
+
+if(${BUILD_CURL_FROM_SOURCE})
+    message("Add and build standalone libcurl")
+    include(FetchContent)
+    FetchContent_Declare(
+        curl_bundle
+        GIT_REPOSITORY https://github.com/curl/curl.git
+        GIT_TAG curl-7_42_1
+        GIT_PROGRESS 1)
+
+    FetchContent_GetProperties(curl_bundle)
+
+    # In libcurl, CMakeLists build static lib is broken add build command via
+    # make
+    if(NOT TARGET libcurl_static_build)
+        if (NOT curl_bundle_POPULATED)
+            FetchContent_Populate(curl_bundle)
+        endif()
+        find_package(openssl)
+        add_custom_command(
+            OUTPUT ${curl_bundle_BINARY_DIR}/lib/libcurl.a
+            WORKING_DIRECTORY ${curl_bundle_SOURCE_DIR}
+            COMMAND
+                export CC=${CMAKE_C_COMPILER} &&
+                export CXX=${CMAKE_CXX_COMPILER} &&
+                export LD=${CMAKE_LINKER} &&
+                export CFLAGS=-fPIC &&
+                export LIBS=-ldl &&
+                autoreconf -i && sh configure --with-ssl="${OPENSSL_ROOT_DIR}"
+                --without-libssh2 --enable-static --enable-shared=no --enable-optimize
+                --disable-manual --without-libidn
+                --disable-ftp --disable-file --disable-ldap --disable-ldaps
+                --disable-rtsp --disable-dict --disable-telnet --disable-tftp
+                --disable-pop3 --disable-imap --disable-smb --disable-smtp
+                --disable-gopher --without-nghttp2 --enable-http
+                --with-pic=PIC
+                --prefix="${curl_bundle_BINARY_DIR}" && make -j 8 && make install)
+        add_custom_target(libcurl_static_build
+                          DEPENDS ${curl_bundle_BINARY_DIR}/lib/libcurl.a)
+        add_dependencies(libcurl_static_build openssl102_static_build)
+        make_directory(${curl_bundle_BINARY_DIR}/include)
+    endif()
+
+    set(CURL_FOUND yes)
+    set(CURL_LIBRARY ${curl_bundle_BINARY_DIR}/lib/libcurl.a)
+    set(CURL_THIRDPARTY_DEPS OpenSSL::SSL OpenSSL::Crypto z)
+    set(CURL_LIBRARIES ${CURL_LIBRARY} ${CURL_THIRDPARTY_DEPS})
+    set(CURL_INCLUDE_DIR ${curl_bundle_BINARY_DIR}/include)
+    set(CURL_INCLUDE_DIRS ${CURL_INCLUDE_DIR})
+    set(CURL_VERSION_STRING 7.42.1)
+
+    # Use libcurl static lib instead of cmake defined shared lib
+    if(NOT TARGET CURL::libcurl)
+        add_library(CURL::libcurl UNKNOWN IMPORTED)
+    endif()
+    add_dependencies(CURL::libcurl libcurl_static_build)
+    message("${CURL_LIBRARY}")
+    set_target_properties(
+        CURL::libcurl
+        PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+                   IMPORTED_LOCATION "${CURL_LIBRARY}"
+                   INTERFACE_INCLUDE_DIRECTORIES "${CURL_INCLUDE_DIRS}"
+                   INTERFACE_LINK_LIBRARIES "${CURL_THIRDPARTY_DEPS}")
+else()
+    include(${CMAKE_ROOT}/Modules/FindCURL.cmake)
+endif()

CMake/Findopenssl.cmake

@@ -0,0 +1,67 @@
+include(FetchContent)
+
+if(${BUILD_CURL_FROM_SOURCE})
+    message("Add and build standalone libopenssl")
+    include(FetchContent)
+
+    # make openssl into bundle
+    FetchContent_Declare(
+        openssl102
+        GIT_REPOSITORY https://github.com/openssl/openssl.git
+        GIT_TAG OpenSSL_1_0_2-stable
+        GIT_PROGRESS 1)
+
+    FetchContent_GetProperties(openssl102)
+
+    if(NOT TARGET openssl102_static_build)
+        if(NOT openssl102_POPULATED)
+            FetchContent_Populate(openssl102)
+        endif()
+        add_custom_command(
+            OUTPUT ${openssl102_BINARY_DIR}/lib/libssl.a
+            WORKING_DIRECTORY ${openssl102_SOURCE_DIR}
+            COMMAND
+                sh config -fPIC no-unit-test no-shared
+                --openssldir="${openssl102_BINARY_DIR}"
+                --prefix="${openssl102_BINARY_DIR}" && make depend -j && make
+                -j 8 && make install)
+        add_custom_target(openssl102_static_build
+                          DEPENDS ${openssl102_BINARY_DIR}/lib/libssl.a)
+        make_directory(${openssl102_BINARY_DIR}/include)
+    endif()
+
+    set(OPENSSL_FOUND yes)
+    set(OPENSSL_ROOT_DIR ${openssl102_BINARY_DIR})
+    set(OPENSSL_INCLUDE_DIR ${OPENSSL_ROOT_DIR}/include)
+    set(OPENSSL_INCLUDE_DIRS ${OPENSSL_INCLUDE_DIR})
+    set(OPENSSL_SSL_LIBRARY ${OPENSSL_ROOT_DIR}/lib/libssl.a)
+    set(OPENSSL_SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY})
+    set(OPENSSL_CRYPTO_LIBRARY ${OPENSSL_ROOT_DIR}/lib/libcrypto.a)
+    set(OPENSSL_CRYPTO_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY})
+    set(OPENSSL_LINK_DIR ${OPENSSL_ROOT_DIR}/lib)
+    set(OPENSSL_LINK_DIRS ${OPENSSL_LINK_DIR})
+
+    if(NOT TARGET OpenSSL::SSL)
+        add_library(OpenSSL::SSL STATIC IMPORTED)
+        add_dependencies(OpenSSL::SSL openssl102_static_build)
+        set_target_properties(
+            OpenSSL::SSL
+            PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+                       IMPORTED_LOCATION "${OPENSSL_SSL_LIBRARY}"
+                       INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIRS}"
+                       INTERFACE_LINK_LIBRARIES "${OPENSSL_SSL_LIBRARY}")
+    endif()
+
+    if(NOT TARGET OpenSSL::Crypto)
+        add_library(OpenSSL::Crypto STATIC IMPORTED)
+        add_dependencies(OpenSSL::Crypto openssl102_static_build)
+        set_target_properties(
+            OpenSSL::Crypto
+            PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+                       IMPORTED_LOCATION "${OPENSSL_CRYPTO_LIBRARY}"
+                       INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIRS}"
+                       INTERFACE_LINK_LIBRARIES "${OPENSSL_CRYPTO_LIBRARY}")
+    endif()
+else()
+    include(${CMAKE_ROOT}/Modules/FindOpenSSL.cmake)
+endif()

CMake/Findphoton.cmake

@@ -4,7 +4,7 @@ set(FETCHCONTENT_QUIET false)
 FetchContent_Declare(
   photon
   GIT_REPOSITORY https://github.com/alibaba/PhotonLibOS.git
-  GIT_TAG v0.6.6
+  GIT_TAG v0.6.13
 )
 
 if(BUILD_TESTING)
@@ -14,4 +14,11 @@ if(BUILD_TESTING)
 else()
   FetchContent_MakeAvailable(photon)
 endif()
+
+if (BUILD_CURL_FROM_SOURCE)
+  find_package(openssl REQUIRED)
+  find_package(curl REQUIRED)
+  add_dependencies(photon_obj CURL::libcurl OpenSSL::SSL OpenSSL::Crypto)
+endif()
+
 set(PHOTON_INCLUDE_DIR ${photon_SOURCE_DIR}/include/)

CMakeLists.txt

@@ -39,7 +39,7 @@ endif()
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED on)
 set(ENABLE_MIMIC_VDSO off)
-
+option(BUILD_CURL_FROM_SOURCE "Compile static libcurl" off)
 find_package(photon REQUIRED)
 find_package(tcmu REQUIRED)
 find_package(yamlcpp)

README.md

@@ -77,6 +77,14 @@ make -j
 sudo make install
 ```
 
+Considering some libcurl and libopenssl has API changes, if want to build a make-sured compatible version libcurl and openssl, and link to executable as static library.
+
+Noticed that building libcurl and openssl depends on `autoconf` `automake` and `libtool`.
+
+```bash
+cmake -D BUILD_CURL_FROM_SOURCE=1 ..
+```
+
 If you want to use the [original libext2fs](https://github.com/tytso/e2fsprogs) instead of our [customized libext2fs](https://github.com/data-accelerator/e2fsprogs).
 
 ```bash
@@ -191,7 +199,7 @@ Default configure file `overlaybd.json` is installed to `/etc/overlaybd/`.
 | enableAudit         | Enable audit or not.                                                                                  |
 | enableThread        | Enable overlaybd device run in seprate thread or not. Note `cacheType` should be `ocf`. `false` is default. |
 | auditPath           | The path for audit file, `/var/log/overlaybd-audit.log` is the default value.                         |
-| registryFsVersion   | registry client version, 'v1' libcurl based, 'v2' is photon http based. 'v1' is the default value.    |
+| registryFsVersion   | registry client version, 'v1' libcurl based, 'v2' is photon http based. 'v2' is the default value.    |
 | prefetchConfig.concurrency    | Prefetch concurrency for reloading trace, `16` is default                                   |
 
 > NOTE: `download` is the config for background downloading. After an overlaybd device is lauched, a background task will be running to fetch the whole blobs into local directories. After downloading, I/O requests are directed to local files. Unlike other options, download config is reloaded when a device launching.

src/CMakeLists.txt

@@ -1,5 +1,5 @@
-find_package(CURL REQUIRED)
-find_package(OpenSSL REQUIRED)
+find_package(curl REQUIRED)
+find_package(openssl REQUIRED)
 find_package(aio REQUIRED)
 find_package(rapidjson REQUIRED)
 
@@ -10,7 +10,6 @@ add_subdirectory(overlaybd)
 add_library(overlaybd_image_lib
   image_file.cpp
   image_service.cpp
-  sure_file.cpp
   switch_file.cpp
   bk_download.cpp
   prefetch.cpp

src/bk_download.cpp

@@ -92,8 +92,7 @@ bool BkDownload::download_done() {
 
     int ret = lfs->rename(old_name.c_str(), new_name.c_str());
     if (ret != 0) {
-        LOG_ERROR("rename(`,`), `:`", old_name, new_name, errno, strerror(errno));
-        return false;
+        LOG_ERRNO_RETURN(0, false, "rename(`,`) failed", old_name, new_name);
     }
     LOG_INFO("download verify done. rename(`,`) success", old_name, new_name);
     return true;
@@ -160,15 +159,15 @@ bool BkDownload::download_blob() {
     DEFER(free(buff));
 
     LOG_INFO("download blob start. (`)", url);
-    while (offset < file_size) {
+    while (offset < (ssize_t)file_size) {
         if (running != 1) {
             LOG_INFO("image file exit when background downloading");
             return false;
         }
         if (!force_download) {
             // check aleady downloaded.
             auto hole_pos = dst->lseek(offset, SEEK_HOLE);
-            if (hole_pos >= offset + bs) {
+            if (hole_pos >= offset + (ssize_t)bs) {
                 // alread downloaded
                 offset += bs;
                 continue;

src/config.h

@@ -140,7 +140,7 @@ struct GlobalConfig : public ConfigUtils::Config {
     APPCFG_PARA(p2pConfig, P2PConfig);
     APPCFG_PARA(exporterConfig, ExporterConfig);
     APPCFG_PARA(auditPath, std::string, "/var/log/overlaybd-audit.log");
-    APPCFG_PARA(registryFsVersion, std::string, "v1");
+    APPCFG_PARA(registryFsVersion, std::string, "v2");
     APPCFG_PARA(cacheConfig, CacheConfig);
     APPCFG_PARA(gzipCacheConfig, GzipCacheConfig);
     APPCFG_PARA(logConfig, LogConfig);

src/exporter_handler.h

@@ -1,12 +1,27 @@
+/*
+   Copyright The Overlaybd Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 #pragma once
 
-#include "metrics.h"
 #include <photon/common/conststr.h>
+#include <photon/common/estring.h>
+#include <photon/common/metric-meter/metrics.h>
 #include <photon/net/http/server.h>
-#include <photon/net/socket.h>
-#include "textexporter.h"
-#include <zlib.h>
 
+#include "textexporter.h"
 
 namespace ExposeMetrics {
 
@@ -23,51 +38,58 @@ namespace ExposeMetrics {
         ret.append(name.help_str()).append("\n");                           \
         ret.append(name.type_str()).append("\n");                           \
         for (auto x : va_##name) {                                          \
-            ret.append(                                                     \
-                   name.render(x.second->val(), nodename.c_str(), x.first)) \
-                .append("\n");                                              \
+            ret.append(name.render(x.second->val(), x.first)).append("\n"); \
         }                                                                   \
         ret.append("\n");                                                   \
     }
 
-
 struct ExposeRender : public photon::net::http::HTTPHandler {
-    EXPOSE_PHOTON_METRICLIST(latency, Metric::AverageLatencyCounter);
-
-    std::string nodename;
-    photon::net::http::DelegateHTTPHandler handler;
+    EXPOSE_PHOTON_METRICLIST(throughput, Metric::QPSCounter);
+    EXPOSE_PHOTON_METRICLIST(qps, Metric::QPSCounter);
+    EXPOSE_PHOTON_METRICLIST(latency, Metric::MaxLatencyCounter);
+    EXPOSE_PHOTON_METRICLIST(count, Metric::AddCounter);
+    EXPOSE_PHOTON_METRICLIST(cache, Metric::ValueCounter);
 
     template <typename... Args>
-    ExposeRender(Args&&... args)
-        : nodename(std::forward<Args>(args)...),
-          handler{this, &ExposeRender::handle_request} {}
+    ExposeRender(Args&&... args) {}
 
     std::string render() {
-        EXPOSE_TEMPLATE(alive, is_alive : gauge{node});
-        EXPOSE_TEMPLATE(latency, blob_read_average_latency : gauge{node, type} #us);
+        EXPOSE_TEMPLATE(alive, OverlayBD_Alive : gauge{node});
+        EXPOSE_TEMPLATE(throughput, OverlayBD_Read_Throughtput
+                        : gauge{node, type, mode} #Bytes / sec);
+        EXPOSE_TEMPLATE(qps, OverlayBD_QPS : gauge{node, type, mode});
+        EXPOSE_TEMPLATE(latency, OverlayBD_MaxLatency
+                        : gauge{node, type, mode} #us);
+        EXPOSE_TEMPLATE(count, OverlayBD_Count : gauge{node, type} #Bytes);
         std::string ret(alive.help_str());
         ret.append("\n")
             .append(alive.type_str())
             .append("\n")
-            .append(alive.render(1, nodename.c_str()))
+            .append(alive.render(1))
             .append("\n\n");
+        LOOP_APPEND_METRIC(ret, throughput);
+        LOOP_APPEND_METRIC(ret, qps);
         LOOP_APPEND_METRIC(ret, latency);
+        LOOP_APPEND_METRIC(ret, count);
         return ret;
     }
 
-    int handle_request(photon::net::http::Request& req, photon::net::http::Response& resp, std::string_view) override {
-        std::string str = render();
-        auto cl = str.size();
-        if (cl > 4096) {
-            LOG_ERROR_RETURN(0, -1, "RetType failed test");
-        }
+    int handle_request(photon::net::http::Request& req,
+                       photon::net::http::Response& resp,
+                       std::string_view) override {
+        auto body = render();
         resp.set_result(200);
-        resp.headers.content_length(cl);
-        resp.write((void*)str.data(), str.size());
-        return 0;
+        resp.keep_alive(true);
+        resp.headers.insert("Content-Type", "text/plain; version=0.0.4");
+        resp.headers.content_length(body.length());
+        ssize_t len = 0;
+        len = resp.write((void*)body.data(), body.length());
+        if (len == (ssize_t)body.length()) {
+            return 0;
+        } else {
+            LOG_ERRNO_RETURN(0, -1, "Failed to write exporter response");
+        }
     }
-
-    photon::net::http::DelegateHTTPHandler get_handler() { return handler; }
 };
 
 #undef LOOP_APPEND_METRIC