Skip to content

Commit

Permalink
[feat.] minor changes.
Browse files Browse the repository at this point in the history 
  - add version number in binary
  - skip background download if 'dir' didn't set in config.v1.json

Signed-off-by: Yifan Yuan <tuji.yyf@alibaba-inc.com>
  • Loading branch information
@BigVan
BigVan committed May 11, 2024
1 parent 27b69e9 commit 94933f9
Show file tree
Hide file tree
Showing 8 changed files with 66 additions and 47 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ jobs:
RELEASE_VERSION="${new_version}rc" #1.1.4rc
fi
RELEASE_VERSION=${RELEASE_VERSION#v}
COMMIT_ID="${RELEASE_VERSION}-$(git rev-parse --short HEAD)"
RELEASE_NO="$(date +%Y%m%d).$(git rev-parse --short HEAD)"
echo "RELEASE_VERSION=${RELEASE_VERSION} RELEASE_NO=${RELEASE_NO}"
Expand All @@ -67,7 +68,7 @@ jobs:
echo "OS=${OS}"
docker buildx build --build-arg BUILD_IMAGE=${BUILD_IMAGE} --build-arg OS=${OS} --build-arg RELEASE_VERSION=${RELEASE_VERSION} --build-arg RELEASE_NO=${RELEASE_NO} -f .github/workflows/release/Dockerfile --platform=${{ matrix.platforms }} -o releases/ .
docker buildx build --build-arg COMMIT_ID=${COMMIT_ID} --build-arg BUILD_IMAGE=${BUILD_IMAGE} --build-arg OS=${OS} --build-arg RELEASE_VERSION=${RELEASE_VERSION} --build-arg RELEASE_NO=${RELEASE_NO} -f .github/workflows/release/Dockerfile --platform=${{ matrix.platforms }} -o releases/ .
# remove unused package
if [[ "${OS}" =~ "ubuntu" ]]; then
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/release/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ COPY . .
ARG OS
ARG RELEASE_VERSION
ARG RELEASE_NO
RUN ls -l /src && chmod 755 .github/workflows/release/build.sh && .github/workflows/release/build.sh ${OS} ${RELEASE_VERSION} ${RELEASE_NO}
ARG COMMIT_ID
RUN ls -l /src && chmod 755 .github/workflows/release/build.sh && .github/workflows/release/build.sh ${OS} ${RELEASE_VERSION} ${RELEASE_NO} ${COMMIT_ID}

FROM scratch AS release
COPY --from=builder /src/build/overlaybd-*.* /
3 changes: 2 additions & 1 deletion .github/workflows/release/build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
OS=${1}
PACKAGE_VERSION=${2}
RELEASE_NO=${3}
COMMIT_ID=${4}
ARCH=`uname -m`
BUILD_TYPE="Release"
COMPILER=""
Expand Down Expand Up @@ -75,6 +76,6 @@ fi
# Build
mkdir build
cd build
${CMAKE} .. -DPACKAGE_VERSION=${PACKAGE_VERSION} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DBUILD_TESTING=0 -DENABLE_DSA=0 -DENABLE_ISAL=0 ${PACKAGE_RELEASE} ${COMPILER}
${CMAKE} .. -DOBD_VER="overlaybd/${COMMIT_ID}" -DPACKAGE_VERSION=${PACKAGE_VERSION} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DBUILD_TESTING=0 -DENABLE_DSA=0 -DENABLE_ISAL=0 ${PACKAGE_RELEASE} ${COMPILER}
make -j8
${CPACK} --verbose
5 changes: 4 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,15 @@ if (NOT (${ARCH} STREQUAL x86_64) AND NOT (${ARCH} STREQUAL aarch64) AND NOT (${
message(FATAL_ERROR "Unknown CPU architecture ${ARCH}")
endif ()

option(OBD_VER "Overlaybd version" "overlaybd/0.0.0-undefined")

set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build/output")
set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build/output")

set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMake")

set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fpic -Wall -Werror=sign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpic -Wall -Werror=sign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpic -Wall -Werror=sign-compare -DOVERLAYBD_VER=${OBD_VER}")

if (${ARCH} STREQUAL aarch64)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsigned-char")
Expand All @@ -39,6 +41,7 @@ endif()
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED on)
set(ENABLE_MIMIC_VDSO off)

option(BUILD_CURL_FROM_SOURCE "Compile static libcurl" off)
option(ORIGIN_EXT2FS "Use original libext2fs" off)
find_package(photon REQUIRED)
Expand Down
4 changes: 4 additions & 0 deletions src/config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@
#include <vector>
#include "overlaybd/config_util.h"

#define MACROTOSTR(x) #x
#define PRINTMACRO(x) MACROTOSTR(x)
static const char OVERLAYBD_VERSION[] = PRINTMACRO(OVERLAYBD_VER);

namespace ImageConfigNS {
const int MAX_LAYER_CNT = 256;

Expand Down
91 changes: 48 additions & 43 deletions src/image_file.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ IFile *ImageFile::__open_ro_target_file(const std::string &path) {
}

IFile *ImageFile::__open_ro_target_remote(const std::string &dir, const std::string &data_digest,
const uint64_t size, int layer_index) {
const uint64_t size, int layer_index) {
std::string url;

if (conf.repoBlobUrl() == "") {
Expand Down Expand Up @@ -153,8 +153,7 @@ IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &di
LOG_ERROR_RETURN(0, nullptr, "empty repoBlobUrl for remote layer");
}
estring url = estring().appends("/", conf.repoBlobUrl(),
(conf.repoBlobUrl().back() != '/') ? "/" : "",
digest);
(conf.repoBlobUrl().back() != '/') ? "/" : "", digest);

LOG_INFO("open file from remotefs: `, size: `", url, size);
IFile *remote_file = image_service.global_fs.remote_fs->open(url.c_str(), O_RDONLY);
Expand All @@ -164,8 +163,14 @@ IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &di
set_failed("failed to open remote file ", url, ": ", err_msg);
LOG_ERRNO_RETURN(0, nullptr, "failed to open remote file `: `", url, err_msg);
}
remote_file->ioctl(SET_SIZE, size);
remote_file->ioctl(SET_LOCAL_DIR, dir);
if (!dir.empty()) {
remote_file->ioctl(SET_SIZE, size);
remote_file->ioctl(SET_LOCAL_DIR, dir);
} else {
LOG_WARN(
"local dir of layer %d (%s) didn't set, skip background anyway",
layer_index, digest.c_str());
}

IFile *tar_file = new_tar_file_adaptor(remote_file);
if (!tar_file) {
Expand All @@ -183,15 +188,15 @@ IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &di
LOG_ERRNO_RETURN(0, nullptr, "failed to open switch file `", url);
}

if (conf.HasMember("download") && conf.download().enable() == 1) {
if (conf.HasMember("download") && conf.download().enable() == 1 && !dir.empty()) {
// download from registry, verify sha256 after downloaded.
IFile *srcfile = image_service.global_fs.srcfs->open(url.c_str(), O_RDONLY);
if (srcfile == nullptr) {
LOG_WARN("failed to open source file, ignore download");
} else {
BKDL::BkDownload *obj =
new BKDL::BkDownload(switch_file, srcfile, size, dir, digest, url, m_status,
conf.download().maxMBps(), conf.download().tryCnt(), conf.download().blockSize());
BKDL::BkDownload *obj = new BKDL::BkDownload(
switch_file, srcfile, size, dir, digest, url, m_status, conf.download().maxMBps(),
conf.download().tryCnt(), conf.download().blockSize());
LOG_DEBUG("add to download list for `", dir);
dl_list.push_back(obj);
}
Expand All @@ -209,8 +214,8 @@ void ImageFile::start_bk_dl_thread() {
uint64_t extra_range = conf.download().delayExtra();
extra_range = (extra_range <= 0) ? 30 : extra_range;
uint64_t delay_sec = (rand() % extra_range) + conf.download().delay();
LOG_INFO("background download is enabled, delay `, maxMBps `, tryCnt `, blockSize `",
delay_sec, conf.download().maxMBps(), conf.download().tryCnt(), conf.download().blockSize());
LOG_INFO("background download is enabled, delay `, maxMBps `, tryCnt `, blockSize `", delay_sec,
conf.download().maxMBps(), conf.download().tryCnt(), conf.download().blockSize());
dl_thread_jh = photon::thread_enable_join(
photon::thread_create11(&BKDL::bk_download_proc, dl_list, delay_sec, m_status));
}
Expand Down Expand Up @@ -256,30 +261,30 @@ void *do_parallel_open_files(ImageFile *imgfile, ParallelOpenTask &tm) {
return nullptr;
}

int ImageFile::open_lower_layer(IFile *&file, ImageConfigNS::LayerConfig &layer,
int index) {
std::string opened;
IFile *ImageFile::open_localfile(ImageConfigNS::LayerConfig &layer, std::string &opened) {
if (layer.file() != "") {
opened = layer.file();
file = __open_ro_file(opened);
} else {
// open downloaded blob or remote blob
if (BKDL::check_downloaded(layer.dir())) {
opened = layer.dir() + "/" + COMMIT_FILE_NAME;
file = __open_ro_file(opened);
} else {
auto sealed = layer.dir() + "/" + SEALED_FILE_NAME;
if (::access(sealed.c_str(), 0) == 0) {
// open sealed blob
opened = sealed;
file = __open_ro_file(opened);
} else {
opened = layer.digest();
file = __open_ro_remote(layer.dir(), layer.digest(), layer.size(), index);
}
}
return __open_ro_file(opened);
}
if (BKDL::check_downloaded(layer.dir())) {
opened = layer.dir() + "/" + COMMIT_FILE_NAME;
return __open_ro_file(opened);
}
auto sealed = layer.dir() + "/" + SEALED_FILE_NAME;
if (::access(sealed.c_str(), 0) == 0) {
// open sealed blob
opened = sealed;
return __open_ro_file(opened);
}
return nullptr;
}
int ImageFile::open_lower_layer(IFile *&file, ImageConfigNS::LayerConfig &layer, int index) {
std::string opened;
file = open_localfile(layer, opened); // try to open localfile if downloaded
if (file == nullptr) {
opened = layer.digest();
file = __open_ro_remote(layer.dir(), layer.digest(), layer.size(), index);
}

if (file == nullptr) {
return -1;
}
Expand All @@ -305,8 +310,8 @@ int ImageFile::open_lower_layer(IFile *&file, ImageConfigNS::LayerConfig &layer,
}
target_file = new_gzfile(target_file, gz_index, true);
if (image_service.global_conf.gzipCacheConfig().enable() && layer.targetDigest() != "") {
target_file = image_service.global_fs.gzcache_fs->
open_cached_gzip_file(target_file, layer.targetDigest().c_str());
target_file = image_service.global_fs.gzcache_fs->open_cached_gzip_file(
target_file, layer.targetDigest().c_str());
}
}
if (target_file != nullptr) {
Expand Down Expand Up @@ -395,7 +400,8 @@ LSMT::IFileRW *ImageFile::open_upper(ImageConfigNS::UpperConfig &upper) {
}

if (upper.target() != "") {
LOG_INFO("turboOCIv1 upper layer : `, `, `, `", upper.index(), upper.data(), upper.target());
LOG_INFO("turboOCIv1 upper layer : `, `, `, `", upper.index(), upper.data(),
upper.target());
target_file = open_localfile_adaptor(upper.target().c_str(), O_RDWR, 0644);
if (!target_file) {
LOG_ERROR("open(`,flags), `:`", upper.target(), errno, strerror(errno));
Expand All @@ -411,16 +417,16 @@ LSMT::IFileRW *ImageFile::open_upper(ImageConfigNS::UpperConfig &upper) {
}
ret = LSMT::open_warpfile_rw(idx_file, data_file, target_file, true);
if (!ret) {
LOG_ERROR("LSMT::open_warpfile_rw(`,`,`,`) return NULL",
(uint64_t)data_file, (uint64_t)idx_file, (uint64_t)target_file, true);
LOG_ERROR("LSMT::open_warpfile_rw(`,`,`,`) return NULL", (uint64_t)data_file,
(uint64_t)idx_file, (uint64_t)target_file, true);
goto ERROR_EXIT;
}
} else {
LOG_INFO("overlaybd upper layer : ` , `", upper.index(), upper.data());
ret = LSMT::open_file_rw(data_file, idx_file, true);
if (!ret) {
LOG_ERROR("LSMT::open_file_rw(`,`,`) return NULL",
(uint64_t)data_file, (uint64_t)idx_file, true);
LOG_ERROR("LSMT::open_file_rw(`,`,`) return NULL", (uint64_t)data_file,
(uint64_t)idx_file, true);
goto ERROR_EXIT;
}
}
Expand Down Expand Up @@ -453,8 +459,7 @@ int ImageFile::init_image_file() {
LOG_INFO("Acceleration layer found at `, ignore the last lower", accel_layer);

std::string trace_file = accel_layer + "/trace";
if (Prefetcher::detect_mode(trace_file) ==
Prefetcher::Mode::Replay) {
if (Prefetcher::detect_mode(trace_file) == Prefetcher::Mode::Replay) {
m_prefetcher = new_prefetcher(trace_file, concurrency);
}

Expand Down Expand Up @@ -519,8 +524,8 @@ void ImageFile::set_auth_failed() {
}
}

template<typename...Ts>
void ImageFile::set_failed(const Ts&...xs) {
template <typename... Ts>
void ImageFile::set_failed(const Ts &...xs) {
if (m_status == 0) // only set exit in image boot phase
{
m_status = -1;
Expand Down
2 changes: 2 additions & 0 deletions src/image_file.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ class ImageFile : public photon::fs::ForwardFile {
template<typename...Ts> void set_failed(const Ts&...xs);
LSMT::IFileRO *open_lowers(std::vector<ImageConfigNS::LayerConfig> &, bool &);
LSMT::IFileRW *open_upper(ImageConfigNS::UpperConfig &);

IFile *open_localfile(ImageConfigNS::LayerConfig &layer, std::string &opened);
IFile *__open_ro_file(const std::string &);
IFile *__open_ro_target_file(const std::string &);
IFile *__open_ro_remote(const std::string &dir, const std::string &, const uint64_t, int);
Expand Down
2 changes: 2 additions & 0 deletions src/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,8 @@ int main(int argc, char **argv) {
LOG_INFO("reset netlink done");
}

LOG_INFO("current version: `", OVERLAYBD_VERSION);

std::vector<struct tcmulib_handler> handlers;
struct tcmulib_handler overlaybd_handler;
overlaybd_handler.name = "Handler for overlaybd devices";
Expand Down

0 comments on commit 94933f9

Please sign in to comment.