Skip to content

Commit

Permalink
[Feat] Overlaybd-streamConv:
Browse files Browse the repository at this point in the history
   - support parsing a YAML file to configure server
   - docker image support

Signed-off-by: Yifan Yuan <tuji.yyf@alibaba-inc.com>
  • Loading branch information
BigVan committed Dec 7, 2023
1 parent 360e819 commit 0cba95e
Show file tree
Hide file tree
Showing 10 changed files with 290 additions and 50 deletions.
10 changes: 10 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,16 @@ set(ENABLE_MIMIC_VDSO off)

find_package(photon REQUIRED)
find_package(tcmu REQUIRED)
find_package(yamlcpp)
if (NOT yamlcpp_FOUND)
FetchContent_Declare(
yamlcpp
GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git
GIT_TAG 0.8.0
)
FetchContent_MakeAvailable(yamlcpp)
endif()


if(BUILD_TESTING)
enable_testing()
Expand Down
19 changes: 19 additions & 0 deletions src/example_config/stream-conv.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# // {
# // "serverConfig": {
# // "uds": "/var/run/stream_conv.sock",
# // "workdir": "/tmp/stream_conv"
# // },
# // "logConfig": {
# // "level": 0,
# // "mode": "stdout"
# // }
# // }
globalConfig:
servAddr: /var/run/stream_conv.sock
workDir: /tmp/stream_conv
logConfig:
level: 0
mode: stdout
rotateNum: 3
limitSizeMB: 10
path: /var/log/overlaybd/stream_convertor.log
1 change: 0 additions & 1 deletion src/overlaybd/gzindex/gzfile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,6 @@ ssize_t GzFile::pread(void *buf, size_t count, off_t offset) {
if (p == nullptr) {
LOG_ERRNO_RETURN(0, -1, "Failed to seek_index(,`)", offset);
}
//LOG_DEBUG("offset:`, index->de_pos:", offset, p->de_pos+0);

return extract(p, offset, (unsigned char*)buf, count);
}
Expand Down
18 changes: 11 additions & 7 deletions src/overlaybd/gzindex/test/test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ int download(const std::string &url, const std::string &out) {
// return 0;
auto base = std::string(basename(url.c_str()));
// download file
std::string cmd = "curl -s -o " + out + " " + url;
std::string cmd = "curl -sL -o " + out + " " + url;
LOG_INFO(VALUE(cmd.c_str()));
auto ret = system(cmd.c_str());
if (ret != 0) {
Expand Down Expand Up @@ -296,15 +296,19 @@ TEST_F(GzIndexTest, stream) {
auto lfs = photon::fs::new_localfs_adaptor(workdir.c_str());
LOG_INFO("start streamFile test");
std::vector<std::string> filelist = {
"https://dadi-shared.oss-cn-beijing.aliyuncs.com/cri-containerd-cni-1.5.2-linux-amd64.tar.gz",
"https://dadi-shared.oss-cn-beijing.aliyuncs.com/containerd-1.4.4-linux-amd64.tar.gz",
"https://dadi-shared.oss-cn-beijing.aliyuncs.com/go1.13.linux-amd64.tar.gz",
"https://dadi-shared.oss-cn-beijing.aliyuncs.com/go1.17.6.linux-amd64.tar.gz"
// "https://dadi-shared.oss-cn-beijing.aliyuncs.com/cri-containerd-cni-1.5.2-linux-amd64.tar.gz",
"https://github.com/containerd/containerd/releases/download/v1.5.17/cri-containerd-cni-1.5.17-linux-amd64.tar.gz",
"https://github.com/containerd/containerd/releases/download/v1.4.4/containerd-1.4.4-linux-amd64.tar.gz",
// "https://dadi-shared.oss-cn-beijing.aliyuncs.com/containerd-1.4.4-linux-amd64.tar.gz",
// "https://dadi-shared.oss-cn-beijing.aliyuncs.com/go1.13.linux-amd64.tar.gz",
"https://go.dev/dl/go1.17.6.linux-amd64.tar.gz",
// "https://dadi-shared.oss-cn-beijing.aliyuncs.com/go1.17.6.linux-amd64.tar.gz"
};
std::vector<std::string> tar_sha256sum = {
"sha256:05e8b01c1ddb6ba4f8c84e7dbc76529bdc09861f9ce17c213a49e8c334f184ed",
"sha256:02adc5074f59777d2ca74c8a0291659f69291865184c987d9c10e58f58b162c2",
// "sha256:05e8b01c1ddb6ba4f8c84e7dbc76529bdc09861f9ce17c213a49e8c334f184ed",
"sha256:0ccf983abf0b0fb64cc969079982bc34761ce22d7a3236a40d49d840d150e09a",
"sha256:1041ec4e2f40156e0731be175388be4c67aeceb44829f988df213e9fd5f26dc9",
// "sha256:1041ec4e2f40156e0731be175388be4c67aeceb44829f988df213e9fd5f26dc9",
"sha256:562688d70dcd1596556e7c671c1266f6e9c22b4f4fb8344efa8bed88fc2bac7b"
};
int i = 0;
Expand Down
12 changes: 11 additions & 1 deletion src/overlaybd/gzip/gz.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <photon/net/socket.h>
#include "../../tools/sha256file.h"
#include "../gzindex/gzfile_index.h"
#include "photon/common/alog.h"
class GzAdaptorFile : public photon::fs::VirtualReadOnlyFile {
public:
GzAdaptorFile() {
Expand Down Expand Up @@ -133,6 +134,9 @@ class GzStreamFile : public IGzFile {
len = sizeof(buf);
}
auto readn = this->read(buf, len);
if (readn <= 0){
LOG_ERRNO_RETURN(EIO, -1, "read buffer error");
}
offset -= readn;
}
return cur_offset;
Expand Down Expand Up @@ -166,9 +170,14 @@ class GzStreamFile : public IGzFile {
if (strm.avail_in < 0) {
LOG_ERRNO_RETURN(0, -1, "read buffer from uds failed");
}
size_t readn = strm.avail_in;
if (strm.avail_in == 0)
break;
if (!check_type) {
if (!((uint8_t)in[0] == 0x1f && (uint8_t)in[1] == 0x8b)){
LOG_ERRNO_RETURN(EIO, -1, "buffer is not gzip type");
}
check_type = true;
}
LOG_DEBUG("recv: `", strm.avail_in);
st_size += strm.avail_in;
strm.next_in = in;
Expand Down Expand Up @@ -243,6 +252,7 @@ class GzStreamFile : public IGzFile {
ssize_t st_size = 0;
IFile *m_file = nullptr;
photon::fs::IFileSystem *m_fs = nullptr;
bool check_type = false;

const static int CHUNK = 32768;
IStream *fstream;
Expand Down
16 changes: 9 additions & 7 deletions src/overlaybd/stream_convertor/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
file(GLOB SOURCE_SERV "*.cpp")

add_executable(stream_conv ${SOURCE_SERV})
target_include_directories(stream_conv PUBLIC
add_executable(overlaybd-streamConv ${SOURCE_SERV})
target_include_directories(overlaybd-streamConv PUBLIC
${PHOTON_INCLUDE_DIR}
${rapidjson_SOURCE_DIR}/include
)
target_link_libraries(stream_conv
photon_static
gzip_lib
gzindex_lib
tar_lib
target_link_libraries(overlaybd-streamConv
photon_static
gzip_lib
gzindex_lib
tar_lib
yaml-cpp
)

# if(BUILD_TESTING)
Expand Down
52 changes: 52 additions & 0 deletions src/overlaybd/stream_convertor/config.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
Copyright The Overlaybd Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#pragma once

#include <string>
#include <vector>
#include "config_utils.h"

namespace App {

struct LogConfigPara : public App::ConfigGroup {
APPCFG_CLASS
APPCFG_PARA(level, uint32_t, 1);
APPCFG_PARA(path, std::string, "/var/log/overlaybd/stream-convertor.log");
APPCFG_PARA(limitSizeMB, uint32_t, 10);
APPCFG_PARA(rotateNum, int, 3);
APPCFG_PARA(mode, std::string, "stdout");
};

// struct ServerConfigPara : public App::ConfigGroup {
// APPCFG_CLASS

// };

struct GlobalConfigPara : public App::ConfigGroup {
APPCFG_CLASS;
APPCFG_PARA(servAddr, std::string, "/var/run/stream_conv.sock");
APPCFG_PARA(workDir, std::string, "/tmp/stream_conv");
//APPCFG_PARA(ServerConfig, ServerConfigPara);
APPCFG_PARA(logConfig, LogConfigPara);
};

struct AppConfig : public App::ConfigGroup {
APPCFG_CLASS

APPCFG_PARA(globalConfig, GlobalConfigPara);
};

} // namespace ImageConfigNS
84 changes: 84 additions & 0 deletions src/overlaybd/stream_convertor/config_utils.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#pragma once
#include <photon/common/utility.h>
#include <yaml-cpp/yaml.h>
#include <iostream>
#include <string>
#include <type_traits>
#include <vector>

namespace App {

struct ConfigGroup : public YAML::Node {
ConfigGroup() = default;
ConfigGroup(const YAML::Node &node) : YAML::Node(node){};
ConfigGroup(const std::string &filename) {
parseYAML(std::move(filename));
}

void parseYAML(const std::string &fn) {
Clone(YAML::LoadFile(fn));
}

static size_t charfilter(char *dst, const char *src, char extract, size_t maxlen = 256UL) {
size_t i;
for (i = 0; (*src) && i < maxlen; i++, src++) {
while (*src == '-') {
src++;
}
if (!(*src))
break;
dst[i] = *(src);
}
dst[i] = 0;
return i;
}
};

#define APPCFG_PARA(ParaName, ParaType, ...) \
inline ParaType ParaName() const { \
return operator[](#ParaName).as<ParaType>(__VA_ARGS__); \
}

#define APPCFG_CLASS using ConfigGroup::ConfigGroup;

// merge two yaml nodes
// generate new node with full data
static YAML::Node mergeConfig(const YAML::Node &lhs, const YAML::Node &rhs) {
// if one of nodes is not type of map
// just return rhs
if (lhs.Type() != YAML::NodeType::Map || rhs.Type() != YAML::NodeType::Map)
return YAML::Clone(rhs);
// both are map, merge two maps
YAML::Node ret = YAML::Clone(lhs);
for (auto &node : rhs) {
auto key = node.first.as<std::string>();
if (ret[key].IsDefined()) {
// if key exists in lhs, merge recursivily
ret[key] = mergeConfig(lhs[key], node.second);
} else {
// just add as new key
ret[key] = Clone(node.second);
}
}
return ret;
}

} // namespace App

namespace YAML {

template <typename T>
struct convert {
template <ENABLE_IF_BASE_OF(App::ConfigGroup, T)>
static Node encode(const T &rhs) {
return rhs;
}

template <ENABLE_IF_BASE_OF(App::ConfigGroup, T)>
static bool decode(const Node &node, T &rhs) {
rhs = T(node);
return true;
}
};

} // namespace YAML

0 comments on commit 0cba95e

Please sign in to comment.