ebpf observer network feature (#196)

* ebpf observer network feature

* ebpf observer network feature

* ebpf observer network feature

* polish codes

* polish codes

* polish codes

* polish codes

* polish codes

* polish codes

* polish codes

* polish codes

* polish codes

Co-authored-by: shaoxuan <liuajiapeng.ljp@alibaba-inc.com>

.github/workflows/e2e.yaml

@@ -45,6 +45,9 @@ jobs:
           curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
           sudo chmod +x /usr/local/bin/docker-compose
 
+      - name: Kernel version
+        run: uname -r
+
       - name: E2E Test
         run: make e2e
 
@@ -58,4 +61,4 @@ jobs:
     needs: [ CI ]
     steps:
       - name: Build Result
-        run: echo "Just to make the GitHub merge button green"
+        run: echo "Just to make the GitHub merge button green"

Makefile

@@ -48,7 +48,7 @@ DIST_DIR = ilogtail-$(VERSION)
 .PHONY: tools
 tools:
 	$(GO_LINT) version || curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_PATH)/bin v1.39.0
-	$(GO_ADDLICENSE) version || GO111MODULE=off $(GO_GET) -u github.com/google/addlicense
+	$(GO_ADDLICENSE) version || go install github.com/google/addlicense@latest
 
 .PHONY: clean
 clean:

core/CMakeLists.txt

@@ -87,6 +87,7 @@ set(SUB_DIRECTORIES_LIST
         )
 if (UNIX)
     set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} streamlog)
+    set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} observer)
 endif ()
 
 # Collect source files for UT.
@@ -126,6 +127,7 @@ target_link_libraries(${LOGTAIL_TARGET}
         )
 if (UNIX)
     target_link_libraries(${LOGTAIL_TARGET} streamlog)
+    target_link_libraries(${LOGTAIL_TARGET} observer)
     target_link_libraries(${LOGTAIL_TARGET} pthread dl uuid)
     if (ENABLE_STATIC_LINK_CRT)
         target_link_libraries(${LOGTAIL_TARGET} -static-libstdc++ -static-libgcc)

core/common/CMakeLists.txt

@@ -28,7 +28,16 @@ elseif (UNIX)
     list(REMOVE_ITEM LIB_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/WinUuid.cpp)
     file(GLOB DMI_UUID_SOURCE_FILES dmi_uuid/*.c dmi_uuid/*.h)
     list(APPEND LIB_SOURCE_FILES ${DMI_UUID_SOURCE_FILES})
+    file(GLOB PICOHTTPPARSER_SOURCE_FILES protocol/picohttpparser/*.c protocol/picohttpparser/*.h)
+    list(APPEND LIB_SOURCE_FILES ${PICOHTTPPARSER_SOURCE_FILES})
 endif ()
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
+    file(GLOB XX_HASH_SOURCE_FILES xxhash/*.c xxhash/*.h)
+else ()
+    file(GLOB XX_HASH_SOURCE_FILES xxhash/xxhash.c xxhash/xxhash.h)
+endif ()
+list(APPEND LIB_SOURCE_FILES ${XX_HASH_SOURCE_FILES})
+
 append_source_files(LIB_SOURCE_FILES)
 add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES})
 target_link_libraries(${PROJECT_NAME} logger)

core/common/DynamicLibHelper.cpp

@@ -0,0 +1,94 @@
+// Copyright 2022 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "DynamicLibHelper.h"
+
+#if defined(__linux__)
+#include <dlfcn.h>
+#elif defined(_MSC_VER)
+#include <Windows.h>
+#endif
+
+namespace logtail {
+
+
+std::string DynamicLibLoader::GetError() {
+#if defined(__linux__)
+    auto dlErr = dlerror();
+    return (dlErr != NULL) ? dlErr : "";
+#elif defined(_MSC_VER)
+    return std::to_string(GetLastError());
+#endif
+}
+
+void DynamicLibLoader::CloseLib(void* libPtr) {
+    if (nullptr == libPtr)
+        return;
+#if defined(__linux__)
+    dlclose(libPtr);
+#elif defined(_MSC_VER)
+    FreeLibrary((HINSTANCE)libPtr);
+#endif
+}
+
+// Release releases the ownership of @mLibPtr to caller.
+void* DynamicLibLoader::Release() {
+    auto ptr = mLibPtr;
+    mLibPtr = nullptr;
+    return ptr;
+}
+
+DynamicLibLoader::~DynamicLibLoader() {
+    CloseLib(mLibPtr);
+}
+
+// LoadDynLib loads dynamic library named @libName from current working dir.
+// For linux, the so name is 'lib+@libName.so'.
+// For Windows, the dll name is '@libName.dll'.
+// @return a non-NULL ptr to indicate lib handle, otherwise nullptr is returned.
+bool DynamicLibLoader::LoadDynLib(const std::string& libName, std::string& error, const std::string dlPrefix) {
+    error.clear();
+
+#if defined(__linux__)
+    mLibPtr = dlopen((dlPrefix + "lib" + libName + ".so").c_str(), RTLD_LAZY);
+#elif defined(_MSC_VER)
+    mLibPtr = LoadLibrary((dlPrefix + libName + ".dll").c_str());
+#endif
+    if (mLibPtr != NULL)
+        return true;
+    error = GetError();
+    return false;
+}
+
+// LoadMethod loads method named @methodName from opened lib.
+// If error is found, @error param will be set, otherwise empty.
+void* DynamicLibLoader::LoadMethod(const std::string& methodName, std::string& error) {
+    error.clear();
+
+#if defined(__linux__)
+    dlerror(); // Clear last error.
+    auto methodPtr = dlsym(mLibPtr, methodName.c_str());
+    error = GetError();
+    return methodPtr;
+#elif defined(_MSC_VER)
+    auto methodPtr = GetProcAddress((HINSTANCE)mLibPtr, methodName.c_str());
+    if (methodPtr != NULL)
+        return methodPtr;
+    error = std::to_string(GetLastError());
+    return NULL;
+#endif
+}
+
+
+} // namespace logtail

core/common/DynamicLibHelper.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2022 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <cstdlib>
+
+namespace logtail {
+
+class DynamicLibLoader {
+    void* mLibPtr = nullptr;
+
+    std::string GetError();
+
+public:
+    static void CloseLib(void* libPtr);
+
+    // Release releases the ownership of @mLibPtr to caller.
+    void* Release();
+
+    ~DynamicLibLoader();
+
+    // LoadDynLib loads dynamic library named @libName from current working dir.
+    // For linux, the so name is 'lib+@libName.so'.
+    // For Windows, the dll name is '@libName.dll'.
+    // @return a non-NULL ptr to indicate lib handle, otherwise nullptr is returned.
+    bool LoadDynLib(const std::string& libName, std::string& error, const std::string dlPrefix = "");
+
+    // LoadMethod loads method named @methodName from opened lib.
+    // If error is found, @error param will be set, otherwise empty.
+    void* LoadMethod(const std::string& methodName, std::string& error);
+};
+
+
+} // namespace logtail

core/common/FileSystemUtil.cpp

@@ -109,16 +109,16 @@ void TrimLastSeperator(std::string& path) {
     }
 }
 
-bool ReadFileContent(const std::string& fileName, std::string& content) {
+bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t maxFileSize) {
     FILE* pFile = fopen(fileName.c_str(), "r");
     if (pFile == NULL) {
         APSARA_LOG_DEBUG(sLogger, ("open file fail", fileName)("errno", strerror(errno)));
         return false;
     }
 
     content.clear();
-    char* buffer = new char[8192];
-    uint32_t readBytes = fread(buffer, 1, 8192, pFile);
+    char* buffer = new char[maxFileSize];
+    uint32_t readBytes = fread(buffer, 1, maxFileSize, pFile);
     if (readBytes > 0) {
         content.append(buffer, readBytes);
         delete[] buffer;

core/common/FileSystemUtil.h

@@ -1,18 +1,18 @@
 /*
- * Copyright 2022 iLogtail Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+* Copyright 2022 iLogtail Authors
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
 
 #pragma once
 #include <string>
@@ -79,7 +79,7 @@ int64_t FTell(FILE* stream);
 void TrimLastSeperator(std::string& path);
 
 // ReadFileContent reads all content of @fileName to @content.
-bool ReadFileContent(const std::string& fileName, std::string& content);
+bool ReadFileContent(const std::string& fileName, std::string& content, uint32_t maxFileSize = 8192);
 
 // OverwriteFile overwrides @fileName with @content.
 bool OverwriteFile(const std::string& fileName, const std::string& content);

core/common/MachineInfoUtil.cpp

@@ -29,6 +29,7 @@
 #endif
 #include "logger/Logger.h"
 #include "StringTools.h"
+#include "FileSystemUtil.h"
 
 
 #if defined(_MSC_VER)
@@ -298,4 +299,83 @@ std::string GetAnyAvailableIP() {
 #endif
 }
 
+bool GetKernelInfo(std::string& kernelRelease, int64_t& kernelVersion) {
+#if defined(__linux__)
+    struct utsname buf;
+    if (-1 == uname(&buf)) {
+        LOG_ERROR(sLogger, ("uname failed, errno", errno));
+        return false;
+    }
+    kernelRelease.assign(buf.release);
+    std::vector<int64_t> versions;
+    char* p = buf.release;
+    char* maxP = buf.release + sizeof(buf.release);
+    while (*p && p < maxP) {
+        if (isdigit(*p)) {
+            versions.push_back(strtol(p, &p, 10));
+        } else {
+            p++;
+        }
+    }
+    kernelVersion = 0;
+    for (size_t i = 0; i < versions.size() && i < (size_t)4; ++i) {
+        kernelVersion = kernelVersion * 1000 + versions[i];
+    }
+    return true;
+#else
+    return false;
+#endif
+}
+
+uint32_t GetHostIpValueByInterface(const std::string& intf) {
+#if defined(__linux__)
+    int sock;
+    struct sockaddr_in sin;
+    struct ifreq ifr;
+    sock = socket(AF_INET, SOCK_DGRAM, 0);
+    if (sock == -1) {
+        return 0;
+    }
+    // use eth0 as the default ETH name
+    strncpy(ifr.ifr_name, intf.size() > 0 ? intf.c_str() : "eth0", IFNAMSIZ);
+    ifr.ifr_name[IFNAMSIZ - 1] = 0;
+    if (ioctl(sock, SIOCGIFADDR, &ifr) < 0) {
+        close(sock);
+        return 0;
+    }
+    memcpy(&sin, &ifr.ifr_addr, sizeof(sin));
+    close(sock);
+    return sin.sin_addr.s_addr;
+#elif defined(_MSC_VER)
+    // TODO: For Windows, interface should be replace to adaptor name.
+    // Delay implementation, assume that GetIpByHostName will succeed.
+    return 0;
+#endif
+}
+
+void GetAllPids(std::unordered_set<int32_t>& pids) {
+#if defined(__linux__)
+    pids.clear();
+    fsutil::Dir procDir("/proc/");
+    if (!procDir.Open()) {
+        return;
+    }
+    fsutil::Entry entry;
+    while (true) {
+        entry = procDir.ReadNext();
+        if (!entry) {
+            return;
+        }
+        if (!entry.IsDir()) {
+            continue;
+        }
+        std::string name = entry.Name();
+        int32_t pid = strtol(name.data(), NULL, 10);
+        if (pid <= 0) {
+            continue;
+        }
+        pids.insert(pid);
+    }
+#endif
+}
 } // namespace logtail