support sockmap/sockhash/cgroup_local_storage maps

This patch supports sockmap, sockhash and cgroup_local_storage
maps. Two C++ APIs, attach_func and detach_func are also
added to allow to attach program to cgroups. So this makes
using C++ APIs for cgroup based networking applications
easier to integrate with bpf programs.

The unit testing is rough as it needs some work to set up
cgroups and establish TCP connections to really test the
result of map operations. But still all supported map
operations in kernel and in C++ APIs are tested syntacically.

Signed-off-by: Yonghong Song <yhs@fb.com>

src/cc/api/BPF.cc

@@ -609,6 +609,29 @@ StatusTuple BPF::unload_func(const std::string& func_name) {
   return StatusTuple(0);
 }
 
+StatusTuple BPF::attach_func(int prog_fd, int attachable_fd,
+                             enum bpf_attach_type attach_type,
+                             uint64_t flags) {
+  int res = bpf_module_->bcc_func_attach(prog_fd, attachable_fd, attach_type, flags);
+  if (res != 0)
+    return StatusTuple(-1, "Can't attach for prog_fd %d, attachable_fd %d, "
+                           "attach_type %d, flags %ld: error %d",
+                       prog_fd, attachable_fd, attach_type, flags, res);
+
+  return StatusTuple(0);
+}
+
+StatusTuple BPF::detach_func(int prog_fd, int attachable_fd,
+                             enum bpf_attach_type attach_type) {
+  int res = bpf_module_->bcc_func_detach(prog_fd, attachable_fd, attach_type);
+  if (res != 0)
+    return StatusTuple(-1, "Can't detach for prog_fd %d, attachable_fd %d, "
+                           "attach_type %d: error %d",
+                       prog_fd, attachable_fd, attach_type, res);
+
+  return StatusTuple(0);
+}
+
 std::string BPF::get_syscall_fnname(const std::string& name) {
   if (syscall_prefix_ == nullptr) {
     KSyms ksym;
@@ -708,6 +731,20 @@ BPFMapInMapTable BPF::get_map_in_map_table(const std::string& name) {
   return BPFMapInMapTable({});
 }
 
+BPFSockmapTable BPF::get_sockmap_table(const std::string& name) {
+  TableStorage::iterator it;
+  if (bpf_module_->table_storage().Find(Path({bpf_module_->id(), name}), it))
+    return BPFSockmapTable(it->second);
+  return BPFSockmapTable({});
+}
+
+BPFSockhashTable BPF::get_sockhash_table(const std::string& name) {
+  TableStorage::iterator it;
+  if (bpf_module_->table_storage().Find(Path({bpf_module_->id(), name}), it))
+    return BPFSockhashTable(it->second);
+  return BPFSockhashTable({});
+}
+
 bool BPF::add_module(std::string module)
 {
   return bcc_buildsymcache_add_module(get_bsymcache(), module.c_str()) != 0 ?

src/cc/api/BPF.h

@@ -154,6 +154,22 @@ class BPF {
     return BPFSkStorageTable<ValueType>({});
   }
 
+  template <class ValueType>
+  BPFCgStorageTable<ValueType> get_cg_storage_table(const std::string& name) {
+    TableStorage::iterator it;
+    if (bpf_module_->table_storage().Find(Path({bpf_module_->id(), name}), it))
+      return BPFCgStorageTable<ValueType>(it->second);
+    return BPFCgStorageTable<ValueType>({});
+  }
+
+  template <class ValueType>
+  BPFPercpuCgStorageTable<ValueType> get_percpu_cg_storage_table(const std::string& name) {
+    TableStorage::iterator it;
+    if (bpf_module_->table_storage().Find(Path({bpf_module_->id(), name}), it))
+      return BPFPercpuCgStorageTable<ValueType>(it->second);
+    return BPFPercpuCgStorageTable<ValueType>({});
+  }
+
   void* get_bsymcache(void) {
     if (bsymcache_ == NULL) {
       bsymcache_ = bcc_buildsymcache_new();
@@ -169,6 +185,10 @@ class BPF {
 
   BPFXskmapTable get_xskmap_table(const std::string& name);
 
+  BPFSockmapTable get_sockmap_table(const std::string& name);
+
+  BPFSockhashTable get_sockhash_table(const std::string& name);
+
   BPFStackTable get_stack_table(const std::string& name,
                                 bool use_debug_file = true,
                                 bool check_debug_file_crc = true);
@@ -210,6 +230,12 @@ class BPF {
                         int& fd);
   StatusTuple unload_func(const std::string& func_name);
 
+  StatusTuple attach_func(int prog_fd, int attachable_fd,
+                          enum bpf_attach_type attach_type,
+                          uint64_t flags);
+  StatusTuple detach_func(int prog_fd, int attachable_fd,
+                          enum bpf_attach_type attach_type);
+
   int free_bcc_memory();
 
  private:

src/cc/api/BPFTable.cc

@@ -702,4 +702,44 @@ StatusTuple BPFMapInMapTable::remove_value(const int& index) {
     return StatusTuple(0);
 }
 
+BPFSockmapTable::BPFSockmapTable(const TableDesc& desc)
+    : BPFTableBase<int, int>(desc) {
+    if(desc.type != BPF_MAP_TYPE_SOCKMAP)
+      throw std::invalid_argument("Table '" + desc.name +
+                                  "' is not a sockmap table");
+}
+
+StatusTuple BPFSockmapTable::update_value(const int& index,
+                                         const int& value) {
+    if (!this->update(const_cast<int*>(&index), const_cast<int*>(&value)))
+      return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
+    return StatusTuple(0);
+}
+
+StatusTuple BPFSockmapTable::remove_value(const int& index) {
+    if (!this->remove(const_cast<int*>(&index)))
+      return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
+    return StatusTuple(0);
+}
+
+BPFSockhashTable::BPFSockhashTable(const TableDesc& desc)
+    : BPFTableBase<int, int>(desc) {
+    if(desc.type != BPF_MAP_TYPE_SOCKHASH)
+      throw std::invalid_argument("Table '" + desc.name +
+                                  "' is not a sockhash table");
+}
+
+StatusTuple BPFSockhashTable::update_value(const int& key,
+                                         const int& value) {
+    if (!this->update(const_cast<int*>(&key), const_cast<int*>(&value)))
+      return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
+    return StatusTuple(0);
+}
+
+StatusTuple BPFSockhashTable::remove_value(const int& key) {
+    if (!this->remove(const_cast<int*>(&key)))
+      return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
+    return StatusTuple(0);
+}
+
 }  // namespace ebpf

src/cc/api/BPFTable.h

@@ -32,6 +32,7 @@
 #include "libbpf.h"
 #include "perf_reader.h"
 #include "table_desc.h"
+#include "linux/bpf.h"
 
 namespace ebpf {
 
@@ -418,6 +419,22 @@ class BPFMapInMapTable : public BPFTableBase<int, int> {
   StatusTuple remove_value(const int& index);
 };
 
+class BPFSockmapTable : public BPFTableBase<int, int> {
+public:
+  BPFSockmapTable(const TableDesc& desc);
+
+  StatusTuple update_value(const int& index, const int& value);
+  StatusTuple remove_value(const int& index);
+};
+
+class BPFSockhashTable : public BPFTableBase<int, int> {
+public:
+  BPFSockhashTable(const TableDesc& desc);
+
+  StatusTuple update_value(const int& key, const int& value);
+  StatusTuple remove_value(const int& key);
+};
+
 template <class ValueType>
 class BPFSkStorageTable : public BPFTableBase<int, ValueType> {
  public:
@@ -447,4 +464,63 @@ class BPFSkStorageTable : public BPFTableBase<int, ValueType> {
   }
 };
 
+template <class ValueType>
+class BPFCgStorageTable : public BPFTableBase<int, ValueType> {
+ public:
+  BPFCgStorageTable(const TableDesc& desc) : BPFTableBase<int, ValueType>(desc) {
+    if (desc.type != BPF_MAP_TYPE_CGROUP_STORAGE)
+      throw std::invalid_argument("Table '" + desc.name +
+                                  "' is not a cgroup_storage table");
+  }
+
+  virtual StatusTuple get_value(struct bpf_cgroup_storage_key& key,
+                                ValueType& value) {
+    if (!this->lookup(const_cast<struct bpf_cgroup_storage_key*>(&key),
+                      get_value_addr(value)))
+      return StatusTuple(-1, "Error getting value: %s", std::strerror(errno));
+    return StatusTuple(0);
+  }
+
+  virtual StatusTuple update_value(struct bpf_cgroup_storage_key& key, const ValueType& value) {
+    if (!this->update(const_cast<struct bpf_cgroup_storage_key*>(&key),
+                      get_value_addr(const_cast<ValueType&>(value))))
+      return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
+    return StatusTuple(0);
+  }
+};
+
+template <class ValueType>
+class BPFPercpuCgStorageTable : public BPFTableBase<int, std::vector<ValueType>> {
+ public:
+  BPFPercpuCgStorageTable(const TableDesc& desc)
+      : BPFTableBase<int, std::vector<ValueType>>(desc) {
+    if (desc.type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
+      throw std::invalid_argument("Table '" + desc.name +
+                                  "' is not a percpu_cgroup_storage table");
+    if (sizeof(ValueType) % 8)
+      throw std::invalid_argument("leaf must be aligned to 8 bytes");
+    ncpus = BPFTable::get_possible_cpu_count();
+  }
+
+  virtual StatusTuple get_value(struct bpf_cgroup_storage_key& key,
+                                std::vector<ValueType>& value) {
+    value.resize(ncpus);
+    if (!this->lookup(const_cast<struct bpf_cgroup_storage_key*>(&key),
+                      get_value_addr(value)))
+      return StatusTuple(-1, "Error getting value: %s", std::strerror(errno));
+    return StatusTuple(0);
+  }
+
+  virtual StatusTuple update_value(struct bpf_cgroup_storage_key& key,
+                                   std::vector<ValueType>& value) {
+    value.resize(ncpus);
+    if (!this->update(const_cast<struct bpf_cgroup_storage_key*>(&key),
+                      get_value_addr(const_cast<std::vector<ValueType>&>(value))))
+      return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
+    return StatusTuple(0);
+  }
+ private:
+  unsigned int ncpus;
+};
+
 }  // namespace ebpf

src/cc/bpf_module.cc

@@ -940,4 +940,16 @@ int BPFModule::bcc_func_load(int prog_type, const char *name,
   return ret;
 }
 
+int BPFModule::bcc_func_attach(int prog_fd, int attachable_fd,
+                               int attach_type, unsigned int flags) {
+  return bpf_prog_attach(prog_fd, attachable_fd,
+                         (enum bpf_attach_type)attach_type, flags);
+}
+
+int BPFModule::bcc_func_detach(int prog_fd, int attachable_fd,
+                               int attach_type) {
+  return bpf_prog_detach2(prog_fd, attachable_fd,
+                          (enum bpf_attach_type)attach_type);
+}
+
 } // namespace ebpf

src/cc/bpf_module.h

@@ -145,6 +145,9 @@ class BPFModule {
                     const char *license, unsigned kern_version,
                     int log_level, char *log_buf, unsigned log_buf_size,
                     const char *dev_name = nullptr);
+  int bcc_func_attach(int prog_fd, int attachable_fd,
+                      int attach_type, unsigned int flags);
+  int bcc_func_detach(int prog_fd, int attachable_fd, int attach_type);
   size_t perf_event_fields(const char *) const;
   const char * perf_event_field(const char *, size_t i) const;
 

src/cc/export/helpers.h

@@ -306,6 +306,44 @@ __attribute__((section("maps/sk_storage"))) \
 struct _name##_table_t _name = { .flags = BPF_F_NO_PREALLOC }; \
 BPF_ANNOTATE_KV_PAIR(_name, int, _leaf_type)
 
+#define BPF_SOCKMAP_COMMON(_name, _max_entries, _kind, _helper_name) \
+struct _name##_table_t { \
+  u32 key; \
+  int leaf; \
+  int (*update) (u32 *, int *); \
+  int (*delete) (int *); \
+  /* ret = map.sock_map_update(ctx, key, flag) */ \
+  int (* _helper_name) (void *, void *, u64); \
+  u32 max_entries; \
+}; \
+__attribute__((section("maps/" _kind))) \
+struct _name##_table_t _name = { .max_entries = (_max_entries) }; \
+BPF_ANNOTATE_KV_PAIR(_name, u32, int)
+
+#define BPF_SOCKMAP(_name, _max_entries) \
+  BPF_SOCKMAP_COMMON(_name, _max_entries, "sockmap", sock_map_update)
+
+#define BPF_SOCKHASH(_name, _max_entries) \
+  BPF_SOCKMAP_COMMON(_name, _max_entries, "sockhash", sock_hash_update)
+
+#define BPF_CGROUP_STORAGE_COMMON(_name, _leaf_type, _kind) \
+struct _name##_table_t { \
+  struct bpf_cgroup_storage_key key; \
+  _leaf_type leaf; \
+  _leaf_type * (*lookup) (struct bpf_cgroup_storage_key *); \
+  int (*update) (struct bpf_cgroup_storage_key *, _leaf_type *); \
+  int (*get_local_storage) (u64); \
+}; \
+__attribute__((section("maps/" _kind))) \
+struct _name##_table_t _name = { 0 }; \
+BPF_ANNOTATE_KV_PAIR(_name, struct bpf_cgroup_storage_key, _leaf_type)
+
+#define BPF_CGROUP_STORAGE(_name, _leaf_type) \
+  BPF_CGROUP_STORAGE_COMMON(_name, _leaf_type, "cgroup_storage")
+
+#define BPF_PERCPU_CGROUP_STORAGE(_name, _leaf_type) \
+  BPF_CGROUP_STORAGE_COMMON(_name, _leaf_type, "percpu_cgroup_storage")
+
 // packet parsing state machine helpers
 #define cursor_advance(_cursor, _len) \
   ({ void *_tmp = _cursor; _cursor += _len; _tmp; })

src/cc/frontends/clang/b_frontend_action.cc

@@ -882,6 +882,12 @@ bool BTypeVisitor::VisitCallExpr(CallExpr *Call) {
               error(GET_BEGINLOC(Call), "get_stackid only available on stacktrace maps");
               return false;
             }
+        } else if (memb_name == "sock_map_update" || memb_name == "sock_hash_update") {
+          string ctx = rewriter_.getRewrittenText(expansionRange(Call->getArg(0)->getSourceRange()));
+          string keyp = rewriter_.getRewrittenText(expansionRange(Call->getArg(1)->getSourceRange()));
+          string flag = rewriter_.getRewrittenText(expansionRange(Call->getArg(2)->getSourceRange()));
+          txt = "bpf_" + string(memb_name) + "(" + ctx + ", " +
+            "bpf_pseudo_fd(1, " + fd + "), " + keyp + ", " + flag + ");";
         } else {
           if (memb_name == "lookup") {
             prefix = "bpf_map_lookup_elem";
@@ -919,6 +925,9 @@ bool BTypeVisitor::VisitCallExpr(CallExpr *Call) {
           } else if (memb_name == "sk_storage_delete") {
             prefix = "bpf_sk_storage_delete";
             suffix = ")";
+          } else if (memb_name == "get_local_storage") {
+            prefix = "bpf_get_local_storage";
+            suffix = ")";
           } else {
             error(GET_BEGINLOC(Call), "invalid bpf_table operation %0") << memb_name;
             return false;
@@ -1284,6 +1293,14 @@ bool BTypeVisitor::VisitVarDecl(VarDecl *Decl) {
       map_type = BPF_MAP_TYPE_ARRAY_OF_MAPS;
     } else if (section_attr == "maps/sk_storage") {
       map_type = BPF_MAP_TYPE_SK_STORAGE;
+    } else if (section_attr == "maps/sockmap") {
+      map_type = BPF_MAP_TYPE_SOCKMAP;
+    } else if (section_attr == "maps/sockhash") {
+      map_type = BPF_MAP_TYPE_SOCKHASH;
+    } else if (section_attr == "maps/cgroup_storage") {
+      map_type = BPF_MAP_TYPE_CGROUP_STORAGE;
+    } else if (section_attr == "maps/percpu_cgroup_storage") {
+      map_type = BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE;
     } else if (section_attr == "maps/extern") {
       if (!fe_.table_storage().Find(maps_ns_path, table_it)) {
         if (!fe_.table_storage().Find(global_path, table_it)) {

tests/cc/CMakeLists.txt

@@ -20,13 +20,15 @@ set(TEST_LIBBCC_SOURCES
 	test_c_api.cc
 	test_array_table.cc
 	test_bpf_table.cc
+	test_cg_storage.cc
 	test_hash_table.cc
 	test_map_in_map.cc
 	test_perf_event.cc
 	test_pinned_table.cc
 	test_prog_table.cc
 	test_shared_table.cc
 	test_sk_storage.cc
+	test_sock_table.cc
 	test_usdt_args.cc
 	test_usdt_probes.cc
 	utils.cc)