diff --git a/patches/bess/0012-Dpdk-concurrent-hash-support-pipeline-improvement.patch b/patches/bess/0012-Dpdk-concurrent-hash-support-pipeline-improvement.patch index fc357b3fa..b91d4dd1f 100644 --- a/patches/bess/0012-Dpdk-concurrent-hash-support-pipeline-improvement.patch +++ b/patches/bess/0012-Dpdk-concurrent-hash-support-pipeline-improvement.patch @@ -1,20 +1,59 @@ -From 460ac23d8143f8475a7819aa8ccbbf930f2940c8 Mon Sep 17 00:00:00 2001 -From: Amar Srivastava -Date: Wed, 23 Dec 2020 02:33:49 -0800 -Subject: [PATCH] dpdk-hash linking and pipeline improvements +From 0bd568524fc36ee714d5a5e2751834b33b94746a Mon Sep 17 00:00:00 2001 +From: Amar Sri +Date: Mon, 15 Mar 2021 08:26:42 -0700 +Subject: [PATCH] Exact match bulk support --- - core/modules/wildcard_match.cc | 202 ++++++++++++++++++++++++++------- - core/modules/wildcard_match.h | 37 ++++-- - core/utils/cuckoo_map.h | 146 +++++++++++++++++++++++- - core/utils/exact_match_table.h | 97 +++++++++++----- - 4 files changed, 403 insertions(+), 79 deletions(-) + core/modules/exact_match.cc | 19 ++-- + core/modules/wildcard_match.cc | 197 ++++++++++++++++++++++++++------- + core/modules/wildcard_match.h | 26 ++++- + core/utils/cuckoo_map.h | 140 ++++++++++++++++++++++- + core/utils/exact_match_table.h | 85 +++++++++----- + 5 files changed, 376 insertions(+), 91 deletions(-) +diff --git a/core/modules/exact_match.cc b/core/modules/exact_match.cc +index 1ca0fd4e..e8229623 100644 +--- a/core/modules/exact_match.cc ++++ b/core/modules/exact_match.cc +@@ -312,7 +312,6 @@ void ExactMatch::setValues(bess::Packet *pkt, ExactMatchKey &action) { + void ExactMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { + gate_idx_t default_gate; + ExactMatchKey keys[bess::PacketBatch::kMaxBurst] __ymm_aligned; +- + default_gate = ACCESS_ONCE(default_gate_); + + const auto buffer_fn = [&](bess::Packet *pkt, const ExactMatchField &f) { +@@ -326,17 +325,15 @@ void ExactMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { + + int cnt = batch->cnt(); + Value default_value(default_gate); +- +- for (int i = 0; i < cnt; i++) { +- bess::Packet *pkt = batch->pkts()[i]; +- ValueTuple res; +- res = table_.Find(keys[i], default_value); +- if (res.gate != default_gate) { +- /* setting respecive values */ +- setValues(pkt, res.action); ++ ValueTuple *res[cnt]; ++ uint64_t hit_mask = table_.Find(&keys[0], &res[0], cnt); ++ for (int j = 0; j < cnt; j++) { ++ if ((hit_mask & (1ULL << j)) == 0) ++ EmitPacket(ctx, batch->pkts()[j], default_gate); ++ else { ++ setValues(batch->pkts()[j], res[j]->action); ++ EmitPacket(ctx, batch->pkts()[j], res[j]->gate); + } +- gate_idx_t g = res.gate; +- EmitPacket(ctx, pkt, g); + } + } + diff --git a/core/modules/wildcard_match.cc b/core/modules/wildcard_match.cc -index ec639a7b..ade76975 100644 +index ec639a7b..6a8d2350 100644 --- a/core/modules/wildcard_match.cc +++ b/core/modules/wildcard_match.cc -@@ -40,15 +40,32 @@ using bess::metadata::Attribute; +@@ -40,13 +40,29 @@ using bess::metadata::Attribute; enum { FieldType = 0, ValueType }; // dst = src & mask. len must be a multiple of sizeof(uint64_t) @@ -27,52 +66,45 @@ index ec639a7b..ade76975 100644 for (size_t i = 0; i < len / 8; i++) { - dst->u64_arr[i] = src.u64_arr[i] & mask.u64_arr[i]; + dst.u64_arr[i] = src.u64_arr[i] & mask.u64_arr[i]; - } - } -+static inline void mask_bulk(const wm_hkey_t *src,void *dst,void **dsptr, -+ const wm_hkey_t &mask, int keys,size_t len) { ++ } ++} ++static inline void mask_bulk(const wm_hkey_t *src, void *dst, void **dsptr, ++ const wm_hkey_t &mask, int keys, size_t len) { + promise(len >= sizeof(uint64_t)); + promise(len <= sizeof(wm_hkey_t)); + size_t i = 0; + wm_hkey_t *dst1 = (wm_hkey_t *)dst; + wm_hkey_t **dstptr = (wm_hkey_t **)dsptr; + -+ for(int j=0;ju64_arr[total_key_size_/sizeof(keys->u64_arr[0])],0,i); -+ } -+ ++void WildcardMatch::Initkeys(wm_hkey_t *keys) { ++ int i = sizeof(wm_hkey_t) - total_key_size_; ++ if (i) ++ memset(&keys->u64_arr[total_key_size_ / sizeof(keys->u64_arr[0])], 0, i); ++} + const Commands WildcardMatch::cmds = { {"get_initial_arg", "EmptyArg", MODULE_CMD_FUNC(&WildcardMatch::GetInitialArg), Command::THREAD_SAFE}, -@@ -162,18 +188,15 @@ inline gate_idx_t WildcardMatch::LookupEntry(const wm_hkey_t &key, +@@ -162,18 +184,15 @@ inline gate_idx_t WildcardMatch::LookupEntry(const wm_hkey_t &key, bess::Packet *pkt) { struct WmData result = { .priority = INT_MIN, .ogate = def_gate, .keyv = {{0}}}; - for (auto &tuple : tuples_) { -- const auto &ht = tuple.ht; -+ const auto &ht = tuple.ht; + const auto &ht = tuple.ht; wm_hkey_t key_masked; - - mask(&key_masked, key, tuple.mask, total_key_size_); @@ -83,51 +115,55 @@ index ec639a7b..ade76975 100644 - if (entry && entry->second.priority >= result.priority) { - result = entry->second; + mask(key_masked, key, tuple.mask, total_key_size_); -+ WmData* entry =nullptr; -+ Initkeys(&key_masked); -+ ht->find_dpdk(&key_masked,((void**)&entry) ); ++ WmData *entry = nullptr; ++ Initkeys(&key_masked); ++ ht->find_dpdk(&key_masked, ((void **)&entry)); + if (entry && entry->priority >= result.priority) { + result = *entry; } } -@@ -232,11 +255,101 @@ inline gate_idx_t WildcardMatch::LookupEntry(const wm_hkey_t &key, +@@ -232,11 +251,105 @@ inline gate_idx_t WildcardMatch::LookupEntry(const wm_hkey_t &key, return result.ogate; } -+inline bool WildcardMatch::LookupBulkEntry(wm_hkey_t *key, -+ gate_idx_t def_gate, -+ int packeti, gate_idx_t (*Outgate)[32] ,int cnt,bess::PacketBatch *batch) { ++inline bool WildcardMatch::LookupBulkEntry(wm_hkey_t *key, gate_idx_t def_gate, ++ int packeti, ++ gate_idx_t (*Outgate)[32], int cnt, ++ bess::PacketBatch *batch) { + bess::Packet *pkt = nullptr; + struct WmData result[cnt]; -+ uint64_t hitmask=0; -+ wm_hkey_t *key_masked = new wm_hkey_t[cnt]; -+ WmData** entry = new WmData*[cnt]; -+ wm_hkey_t **key_ptr = new wm_hkey_t*[cnt]; -+ int count =0; -+ for(int i=0;ilookup_bulk_data((const void**)key_ptr,cnt, &hitmask, (void**)&entry[0]); -+ if(num ==0) continue; ++ mask_bulk(key, key_masked, (void **)key_ptr, tuple.mask, cnt, ++ total_key_size_); ++ int num = ht->lookup_bulk_data((const void **)key_ptr, cnt, &hitmask, ++ (void **)&entry[0]); ++ if (num == 0) ++ continue; + -+ for(int init=0;initpriority >= result[init].priority)) { -+ result[init] = *entry[init]; } ++ for (int init = 0; init < cnt; init++) { ++ if ((hitmask & (1ULL << init)) && ++ (entry[init]->priority >= result[init].priority)) { ++ result[init] = *entry[init]; + } ++ } + count++; + } -+ for(int init=0;initpkts()[packeti+init]; -+ size_t num_values_ = values_.size(); -+ for (size_t i = 0; i < num_values_; i++) { ++ for (int init = 0; init < cnt; init++) { ++ /* if lookup was successful, then set values (if possible) */ ++ if (result[init].ogate != default_gate_) { ++ pkt = batch->pkts()[packeti + init]; ++ size_t num_values_ = values_.size(); ++ for (size_t i = 0; i < num_values_; i++) { + int value_size = values_[i].size; + int value_pos = values_[i].pos; + int value_off = values_[i].offset; @@ -137,7 +173,8 @@ index ec639a7b..ade76975 100644 + DLOG(INFO) << "off: " << (int)value_off << ", sz: " << value_size + << std::endl; + if (value_attr_id < 0) { /* if it is offset-based */ -+ memcpy(data, reinterpret_cast(&result[init].keyv) + value_pos, ++ memcpy(data, ++ reinterpret_cast(&result[init].keyv) + value_pos, + value_size); + } else { /* if it is attribute-based */ + typedef struct { @@ -145,47 +182,46 @@ index ec639a7b..ade76975 100644 + } value_t; + uint8_t *buf = (uint8_t *)&result[init].keyv + value_pos; + -+ DLOG(INFO) << "Setting value " << std::hex -+ << *(reinterpret_cast(buf)) -+ << " for attr_id: " << value_attr_id -+ << " of size: " << value_size -+ << " at value_pos: " << value_pos << std::endl; ++ DLOG(INFO) << "Setting value " << std::hex ++ << *(reinterpret_cast(buf)) ++ << " for attr_id: " << value_attr_id ++ << " of size: " << value_size ++ << " at value_pos: " << value_pos << std::endl; + -+ switch (value_size) { -+ case 1: -+ set_attr(this, value_attr_id, pkt, *((uint8_t *)buf)); -+ break; -+ case 2: -+ set_attr(this, value_attr_id, pkt, -+ *((uint16_t *)((uint8_t *)buf))); -+ break; -+ case 4: -+ set_attr(this, value_attr_id, pkt, -+ *((uint32_t *)((uint8_t *)buf))); -+ break; -+ case 8: -+ set_attr(this, value_attr_id, pkt, -+ *((uint64_t *)((uint8_t *)buf))); -+ break; -+ default: { -+ void *mt_ptr = -+ _ptr_attr_with_offset(attr_offset(value_attr_id), pkt); -+ bess::utils::CopySmall(mt_ptr, buf, value_size); -+ } break; ++ switch (value_size) { ++ case 1: ++ set_attr(this, value_attr_id, pkt, *((uint8_t *)buf)); ++ break; ++ case 2: ++ set_attr(this, value_attr_id, pkt, ++ *((uint16_t *)((uint8_t *)buf))); ++ break; ++ case 4: ++ set_attr(this, value_attr_id, pkt, ++ *((uint32_t *)((uint8_t *)buf))); ++ break; ++ case 8: ++ set_attr(this, value_attr_id, pkt, ++ *((uint64_t *)((uint8_t *)buf))); ++ break; ++ default: { ++ void *mt_ptr = _ptr_attr_with_offset( ++ attr_offset(value_attr_id), pkt); ++ bess::utils::CopySmall(mt_ptr, buf, value_size); ++ } break; ++ } + } + } + } -+ } + -+ (*Outgate)[init] = result[init].ogate; -+ ++ (*Outgate)[init] = result[init].ogate; + } + delete[] entry; + delete[] key_masked; + delete[] key_ptr; + return 1; -+ } -+ ++} ++ void WildcardMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { gate_idx_t default_gate; - @@ -195,26 +231,24 @@ index ec639a7b..ade76975 100644 int cnt = batch->cnt(); // Initialize the padding with zero -@@ -272,9 +385,15 @@ void WildcardMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { +@@ -272,9 +385,13 @@ void WildcardMatch::ProcessBatch(Context *ctx, bess::PacketBatch *batch) { } } - for (int i = 0; i < cnt; i++) { - bess::Packet *pkt = batch->pkts()[i]; - EmitPacket(ctx, pkt, LookupEntry(keys[i], default_gate, pkt)); -+ for (int i = 0; i < cnt; ) { -+ -+ int num= (cnt-i >= 32) ?32:cnt-i; -+ LookupBulkEntry(&keys[i],default_gate,i,&Outgate,num,batch); -+ for(int j=i;j<(i+num);j++) -+ { ++ for (int i = 0; i < cnt;) { ++ int num = (cnt - i >= 32) ? 32 : cnt - i; ++ LookupBulkEntry(&keys[i], default_gate, i, &Outgate, num, batch); ++ for (int j = i; j < (i + num); j++) { + EmitPacket(ctx, batch->pkts()[j], Outgate[j]); + } -+ i= i+num; ++ i = i + num; } } -@@ -282,7 +401,7 @@ std::string WildcardMatch::GetDesc() const { +@@ -282,7 +399,7 @@ std::string WildcardMatch::GetDesc() const { int num_rules = 0; for (const auto &tuple : tuples_) { @@ -223,7 +257,7 @@ index ec639a7b..ade76975 100644 } return bess::utils::Format("%zu fields, %d rules", fields_.size(), num_rules); -@@ -412,29 +531,24 @@ int WildcardMatch::AddTuple(wm_hkey_t *mask) { +@@ -412,29 +529,25 @@ int WildcardMatch::AddTuple(wm_hkey_t *mask) { bool WildcardMatch::DelEntry(int idx, wm_hkey_t *key) { struct WmTuple &tuple = tuples_[idx]; @@ -232,8 +266,9 @@ index ec639a7b..ade76975 100644 - if (!ret) { - return ret; + Initkeys(key); -+ int ret = tuple.ht->Remove(*key, wm_hash(total_key_size_), wm_eq(total_key_size_)); -+ if (ret>=0){ ++ int ret = ++ tuple.ht->Remove(*key, wm_hash(total_key_size_), wm_eq(total_key_size_)); ++ if (ret >= 0) { + return true; } - @@ -251,8 +286,7 @@ index ec639a7b..ade76975 100644 gate_idx_t gate = arg.gate(); int priority = arg.priority(); - -- wm_hkey_t key = {{0}}; -+ wm_hkey_t key = {{0}}; + wm_hkey_t key = {{0}}; wm_hkey_t mask = {{0}}; - struct WmData data; @@ -260,7 +294,7 @@ index ec639a7b..ade76975 100644 CommandResponse err = ExtractKeyMask(arg, &key, &mask); if (err.error().code() != 0) { return err; -@@ -444,14 +558,13 @@ CommandResponse WildcardMatch::CommandAdd( +@@ -444,14 +557,13 @@ CommandResponse WildcardMatch::CommandAdd( return CommandFailure(EINVAL, "Invalid gate: %hu", gate); } @@ -276,7 +310,7 @@ index ec639a7b..ade76975 100644 int idx = FindTuple(&mask); if (idx < 0) { idx = AddTuple(&mask); -@@ -459,13 +572,11 @@ CommandResponse WildcardMatch::CommandAdd( +@@ -459,13 +571,11 @@ CommandResponse WildcardMatch::CommandAdd( return CommandFailure(-idx, "failed to add a new wildcard pattern"); } } @@ -285,16 +319,16 @@ index ec639a7b..ade76975 100644 - wm_eq(total_key_size_)); - if (ret == nullptr) { + Initkeys(&key); -+ struct WmData* data_t = new WmData(data); ++ struct WmData *data_t = new WmData(data); + int ret = tuples_[idx].ht->insert_dpdk(&key, data_t); -+ if (ret <0) ++ if (ret < 0) return CommandFailure(EINVAL, "failed to add a rule"); - } - return CommandSuccess(); } -@@ -485,7 +596,7 @@ CommandResponse WildcardMatch::CommandDelete( +@@ -485,7 +595,7 @@ CommandResponse WildcardMatch::CommandDelete( } int ret = DelEntry(idx, &key); @@ -303,7 +337,7 @@ index ec639a7b..ade76975 100644 return CommandFailure(-ret, "failed to delete a rule"); } -@@ -499,7 +610,7 @@ CommandResponse WildcardMatch::CommandClear(const bess::pb::EmptyArg &) { +@@ -499,7 +609,7 @@ CommandResponse WildcardMatch::CommandClear(const bess::pb::EmptyArg &) { void WildcardMatch::Clear() { for (auto &tuple : tuples_) { @@ -312,7 +346,7 @@ index ec639a7b..ade76975 100644 } } -@@ -521,9 +632,12 @@ CommandResponse WildcardMatch::GetInitialArg(const bess::pb::EmptyArg &) { +@@ -521,9 +631,12 @@ CommandResponse WildcardMatch::GetInitialArg(const bess::pb::EmptyArg &) { // Retrieves a WildcardMatchConfig that would restore this module's // runtime configuration. CommandResponse WildcardMatch::GetRuntimeConfig(const bess::pb::EmptyArg &) { @@ -321,26 +355,26 @@ index ec639a7b..ade76975 100644 using rule_t = bess::pb::WildcardMatchCommandAddArg; - + const wm_hkey_t *key = 0; -+ WmData *data; -+ uint32_t *next =0; ++ WmData *data; ++ uint32_t *next = 0; resp.set_default_gate(default_gate_); // Each tuple provides a single mask, which may have many data-matches. -@@ -531,7 +645,11 @@ CommandResponse WildcardMatch::GetRuntimeConfig(const bess::pb::EmptyArg &) { +@@ -531,7 +644,11 @@ CommandResponse WildcardMatch::GetRuntimeConfig(const bess::pb::EmptyArg &) { wm_hkey_t mask = tuple.mask; // Each entry in the hash table has priority, ogate, and the data // (one datum per field, under the mask for this field). - for (auto &entry : tuple.ht) { -+ // for (auto &entry : tuple.ht) -+ while( (tuple.ht->Iterate((const void**)&key, (void**)&data, next)) >= (int)0) -+ { -+ entry.first = *key; -+ entry.second = *data; ++ // for (auto &entry : tuple.ht) ++ while ((tuple.ht->Iterate((const void **)&key, (void **)&data, next)) >= ++ (int)0) { ++ entry.first = *key; ++ entry.second = *data; // Create the rule instance rule_t *rule = resp.add_rules(); rule->set_priority(entry.second.priority); diff --git a/core/modules/wildcard_match.h b/core/modules/wildcard_match.h -index 85deeafc..a76cec4c 100644 +index 85deeafc..9626c290 100644 --- a/core/modules/wildcard_match.h +++ b/core/modules/wildcard_match.h @@ -45,6 +45,7 @@ using bess::utils::HashResult; @@ -351,57 +385,37 @@ index 85deeafc..a76cec4c 100644 static_assert(MAX_FIELD_SIZE <= sizeof(uint64_t), "field cannot be larger than 8 bytes"); -@@ -128,9 +129,19 @@ class wm_hash { +@@ -128,7 +129,12 @@ class wm_hash { private: size_t len_; }; - -+struct rte_hash_parameters dpdk_params1{ -+ .name= "test2", -+ .entries = 1<<19, -+ .reserved = 0, -+ .key_len = sizeof(wm_hkey_t), -+ .hash_func = rte_hash_crc, -+ .hash_func_init_val = 0, -+ .socket_id = (int)rte_socket_id(), -+ .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY}; ++struct rte_hash_parameters dpdk_params1 { ++ .name = "test2", .entries = 1 << 19, .reserved = 0, ++ .key_len = sizeof(wm_hkey_t), .hash_func = rte_hash_crc, ++ .hash_func_init_val = 0, .socket_id = (int)rte_socket_id(), ++ .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY ++}; class WildcardMatch final : public Module { public: -+ -+ static const gate_idx_t kNumOGates = MAX_GATES; - - static const Commands cmds; -@@ -143,8 +154,8 @@ class WildcardMatch final : public Module { - fields_(), - values_(), - tuples_() { -- max_allowed_workers_ = Worker::kMaxWorkers; -- } -+ max_allowed_workers_ = Worker::kMaxWorkers; -+ } - - CommandResponse Init(const bess::pb::WildcardMatchArg &arg); - -@@ -161,16 +172,28 @@ class WildcardMatch final : public Module { +@@ -161,16 +167,29 @@ class WildcardMatch final : public Module { CommandResponse CommandClear(const bess::pb::EmptyArg &arg); CommandResponse CommandSetDefaultGate( const bess::pb::WildcardMatchCommandSetDefaultGateArg &arg); -- + void Initkeys(wm_hkey_t *keys); -+ + private: struct WmTuple { - CuckooMap ht; -+ WmTuple() -+ { -+ -+ std::ostringstream address; ++ WmTuple() { ++ std::ostringstream address; + address << (void const *)this; + std::string name = "wildcard" + address.str(); -+ dpdk_params1.name= name.c_str(); -+ ht = new CuckooMap(0,0,&dpdk_params1); -+ } ++ dpdk_params1.name = name.c_str(); ++ ht = new CuckooMap( ++ 0, 0, &dpdk_params1); ++ } + CuckooMap *ht; wm_hkey_t mask; }; @@ -409,12 +423,14 @@ index 85deeafc..a76cec4c 100644 gate_idx_t LookupEntry(const wm_hkey_t &key, gate_idx_t def_gate, bess::Packet *pkt); -+ bool LookupBulkEntry(wm_hkey_t *key,gate_idx_t def_gate,int i, gate_idx_t (*Outgate)[32],int cnt,bess::PacketBatch *batch) ; ++ bool LookupBulkEntry(wm_hkey_t *key, gate_idx_t def_gate, int i, ++ gate_idx_t (*Outgate)[32], int cnt, ++ bess::PacketBatch *batch); + CommandResponse AddFieldOne(const bess::pb::Field &field, struct WmField *f, uint8_t type); -@@ -182,9 +205,7 @@ class WildcardMatch final : public Module { +@@ -182,9 +201,7 @@ class WildcardMatch final : public Module { int FindTuple(wm_hkey_t *mask); int AddTuple(wm_hkey_t *mask); bool DelEntry(int idx, wm_hkey_t *key); @@ -424,320 +440,301 @@ index 85deeafc..a76cec4c 100644 gate_idx_t default_gate_; size_t total_key_size_; /* a multiple of sizeof(uint64_t) */ -@@ -194,6 +215,8 @@ class WildcardMatch final : public Module { +@@ -194,6 +211,7 @@ class WildcardMatch final : public Module { std::vector fields_; std::vector values_; std::vector tuples_; + std::vector data_; -+ }; #endif // BESS_MODULES_WILDCARDMATCH_H_ diff --git a/core/utils/cuckoo_map.h b/core/utils/cuckoo_map.h -index c855b93d..1d83d8c0 100644 +index c855b93d..24c69334 100644 --- a/core/utils/cuckoo_map.h +++ b/core/utils/cuckoo_map.h @@ -50,6 +50,8 @@ #include "../debug.h" #include "common.h" -+#include +#include ++#include namespace bess { namespace utils { -@@ -74,6 +76,11 @@ typedef uint32_t EntryIndex; +@@ -74,6 +76,12 @@ typedef uint32_t EntryIndex; template , typename E = std::equal_to> class CuckooMap { + private: -+ struct rte_hash *hash = nullptr; -+ bool IsDpdk =false; -+ uint32_t key_len =0; -+ rte_hash_parameters rt; ++ struct rte_hash* hash = nullptr; ++ bool IsDpdk = false; ++ uint32_t key_len = 0; ++ rte_hash_parameters rt; ++ public: typedef std::pair Entry; -@@ -149,20 +156,45 @@ class CuckooMap { +@@ -149,20 +157,39 @@ class CuckooMap { }; CuckooMap(size_t reserve_buckets = kInitNumBucket, - size_t reserve_entries = kInitNumEntries) -+ size_t reserve_entries = kInitNumEntries,void* dpdk_params=0) ++ size_t reserve_entries = kInitNumEntries, void* dpdk_params = 0) : bucket_mask_(reserve_buckets - 1), num_entries_(0), buckets_(reserve_buckets), entries_(reserve_entries), free_entry_indices_() { -+ -+ if(dpdk_params) -+ { -+ if( hash == NULL) -+ { -+ rt = *((rte_hash_parameters *) dpdk_params); -+ key_len = rt.key_len; -+ hash = rte_hash_create(&rt); -+ -+ if(hash==NULL) -+ throw std::runtime_error("DPDK rte_hash_create() returned null , cant proceed further"); -+ } -+ IsDpdk = true; -+ } -+ else -+ { - // the number of buckets must be a power of 2 - CHECK_EQ(align_ceil_pow2(reserve_buckets), reserve_buckets); - - for (int i = reserve_entries - 1; i >= 0; --i) { - free_entry_indices_.push(i); +- // the number of buckets must be a power of 2 +- CHECK_EQ(align_ceil_pow2(reserve_buckets), reserve_buckets); ++ if (dpdk_params) { ++ if (hash == NULL) { ++ rt = *((rte_hash_parameters*)dpdk_params); ++ key_len = rt.key_len; ++ hash = rte_hash_create(&rt); ++ ++ if (hash == NULL) ++ throw std::runtime_error( ++ "DPDK rte_hash_create() returned null , cant proceed further"); ++ } ++ IsDpdk = true; ++ } else { ++ // the number of buckets must be a power of 2 ++ CHECK_EQ(align_ceil_pow2(reserve_buckets), reserve_buckets); + +- for (int i = reserve_entries - 1; i >= 0; --i) { +- free_entry_indices_.push(i); ++ for (int i = reserve_entries - 1; i >= 0; --i) { ++ free_entry_indices_.push(i); ++ } } -+ } } -+ ~CuckooMap() -+ { -+ if(hash != nullptr) -+ { ++ ~CuckooMap() { ++ if (hash != nullptr) { + rte_hash_free(hash); -+ hash=nullptr; ++ hash = nullptr; + } + } // Not allowing copying for now CuckooMap(CuckooMap&) = delete; CuckooMap& operator=(CuckooMap&) = delete; -@@ -176,6 +208,17 @@ class CuckooMap { +@@ -176,6 +203,16 @@ class CuckooMap { template Entry* DoEmplace(const K& key, const H& hasher, const E& eq, Args&&... args) { -+ -+ if(IsDpdk) -+ { -+ Entry *entry1 = new Entry; ++ if (IsDpdk) { ++ Entry* entry1 = new Entry; + new (&entry1->second) V(std::forward(args)...); -+ int ret1 = rte_hash_add_key_data(hash, &key,(void*)(&entry1->second)); -+ if(ret1<0)return nullptr; -+ entry1->first=key; ++ int ret1 = rte_hash_add_key_data(hash, &key, (void*)(&entry1->second)); ++ if (ret1 < 0) ++ return nullptr; ++ entry1->first = key; + return entry1; + } + Entry* entry; HashResult primary = Hash(key, hasher); -@@ -220,6 +263,53 @@ class CuckooMap { +@@ -220,6 +257,50 @@ class CuckooMap { return DoEmplace(key, hasher, eq, std::move(value)); } -+ int insert_dpdk(const void *key, void *data=0, hash_sig_t sig=0) -+ { -+ if(IsDpdk) -+ { -+ if(data && !sig) -+ return rte_hash_add_key_data(hash, key, data); -+ if(data &&sig) -+ return rte_hash_add_key_with_hash_data(hash,key, sig, data); -+ if (!data && !sig) -+ return rte_hash_add_key(hash, key); -+ -+ } -+ return -1; ++ int insert_dpdk(const void* key, void* data = 0, hash_sig_t sig = 0) { ++ if (IsDpdk) { ++ if (data && !sig) ++ return rte_hash_add_key_data(hash, key, data); ++ if (data && sig) ++ return rte_hash_add_key_with_hash_data(hash, key, sig, data); ++ if (!data && !sig) ++ return rte_hash_add_key(hash, key); ++ } ++ return -1; + } + -+ int find_dpdk(const void *key, void **data=0, hash_sig_t sig=0)/*(const K& key, void* out) */ -+ { -+ if(IsDpdk ) -+ { -+ if(data && !sig) -+ return rte_hash_lookup_data(hash, key, data); -+ if(data && sig) -+ return rte_hash_lookup_with_hash_data(hash,key, sig, data); -+ if(!data && !sig) -+ return rte_hash_lookup(hash, key); -+ if(!data && sig) -+ return rte_hash_lookup_with_hash(hash,key,sig); -+ } -+ return -1; ++ int find_dpdk(const void* key, void** data = 0, ++ hash_sig_t sig = 0) /*(const K& key, void* out) */ ++ { ++ if (IsDpdk) { ++ if (data && !sig) ++ return rte_hash_lookup_data(hash, key, data); ++ if (data && sig) ++ return rte_hash_lookup_with_hash_data(hash, key, sig, data); ++ if (!data && !sig) ++ return rte_hash_lookup(hash, key); ++ if (!data && sig) ++ return rte_hash_lookup_with_hash(hash, key, sig); + } ++ return -1; ++ } + -+ int find_dpdk(const void *key, void **data=0, hash_sig_t sig=0) const /*(const K& key, void* out) */ -+ { -+ if(IsDpdk ) -+ { -+ if(data && !sig) -+ return rte_hash_lookup_data(hash, key, data); -+ if(data && sig) -+ return rte_hash_lookup_with_hash_data(hash,key, sig, data); -+ if(!data && !sig) -+ return rte_hash_lookup(hash, key); -+ if(!data && sig) -+ return rte_hash_lookup_with_hash(hash,key,sig); -+ } -+ return -1; ++ int find_dpdk(const void* key, void** data = 0, ++ hash_sig_t sig = 0) const /*(const K& key, void* out) */ ++ { ++ if (IsDpdk) { ++ if (data && !sig) ++ return rte_hash_lookup_data(hash, key, data); ++ if (data && sig) ++ return rte_hash_lookup_with_hash_data(hash, key, sig, data); ++ if (!data && !sig) ++ return rte_hash_lookup(hash, key); ++ if (!data && sig) ++ return rte_hash_lookup_with_hash(hash, key, sig); + } ++ return -1; ++ } + // Emplace/update-in-place a key value pair // On success returns a pointer to the inserted entry, nullptr otherwise. // NOTE: when Emplace() returns nullptr, the constructor of `V` may not be -@@ -242,6 +332,17 @@ class CuckooMap { +@@ -242,6 +323,16 @@ class CuckooMap { // const version of Find() const Entry* Find(const K& key, const H& hasher = H(), const E& eq = E()) const { -+ -+ if(IsDpdk) -+ { -+ Entry* ans = new Entry; -+ V* data; -+ int ret = rte_hash_lookup_data(hash, &key,(void**)&data); -+ if(ret<0) return NULL; -+ ans->first =key; -+ ans->second =*data; -+ return ans; -+ } ++ if (IsDpdk) { ++ Entry* ans = new Entry; ++ V* data; ++ int ret = rte_hash_lookup_data(hash, &key, (void**)&data); ++ if (ret < 0) ++ return NULL; ++ ans->first = key; ++ ans->second = *data; ++ return ans; ++ } EntryIndex idx = FindWithHash(Hash(key, hasher), key, eq); if (idx == kInvalidEntryIdx) { return nullptr; -@@ -255,6 +356,13 @@ class CuckooMap { +@@ -255,6 +346,13 @@ class CuckooMap { // Remove the stored entry by the key // Return false if not exist. bool Remove(const K& key, const H& hasher = H(), const E& eq = E()) { -+ -+ if(IsDpdk) -+ { -+ int ret = rte_hash_del_key(hash, &key); -+ if(ret < 0)return false; -+ else return true; ++ if (IsDpdk) { ++ int ret = rte_hash_del_key(hash, &key); ++ if (ret < 0) ++ return false; ++ else ++ return true; + } HashResult pri = Hash(key, hasher); if (RemoveFromBucket(pri, pri & bucket_mask_, key, eq)) { return true; -@@ -267,6 +375,19 @@ class CuckooMap { +@@ -267,6 +365,17 @@ class CuckooMap { } void Clear() { -+ -+ if(IsDpdk) -+ { -+ if(hash) -+ { -+ rte_hash_free(hash); -+ hash=nullptr; -+ hash = rte_hash_create(&rt); -+ if(hash == NULL) -+ throw std::runtime_error("DPDK rte_hash_create() returned null , cant proceed further"); ++ if (IsDpdk) { ++ if (hash) { ++ rte_hash_free(hash); ++ hash = nullptr; ++ hash = rte_hash_create(&rt); ++ if (hash == NULL) ++ throw std::runtime_error( ++ "DPDK rte_hash_create() returned null , cant proceed further"); + } -+ return; ++ return; + } buckets_.clear(); entries_.clear(); -@@ -286,8 +407,29 @@ class CuckooMap { +@@ -286,7 +395,26 @@ class CuckooMap { } // Return the number of stored entries - size_t Count() const { return num_entries_; } -+ size_t Count() const -+ { -+ if(IsDpdk) -+ return rte_hash_count(hash); -+ else -+ return num_entries_; ++ size_t Count() const { ++ if (IsDpdk) ++ return rte_hash_count(hash); ++ else ++ return num_entries_; ++ } ++ ++ // bulk data look up bess func ++ int32_t lookup_bulk_data(const void** keys, uint32_t num_keys, ++ uint64_t* hit_mask, void* data[]) { ++ if (IsDpdk) ++ return rte_hash_lookup_bulk_data(hash, keys, num_keys, hit_mask, data); ++ return -1; ++ } ++ // iterate for dpdk hash ++ int32_t Iterate(const void** key, void** data, uint32_t* next) { ++ if (IsDpdk) ++ return rte_hash_iterate(hash, key, data, next); ++ return -1; + } -+ //bulk data look up bess func -+ int32_t lookup_bulk_data(const void **keys, -+ uint32_t num_keys, uint64_t *hit_mask, void *data[]) -+ { -+ if(IsDpdk) -+ return rte_hash_lookup_bulk_data(hash, keys,num_keys, hit_mask, data); -+ return -1; -+ } -+//iterate for dpdk hash -+ int32_t Iterate(const void **key, void **data, uint32_t *next) -+ { -+ if(IsDpdk) -+ return rte_hash_iterate(hash, key, data, next); -+ return -1; -+ } protected: // Tunable macros - static const int kInitNumBucket = 4; diff --git a/core/utils/exact_match_table.h b/core/utils/exact_match_table.h -index 7c0cfda4..28301580 100644 +index 7c0cfda4..fd472fdd 100644 --- a/core/utils/exact_match_table.h +++ b/core/utils/exact_match_table.h -@@ -155,15 +155,37 @@ typedef std::vector> ExactMatchRuleFields; +@@ -155,15 +155,31 @@ typedef std::vector> ExactMatchRuleFields; template class ExactMatchTable { public: -- using EmTable = -- CuckooMap; ++ struct rte_hash_parameters dpdk_params { ++ .name = "test1", .entries = 1 << 20, .reserved = 0, ++ .key_len = sizeof(ExactMatchKey), .hash_func = rte_hash_crc, ++ .hash_func_init_val = 0, .socket_id = (int)rte_socket_id(), ++ .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY ++ }; ++ + using EmTable = + CuckooMap; - -+ struct rte_hash_parameters dpdk_params -+ { -+ .name= "test1", -+ .entries = 1<<20, -+ .reserved = 0, -+ .key_len = sizeof(ExactMatchKey), -+ .hash_func = rte_hash_crc, -+ .hash_func_init_val = 0, -+ .socket_id = (int)rte_socket_id(), -+ .extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY}; -+ -+ using EmTable = CuckooMap; ExactMatchTable() - : raw_key_size_(), - total_key_size_(), - num_fields_(), +- : raw_key_size_(), +- total_key_size_(), +- num_fields_(), - fields_(), - table_() {} -+ fields_() -+ { -+ std::ostringstream address; -+ address << (void const *)this; -+ std::string name = "Exactmatch" + address.str(); -+ dpdk_params.name= name.c_str(); -+ table_.reset( new CuckooMap(0,0,&dpdk_params) ); -+ } -+ -+ void Initkeys(ExactMatchKey *keys) -+ { -+ int i = sizeof(ExactMatchKey) - total_key_size_; -+ if(i) -+ memset(&keys->u64_arr[total_key_size_/sizeof(keys->u64_arr[0])],0,i); -+ } ++ : raw_key_size_(), total_key_size_(), num_fields_(), fields_() { ++ std::ostringstream address; ++ address << (void const *)this; ++ std::string name = "Exactmatch" + address.str(); ++ dpdk_params.name = name.c_str(); ++ table_.reset( ++ new CuckooMap( ++ 0, 0, &dpdk_params)); ++ } ++ ++ void Initkeys(ExactMatchKey *keys) { ++ int i = sizeof(ExactMatchKey) - total_key_size_; ++ if (i) ++ memset(&keys->u64_arr[total_key_size_ / sizeof(keys->u64_arr[0])], 0, i); ++ } // Add a new rule. // -@@ -182,12 +204,12 @@ class ExactMatchTable { - +@@ -183,10 +199,10 @@ class ExactMatchTable { if ((err = gather_key(fields, &key)).first != 0) { return err; -- } + } - - table_.Insert(key, val, ExactMatchKeyHash(total_key_size_), - ExactMatchKeyEq(total_key_size_)); - -- return MakeError(0); -+ } -+ Initkeys(&key); -+ const void* Key_t = (const void*)&key; -+ T* val_t = new T(val); -+ table_->insert_dpdk(Key_t, val_t); -+ return MakeError(0); ++ Initkeys(&key); ++ const void *Key_t = (const void *)&key; ++ T *val_t = new T(val); ++ table_->insert_dpdk(Key_t, val_t); + return MakeError(0); } - // Delete an existing rule. -@@ -206,8 +228,9 @@ class ExactMatchTable { +@@ -206,9 +222,10 @@ class ExactMatchTable { if ((err = gather_key(fields, &key)).first != 0) { return err; } + Initkeys(&key); - bool ret = table_.Remove(key, ExactMatchKeyHash(total_key_size_), +- ExactMatchKeyEq(total_key_size_)); + bool ret = table_->Remove(key, ExactMatchKeyHash(total_key_size_), - ExactMatchKeyEq(total_key_size_)); ++ ExactMatchKeyEq(total_key_size_)); if (!ret) { return MakeError(ENOENT, "rule doesn't exist"); -@@ -217,9 +240,9 @@ class ExactMatchTable { + } +@@ -217,9 +234,9 @@ class ExactMatchTable { } // Remove all rules from the table. @@ -749,57 +746,49 @@ index 7c0cfda4..28301580 100644 // Extract an ExactMatchKey from `buf` based on the fields that have been // added to this table. -@@ -272,9 +295,16 @@ class ExactMatchTable { +@@ -272,23 +289,31 @@ class ExactMatchTable { // Returns the value if `key` matches a rule, otherwise `default_value`. T Find(const ExactMatchKey &key, const T &default_value) const { const auto &table = table_; - const auto *entry = table.Find(key, ExactMatchKeyHash(total_key_size_), - ExactMatchKeyEq(total_key_size_)); - return entry ? entry->second : default_value; -+ ExactMatchKey &t1 = const_cast(key) ; -+ const_cast(this)->Initkeys(&t1); ++ ExactMatchKey &t1 = const_cast(key); ++ const_cast(this)->Initkeys(&t1); + void *data = nullptr; -+ table->find_dpdk( &key,&data); -+ if(data) -+ { -+ T data_t = *((T*)data) ; -+ return data_t; -+ } -+ else return default_value; ++ table->find_dpdk(&key, &data); ++ if (data) { ++ T data_t = *((T *)data); ++ return data_t; ++ } else ++ return default_value; } // Find entries for `n` `keys` in the table and store their values in in -@@ -283,11 +313,24 @@ class ExactMatchTable { - void Find(const ExactMatchKey *keys, T *vals, size_t n, - T default_value) const { + // `vals`. Keys without entries will have their corresponding entires in + // `vals` set to `default_value`. +- void Find(const ExactMatchKey *keys, T *vals, size_t n, +- T default_value) const { ++ uint64_t Find(ExactMatchKey *keys, T **vals, int n) { const auto &table = table_; - for (size_t i = 0; i < n; i++) { - const auto *entry = - table.Find(keys[i], ExactMatchKeyHash(total_key_size_), - ExactMatchKeyEq(total_key_size_)); - vals[i] = entry ? entry->second : default_value; -+ uint64_t *hit_mask; -+ int num =0; -+ for (size_t i = 0; i < n; i=i+32) -+ { -+ num= (n-i >= 32) ?32:n-i; -+ for(int h=0;hlookup_bulk_data((const void **)&key_ptr, n, &hit_mask, ++ (void **)&vals[0]); ++ return hit_mask; } -@@ -318,9 +361,9 @@ class ExactMatchTable { + uint32_t total_key_size() const { return total_key_size_; } +@@ -318,9 +343,9 @@ class ExactMatchTable { // Returns the ith field. const ExactMatchField &get_field(size_t i) const { return fields_[i]; } @@ -811,7 +800,7 @@ index 7c0cfda4..28301580 100644 private: Error MakeError(int code, const std::string &msg = "") { -@@ -447,12 +490,10 @@ class ExactMatchTable { +@@ -447,14 +472,14 @@ class ExactMatchTable { // aligned total key size size_t total_key_size_; @@ -820,10 +809,16 @@ index 7c0cfda4..28301580 100644 ExactMatchField fields_[MAX_FIELDS]; - - EmTable table_; --}; -+ std::unique_ptr> table_; -+ }; ++ std::unique_ptr< ++ CuckooMap> ++ table_; + }; - } // namespace bess +-} // namespace bess } // namespace utils ++} // namespace bess + + #endif // BESS_UTILS_EXACT_MATCH_TABLE_H_ -- +2.25.1 +