From 5604935e4e1cbc16611d2d97f50b717aa31e8ec5 Mon Sep 17 00:00:00 2001 From: Pravin B Shelar Date: Wed, 23 Apr 2014 08:34:08 -0700 Subject: [PATCH] datapath: Add flow mask cache. On every packet OVS needs to lookup flow-table with every mask until it finds a match. The packet flow-key is first masked with mask in the list and then the masked key is looked up in flow-table. Therefore number of masks can affect packet processing performance. Following patch adds mask index to mask cache from last pakcet lookup in same flow. Index of mask is stored in this cache. This cache is searched by 5 tuple hash (skb rxhash). Signed-off-by: Pravin B Shelar Acked-by: Thomas Graf --- datapath/datapath.c | 3 +- datapath/flow_table.c | 104 +++++++++++++++++++++++++++++++++++++----- datapath/flow_table.h | 11 ++++- 3 files changed, 103 insertions(+), 15 deletions(-) diff --git a/datapath/datapath.c b/datapath/datapath.c index fcaafd16955..10706f5117f 100644 --- a/datapath/datapath.c +++ b/datapath/datapath.c @@ -253,7 +253,8 @@ void ovs_dp_process_packet_with_key(struct sk_buff *skb, stats = this_cpu_ptr(dp->stats_percpu); /* Look up flow. */ - flow = ovs_flow_tbl_lookup_stats(&dp->table, pkt_key, &n_mask_hit); + flow = ovs_flow_tbl_lookup_stats(&dp->table, pkt_key, skb_get_rxhash(skb), + &n_mask_hit); if (unlikely(!flow)) { struct dp_upcall_info upcall; diff --git a/datapath/flow_table.c b/datapath/flow_table.c index be2d7d51fcf..cc0eaf288cd 100644 --- a/datapath/flow_table.c +++ b/datapath/flow_table.c @@ -49,6 +49,10 @@ #define TBL_MIN_BUCKETS 1024 #define REHASH_INTERVAL (10 * 60 * HZ) +#define MC_HASH_SHIFT 8 +#define MC_HASH_ENTRIES (1u << MC_HASH_SHIFT) +#define MC_HASH_SEGS ((sizeof(uint32_t) * 8) / MC_HASH_SHIFT) + static struct kmem_cache *flow_cache; struct kmem_cache *flow_stats_cache __read_mostly; @@ -211,10 +215,16 @@ int ovs_flow_tbl_init(struct flow_table *table) { struct table_instance *ti; - ti = table_instance_alloc(TBL_MIN_BUCKETS); + table->mask_cache = __alloc_percpu(sizeof(struct mask_cache_entry) * + MC_HASH_ENTRIES, __alignof__(struct mask_cache_entry)); + if (!table->mask_cache) + return -ENOMEM; - if (!ti) + ti = table_instance_alloc(TBL_MIN_BUCKETS); + if (!ti) { + free_percpu(table->mask_cache); return -ENOMEM; + } rcu_assign_pointer(table->ti, ti); INIT_LIST_HEAD(&table->mask_list); @@ -265,6 +275,7 @@ void ovs_flow_tbl_destroy(struct flow_table *table) { struct table_instance *ti = (struct table_instance __force *)table->ti; + free_percpu(table->mask_cache); table_instance_destroy(ti, false); } @@ -420,7 +431,8 @@ bool ovs_flow_cmp_unmasked_key(const struct sw_flow *flow, static struct sw_flow *masked_flow_lookup(struct table_instance *ti, const struct sw_flow_key *unmasked, - struct sw_flow_mask *mask) + struct sw_flow_mask *mask, + u32 *n_mask_hit) { struct sw_flow *flow; struct hlist_head *head; @@ -432,6 +444,7 @@ static struct sw_flow *masked_flow_lookup(struct table_instance *ti, ovs_flow_mask_key(&masked_key, unmasked, mask); hash = flow_hash(&masked_key, key_start, key_end); head = find_bucket(ti, hash); + (*n_mask_hit)++; hlist_for_each_entry_rcu(flow, head, hash_node[ti->node_ver]) { if (flow->mask == mask && flow->hash == hash && flow_cmp_masked_key(flow, &masked_key, @@ -441,30 +454,97 @@ static struct sw_flow *masked_flow_lookup(struct table_instance *ti, return NULL; } -struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *tbl, - const struct sw_flow_key *key, - u32 *n_mask_hit) + +static struct sw_flow *flow_lookup(struct flow_table *tbl, + struct table_instance *ti, + const struct sw_flow_key *key, + u32 *n_mask_hit) { - struct table_instance *ti = rcu_dereference_ovsl(tbl->ti); struct sw_flow_mask *mask; struct sw_flow *flow; - *n_mask_hit = 0; list_for_each_entry_rcu(mask, &tbl->mask_list, list) { - (*n_mask_hit)++; - flow = masked_flow_lookup(ti, key, mask); + flow = masked_flow_lookup(ti, key, mask, n_mask_hit); if (flow) /* Found */ return flow; } return NULL; } +/* + * mask_cache maps flow to probable mask. This cache is not tightly + * coupled cache, It means updates to mask list can result in inconsistent + * cache entry in mask cache. + * This is per cpu cache and is divided in MC_HASH_SEGS segments. + * In case of a hash collision the entry is hashed in next segment. + * */ +struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *tbl, + const struct sw_flow_key *key, + u32 skb_hash, + u32 *n_mask_hit) +{ + struct table_instance *ti = rcu_dereference_ovsl(tbl->ti); + struct mask_cache_entry *entries, *ce, *del; + struct sw_flow *flow; + u32 hash = skb_hash; + int seg; + + *n_mask_hit = 0; + if (unlikely(!skb_hash)) + return flow_lookup(tbl, ti, key, n_mask_hit); + + del = NULL; + entries = this_cpu_ptr(tbl->mask_cache); + + for (seg = 0; seg < MC_HASH_SEGS; seg++) { + int index; + + index = hash & (MC_HASH_ENTRIES - 1); + ce = &entries[index]; + + if (ce->skb_hash == skb_hash) { + struct sw_flow_mask *mask; + int i; + + i = 0; + list_for_each_entry_rcu(mask, &tbl->mask_list, list) { + if (ce->mask_index == i++) { + flow = masked_flow_lookup(ti, key, mask, + n_mask_hit); + if (flow) /* Found */ + return flow; + + break; + } + } + del = ce; + break; + } + + if (!del || (del->skb_hash && !ce->skb_hash)) { + del = ce; + } + + hash >>= MC_HASH_SHIFT; + } + + flow = flow_lookup(tbl, ti, key, n_mask_hit); + + if (flow) { + del->skb_hash = skb_hash; + del->mask_index = (*n_mask_hit - 1); + } + return flow; +} + struct sw_flow *ovs_flow_tbl_lookup(struct flow_table *tbl, const struct sw_flow_key *key) { + struct table_instance *ti = rcu_dereference_ovsl(tbl->ti); u32 __always_unused n_mask_hit; - return ovs_flow_tbl_lookup_stats(tbl, key, &n_mask_hit); + n_mask_hit = 0; + return flow_lookup(tbl, ti, key, &n_mask_hit); } int ovs_flow_tbl_num_masks(const struct flow_table *table) @@ -565,7 +645,7 @@ static int flow_mask_insert(struct flow_table *tbl, struct sw_flow *flow, return -ENOMEM; mask->key = new->key; mask->range = new->range; - list_add_rcu(&mask->list, &tbl->mask_list); + list_add_tail_rcu(&mask->list, &tbl->mask_list); } else { BUG_ON(!mask->ref_count); mask->ref_count++; diff --git a/datapath/flow_table.h b/datapath/flow_table.h index ddf0c01e241..281d1da8d71 100644 --- a/datapath/flow_table.h +++ b/datapath/flow_table.h @@ -36,6 +36,11 @@ #include "flow.h" +struct mask_cache_entry { + u32 skb_hash; + u32 mask_index; +}; + struct table_instance { struct flex_array *buckets; unsigned int n_buckets; @@ -47,6 +52,7 @@ struct table_instance { struct flow_table { struct table_instance __rcu *ti; + struct mask_cache_entry __percpu *mask_cache; struct list_head mask_list; unsigned long last_rehash; unsigned int count; @@ -72,8 +78,9 @@ int ovs_flow_tbl_num_masks(const struct flow_table *table); struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *table, u32 *bucket, u32 *idx); struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *, - const struct sw_flow_key *, - u32 *n_mask_hit); + const struct sw_flow_key *, + u32 skb_hash, + u32 *n_mask_hit); struct sw_flow *ovs_flow_tbl_lookup(struct flow_table *, const struct sw_flow_key *);