Merge pull request Open-CAS#247 from arutk/percpu_freelist_rebased

 Per-execution-context freelists

env/posix/ocf_env.c

@@ -130,8 +130,64 @@ void env_stack_trace(void)
 }
 
 /* *** CRC *** */
-
 uint32_t env_crc32(uint32_t crc, uint8_t const *data, size_t len)
 {
 	return crc32(crc, data, len);
 }
+
+/* *** execution contexts *** */
+pthread_mutex_t *exec_context_mutex;
+
+static void __attribute__((constructor)) init_execution_context(void)
+{
+	unsigned count = env_get_execution_context_count();
+	unsigned i;
+
+	ENV_BUG_ON(count == 0);
+	exec_context_mutex = malloc(count * sizeof(exec_context_mutex[0]));
+	ENV_BUG_ON(exec_context_mutex == NULL);
+	for (i = 0; i < count; i++)
+		ENV_BUG_ON(pthread_mutex_init(&exec_context_mutex[i], NULL));
+}
+
+static void __attribute__((destructor)) deinit_execution_context(void)
+{
+	unsigned count = env_get_execution_context_count();
+	unsigned i;
+
+	ENV_BUG_ON(count == 0);
+	ENV_BUG_ON(exec_context_mutex == NULL);
+
+	for (i = 0; i < count; i++)
+		ENV_BUG_ON(pthread_mutex_destroy(&exec_context_mutex[i]));
+	free(exec_context_mutex);
+}
+
+/* get_execuction_context must assure that after the call finishes, the caller
+ * will not get preempted from current execution context. For userspace env
+ * we simulate this behavior by acquiring per execution context mutex. As a
+ * result the caller might actually get preempted, but no other thread will
+ * execute in this context by the time the caller puts current execution ctx. */
+unsigned env_get_execution_context(void)
+{
+	unsigned cpu;
+
+	cpu = sched_getcpu();
+	cpu = (cpu == -1) ?  0 : cpu;
+
+	ENV_BUG_ON(pthread_mutex_lock(&exec_context_mutex[cpu]));
+
+	return cpu;
+}
+
+void env_put_execution_context(unsigned ctx)
+{
+	pthread_mutex_unlock(&exec_context_mutex[ctx]);
+}
+
+unsigned env_get_execution_context_count(void)
+{
+	int num = sysconf(_SC_NPROCESSORS_ONLN);
+
+	return (num == -1) ? 0 : num;
+}

env/posix/ocf_env.h

@@ -455,6 +455,11 @@ static inline void env_spinlock_init(env_spinlock *l)
 	ENV_BUG_ON(pthread_spin_init(&l->lock, 0));
 }
 
+static inline int env_spinlock_trylock(env_spinlock *l)
+{
+	return pthread_spin_trylock(&l->lock) ? -OCF_ERR_NO_LOCK : 0;
+}
+
 static inline void env_spinlock_lock(env_spinlock *l)
 {
 	ENV_BUG_ON(pthread_spin_lock(&l->lock));
@@ -644,4 +649,8 @@ uint32_t env_crc32(uint32_t crc, uint8_t const *data, size_t len);
 
 #define ENV_PRIu64 "lu"
 
+unsigned env_get_execution_context(void);
+void env_put_execution_context(unsigned ctx);
+unsigned env_get_execution_context_count(void);
+
 #endif /* __OCF_ENV_H__ */

src/engine/cache_engine.c

@@ -169,7 +169,8 @@ static inline bool ocf_seq_cutoff_is_on(ocf_cache_t cache)
 	if (!ocf_cache_is_device_attached(cache))
 		return false;
 
-	return (cache->device->freelist_part->curr_size <= SEQ_CUTOFF_FULL_MARGIN);
+	return (ocf_freelist_num_free(cache->freelist) <=
+			SEQ_CUTOFF_FULL_MARGIN);
 }
 
 bool ocf_seq_cutoff_check(ocf_core_t core, uint32_t dir, uint64_t addr,

src/engine/engine_common.c

@@ -7,6 +7,7 @@
 #include "../ocf_priv.h"
 #include "../ocf_cache_priv.h"
 #include "../ocf_queue_priv.h"
+#include "../ocf_freelist.h"
 #include "engine_common.h"
 #define OCF_ENGINE_DEBUG_IO_NAME "common"
 #include "engine_debug.h"
@@ -250,19 +251,11 @@ static void ocf_engine_map_cache_line(struct ocf_request *req,
 	ocf_part_id_t part_id = req->part_id;
 	ocf_cleaning_t clean_policy_type;
 
-	if (cache->device->freelist_part->curr_size == 0) {
+	if (!ocf_freelist_get_cache_line(cache->freelist, cache_line)) {
 		req->info.mapping_error = 1;
 		return;
 	}
 
-	*cache_line = cache->device->freelist_part->head;
-
-	/* add_to_collision_list changes .next_col and other fields for entry
-	 * so updated last_cache_line_give must be updated before calling it.
-	 */
-
-	ocf_metadata_remove_from_free_list(cache, *cache_line);
-
 	ocf_metadata_add_to_partition(cache, part_id, *cache_line);
 
 	/* Add the block to the corresponding collision list */

src/eviction/eviction.c

@@ -107,11 +107,13 @@ int space_managment_evict_do(struct ocf_cache *cache,
 		struct ocf_request *req, uint32_t evict_cline_no)
 {
 	uint32_t evicted;
+	uint32_t free;
 
-	if (evict_cline_no <= cache->device->freelist_part->curr_size)
+	free = ocf_freelist_num_free(cache->freelist);
+	if (evict_cline_no <= free)
 		return LOOKUP_MAPPED;
 
-	evict_cline_no -= cache->device->freelist_part->curr_size;
+	evict_cline_no -= free;
 	evicted = ocf_evict_do(cache, req->io_queue, evict_cline_no,
 			req->part_id);
 

src/eviction/eviction.h

@@ -32,8 +32,7 @@ union eviction_policy_meta {
  * set core_id to -2 to purge the whole cache partition
  */
 struct eviction_policy_ops {
-	void (*init_cline)(ocf_cache_t cache,
-			ocf_cache_line_t cline);
+	void (*init_cline)(ocf_cache_t cache, ocf_cache_line_t cline);
 	void (*rm_cline)(ocf_cache_t cache,
 			ocf_cache_line_t cline);
 	bool (*can_evict)(ocf_cache_t cache);

src/eviction/lru.h

@@ -8,8 +8,7 @@
 #include "eviction.h"
 #include "lru_structs.h"
 
-void evp_lru_init_cline(struct ocf_cache *cache,
-		ocf_cache_line_t cline);
+void evp_lru_init_cline(struct ocf_cache *cache, ocf_cache_line_t cline);
 void evp_lru_rm_cline(struct ocf_cache *cache, ocf_cache_line_t cline);
 bool evp_lru_can_evict(struct ocf_cache *cache);
 uint32_t evp_lru_req_clines(struct ocf_cache *cache, ocf_queue_t io_queue,

src/metadata/metadata.c

@@ -53,16 +53,16 @@ int ocf_metadata_init_variable_size(struct ocf_cache *cache, uint64_t device_siz
 			cache_line_size, layout);
 }
 
-void ocf_metadata_init_freelist_partition(struct ocf_cache *cache)
+void ocf_metadata_init_hash_table(struct ocf_cache *cache)
 {
 	OCF_DEBUG_TRACE(cache);
-	cache->metadata.iface.init_freelist(cache);
+	cache->metadata.iface.init_hash_table(cache);
 }
 
-void ocf_metadata_init_hash_table(struct ocf_cache *cache)
+void ocf_metadata_init_collision(struct ocf_cache *cache)
 {
 	OCF_DEBUG_TRACE(cache);
-	cache->metadata.iface.init_hash_table(cache);
+	cache->metadata.iface.init_collision(cache);
 }
 
 void ocf_metadata_deinit(struct ocf_cache *cache)

src/metadata/metadata.h

@@ -60,6 +60,13 @@ void ocf_metadata_init_freelist_partition(struct ocf_cache *cache);
  */
 void ocf_metadata_init_hash_table(struct ocf_cache *cache);
 
+/**
+ * @brief Initialize collision table
+ *
+ * @param cache - Cache instance
+ */
+void ocf_metadata_init_collision(struct ocf_cache *cache);
+
 /**
  * @brief De-Initialize metadata
  *
@@ -207,4 +214,10 @@ typedef void (*ocf_metadata_load_properties_end_t)(void *priv, int error,
 void ocf_metadata_load_properties(ocf_volume_t volume,
 		ocf_metadata_load_properties_end_t cmpl, void *priv);
 
+static inline ocf_cache_line_t ocf_metadata_collision_table_entries(
+		struct ocf_cache *cache)
+{
+	return cache->device->collision_table_entries;
+}
+
 #endif /* METADATA_H_ */

src/metadata/metadata_hash.c

@@ -13,6 +13,7 @@
 #include "../utils/utils_pipeline.h"
 #include "../ocf_def_priv.h"
 #include "../ocf_priv.h"
+#include "../ocf_freelist.h"
 
 #define OCF_METADATA_HASH_DEBUG 0
 
@@ -1025,51 +1026,28 @@ static int ocf_metadata_hash_init_variable_size(struct ocf_cache *cache,
 }
 
 static inline void _ocf_init_collision_entry(struct ocf_cache *cache,
-		ocf_cache_line_t idx, ocf_cache_line_t next,
-		ocf_cache_line_t prev)
+		ocf_cache_line_t idx)
 {
 	ocf_cache_line_t invalid_idx = cache->device->collision_table_entries;
-	ocf_part_id_t invalid_part_id = PARTITION_INVALID;
 
-	ocf_metadata_set_partition_info(cache, idx,
-			invalid_part_id, next, prev);
 	ocf_metadata_set_collision_info(cache, idx, invalid_idx, invalid_idx);
 	ocf_metadata_set_core_info(cache, idx,
 			OCF_CORE_MAX, ULONG_MAX);
 	metadata_init_status_bits(cache, idx);
 }
 
 /*
- * Modified initialization of freelist partition
+ * Initialize collision table
  */
-static void ocf_metadata_hash_init_freelist(struct ocf_cache *cache)
+static void ocf_metadata_hash_init_collision(struct ocf_cache *cache)
 {
-	uint32_t step = 0;
 	unsigned int i;
-	ocf_cache_line_t prev, next;
-	ocf_cache_line_t idx;
-	ocf_cache_line_t collision_table_entries =
-			cache->device->collision_table_entries;
 
-	prev = collision_table_entries;
-	idx = 0;
-	for (i = 0; i < cache->device->collision_table_entries - 1; i++) {
-		next = ocf_metadata_map_phy2lg(cache, i + 1);
-		_ocf_init_collision_entry(cache, idx, next, prev);
-		prev = idx;
-		idx = next;
-		OCF_COND_RESCHED_DEFAULT(step);
+	for (i = 0; i < cache->device->collision_table_entries; i++) {
+		_ocf_init_collision_entry(cache, i);
 	}
-	_ocf_init_collision_entry(cache, idx, collision_table_entries, prev);
-
-	/* Initialize freelist partition */
-	cache->device->freelist_part->head = 0;
-	cache->device->freelist_part->tail = idx;
-	cache->device->freelist_part->curr_size = cache->device->
-			collision_table_entries;
 }
 
-
 /*
  * Initialize hash table
  */
@@ -1850,8 +1828,6 @@ static void _recovery_rebuild_cline_metadata(ocf_cache_t cache,
 
 	part_id = PARTITION_DEFAULT;
 
-	ocf_metadata_remove_from_free_list(cache, cache_line);
-
 	ocf_metadata_add_to_partition(cache, part_id, cache_line);
 
 	hash_index = ocf_metadata_hash_func(cache, core_line, core_id);
@@ -1917,10 +1893,12 @@ static void _recovery_rebuild_metadata(ocf_pipeline_t pipeline,
 	ocf_core_id_t core_id;
 	uint64_t core_line;
 	unsigned char step = 0;
+	const uint64_t collision_table_entries =
+			ocf_metadata_collision_table_entries(cache);
 
 	OCF_METADATA_LOCK_WR();
 
-	for (cline = 0; cline < cache->device->collision_table_entries; cline++) {
+	for (cline = 0; cline < collision_table_entries; cline++) {
 		ocf_metadata_get_core_info(cache, cline, &core_id, &core_line);
 		if (core_id != OCF_CORE_MAX &&
 				(!dirty_only || metadata_test_dirty(cache,
@@ -2648,7 +2626,7 @@ static const struct ocf_metadata_iface metadata_hash_iface = {
 	.init_variable_size = ocf_metadata_hash_init_variable_size,
 	.deinit_variable_size = ocf_metadata_hash_deinit_variable_size,
 	.init_hash_table = ocf_metadata_hash_init_hash_table,
-	.init_freelist = ocf_metadata_hash_init_freelist,
+	.init_collision = ocf_metadata_hash_init_collision,
 
 	.layout_iface = NULL,
 	.pages = ocf_metadata_hash_pages,

src/metadata/metadata_misc.c

@@ -5,6 +5,7 @@
 
 #include "ocf/ocf.h"
 #include "metadata.h"
+#include "../ocf_freelist.h"
 #include "../utils/utils_cache_line.h"
 
 static bool _is_cache_line_acting(struct ocf_cache *cache,
@@ -100,7 +101,7 @@ void ocf_metadata_sparse_cache_line(struct ocf_cache *cache,
 
 	ocf_metadata_remove_from_partition(cache, partition_id, cache_line);
 
-	ocf_metadata_add_to_free_list(cache, cache_line);
+	ocf_freelist_put_cache_line(cache->freelist, cache_line);
 }
 
 static void _ocf_metadata_sparse_cache_line(struct ocf_cache *cache,