vchiq_arm: Add completion records under the mutex

An issue was observed when flushing openmax components
which generate a large number of messages returning
buffers to host.

We occasionally found a duplicate message from 16
messages prior, resulting in a buffer returned twice.

While only one thread adds completions, without the
mutex you don't get the protection of the automatic
memory barrier you get with synchronisation objects.

Signed-off-by: Phil Elwell <phil@raspberrypi.org>

drivers/misc/vc04_services/interface/vchiq_arm/vchiq_arm.c

@@ -210,18 +210,26 @@ add_completion(VCHIQ_INSTANCE_T instance, VCHIQ_REASON_T reason,
 	VCHIQ_COMPLETION_DATA_T *completion;
 	DEBUG_INITIALISE(g_state.local)
 
+	mutex_lock(&instance->completion_mutex);
+
 	while (instance->completion_insert ==
 		(instance->completion_remove + MAX_COMPLETIONS)) {
 		/* Out of space - wait for the client */
 		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
 		vchiq_log_trace(vchiq_arm_log_level,
 			"add_completion - completion queue full");
 		DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT);
+
+		mutex_unlock(&instance->completion_mutex);
 		if (down_interruptible(&instance->remove_event) != 0) {
 			vchiq_log_info(vchiq_arm_log_level,
 				"service_callback interrupted");
 			return VCHIQ_RETRY;
-		} else if (instance->closing) {
+		}
+
+		mutex_lock(&instance->completion_mutex);
+		if (instance->closing) {
+			mutex_unlock(&instance->completion_mutex);
 			vchiq_log_info(vchiq_arm_log_level,
 				"service_callback closing");
 			return VCHIQ_SUCCESS;
@@ -254,8 +262,11 @@ add_completion(VCHIQ_INSTANCE_T instance, VCHIQ_REASON_T reason,
 	if (reason == VCHIQ_MESSAGE_AVAILABLE)
 		user_service->message_available_pos =
 			instance->completion_insert;
+
 	instance->completion_insert++;
 
+	mutex_unlock(&instance->completion_mutex);
+
 	up(&instance->insert_event);
 
 	return VCHIQ_SUCCESS;