dmaengine: idxd: fix lockdep warning on device driver removal

Jacob reported that with lockdep debug turned on, idxd_device_driver
removal causes kernel splat from lock assert warning for
idxd_device_wqs_clear_state(). Make sure
idxd_device_wqs_clear_state() holds the wq lock for each wq when
cleaning the wq state. Move the call outside of the device spinlock.

Reported-by: Jacob Pan <jacob.jun.pan@intel.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Link: https://lore.kernel.org/r/165231364426.986304.9294302800482492780.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Vinod Koul <vkoul@kernel.org>

drivers/dma/idxd/device.c

@@ -578,19 +578,15 @@ int idxd_device_disable(struct idxd_device *idxd)
 		return -ENXIO;
 	}
 
-	spin_lock(&idxd->dev_lock);
 	idxd_device_clear_state(idxd);
-	idxd->state = IDXD_DEV_DISABLED;
-	spin_unlock(&idxd->dev_lock);
 	return 0;
 }
 
 void idxd_device_reset(struct idxd_device *idxd)
 {
 	idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);
-	spin_lock(&idxd->dev_lock);
 	idxd_device_clear_state(idxd);
-	idxd->state = IDXD_DEV_DISABLED;
+	spin_lock(&idxd->dev_lock);
 	idxd_unmask_error_interrupts(idxd);
 	spin_unlock(&idxd->dev_lock);
 }
@@ -717,23 +713,27 @@ static void idxd_device_wqs_clear_state(struct idxd_device *idxd)
 {
 	int i;
 
-	lockdep_assert_held(&idxd->dev_lock);
 	for (i = 0; i < idxd->max_wqs; i++) {
 		struct idxd_wq *wq = idxd->wqs[i];
 
 		if (wq->state == IDXD_WQ_ENABLED) {
+			mutex_lock(&wq->wq_lock);
 			idxd_wq_disable_cleanup(wq);
 			idxd_wq_device_reset_cleanup(wq);
 			wq->state = IDXD_WQ_DISABLED;
+			mutex_unlock(&wq->wq_lock);
 		}
 	}
 }
 
 void idxd_device_clear_state(struct idxd_device *idxd)
 {
+	idxd_device_wqs_clear_state(idxd);
+	spin_lock(&idxd->dev_lock);
 	idxd_groups_clear_state(idxd);
 	idxd_engines_clear_state(idxd);
-	idxd_device_wqs_clear_state(idxd);
+	idxd->state = IDXD_DEV_DISABLED;
+	spin_unlock(&idxd->dev_lock);
 }
 
 static void idxd_group_config_write(struct idxd_group *group)