crypto: qat - add support for ring pair level telemetry

pilotAlpal · herbertx · commit eb52707716e3 · 2023-12-29T11:25:56.000+08:00
Expose through debugfs ring pair telemetry data for QAT GEN4 devices.

This allows to gather metrics about the PCIe channel and device TLB for
a selected ring pair. It is possible to monitor maximum 4 ring pairs at
the time per device.

For details, refer to debugfs-driver-qat_telemetry in Documentation/ABI.

This patch is based on earlier work done by Wojciech Ziemba.

Signed-off-by: Lucas Segarra Fernandez &lt;lucas.segarra.fernandez@intel.com&gt;
Reviewed-by: Giovanni Cabiddu &lt;giovanni.cabiddu@intel.com&gt;
Reviewed-by: Damian Muszynski &lt;damian.muszynski@intel.com&gt;
Signed-off-by: Herbert Xu &lt;herbert@gondor.apana.org.au&gt;
diff --git a/Documentation/ABI/testing/debugfs-driver-qat_telemetry b/Documentation/ABI/testing/debugfs-driver-qat_telemetry
@@ -101,3 +101,128 @@ Description:	(RO) Reports device telemetry counters.
 		attribute is not reported.
 
 		This attribute is only available for qat_4xxx devices.
+
+What:		/sys/kernel/debug/qat_<device>_<BDF>/telemetry/rp_<A/B/C/D>_data
+Date:		March 2024
+KernelVersion:	6.8
+Contact:	qat-linux@intel.com
+Description:	(RW) Selects up to 4 Ring Pairs (RP) to monitor, one per file,
+		and report telemetry counters related to each.
+
+		Allowed values to write:
+		========================
+		* 0 to ``<num_rps - 1>``:
+		  Ring pair to be monitored. The value of ``num_rps`` can be
+		  retrieved through ``/sys/bus/pci/devices/<BDF>/qat/num_rps``.
+		  See Documentation/ABI/testing/sysfs-driver-qat.
+
+		Reads report metrics about performance and utilization of
+		the selected RP:
+
+		=======================	========================================
+		Field			Description
+		=======================	========================================
+		sample_cnt		number of acquisitions of telemetry data
+					from the device. Reads are performed
+					every 1000 ms
+		rp_num			RP number associated with slot <A/B/C/D>
+		service_type		service associated to the RP
+		pci_trans_cnt		number of PCIe partial transactions
+		gp_lat_acc_avg		average get to put latency [ns]
+		bw_in			PCIe, write bandwidth [Mbps]
+		bw_out			PCIe, read bandwidth [Mbps]
+		at_glob_devtlb_hit	Message descriptor DevTLB hit rate
+		at_glob_devtlb_miss	Message descriptor DevTLB miss rate
+		tl_at_payld_devtlb_hit	Payload DevTLB hit rate
+		tl_at_payld_devtlb_miss	Payload DevTLB miss rate
+		======================= ========================================
+
+		Example.
+
+		Writing the value '32' to the file ``rp_C_data`` starts the
+		collection of telemetry metrics for ring pair 32::
+
+		  echo 32 > /sys/kernel/debug/qat_4xxx_0000:6b:00.0/telemetry/rp_C_data
+
+		Once a ring pair is selected, statistics can be read accessing
+		the file::
+
+		  cat /sys/kernel/debug/qat_4xxx_0000:6b:00.0/telemetry/rp_C_data
+
+		If ``control`` is set to 1, only the current values of the
+		counters are displayed::
+
+		  <counter_name> <current>
+
+		If ``control`` is 2, 3 or 4, counters are displayed in the
+		following format::
+
+		  <counter_name> <current> <min> <max> <avg>
+
+
+		On QAT GEN4 devices there are 64 RPs on a PF, so the allowed
+		values are 0..63. This number is absolute to the device.
+		If Virtual Functions (VF) are used, the ring pair number can
+		be derived from the Bus, Device, Function of the VF:
+
+		============ ====== ====== ====== ======
+		PCI BDF/VF   RP0    RP1    RP2    RP3
+		============ ====== ====== ====== ======
+		0000:6b:0.1  RP  0  RP  1  RP  2  RP  3
+		0000:6b:0.2  RP  4  RP  5  RP  6  RP  7
+		0000:6b:0.3  RP  8  RP  9  RP 10  RP 11
+		0000:6b:0.4  RP 12  RP 13  RP 14  RP 15
+		0000:6b:0.5  RP 16  RP 17  RP 18  RP 19
+		0000:6b:0.6  RP 20  RP 21  RP 22  RP 23
+		0000:6b:0.7  RP 24  RP 25  RP 26  RP 27
+		0000:6b:1.0  RP 28  RP 29  RP 30  RP 31
+		0000:6b:1.1  RP 32  RP 33  RP 34  RP 35
+		0000:6b:1.2  RP 36  RP 37  RP 38  RP 39
+		0000:6b:1.3  RP 40  RP 41  RP 42  RP 43
+		0000:6b:1.4  RP 44  RP 45  RP 46  RP 47
+		0000:6b:1.5  RP 48  RP 49  RP 50  RP 51
+		0000:6b:1.6  RP 52  RP 53  RP 54  RP 55
+		0000:6b:1.7  RP 56  RP 57  RP 58  RP 59
+		0000:6b:2.0  RP 60  RP 61  RP 62  RP 63
+		============ ====== ====== ====== ======
+
+		The mapping is only valid for the BDFs of VFs on the host.
+
+
+		The service provided on a ring-pair varies depending on the
+		configuration. The configuration for a given device can be
+		queried and set using ``cfg_services``.
+		See Documentation/ABI/testing/sysfs-driver-qat for details.
+
+		The following table reports how ring pairs are mapped to VFs
+		on the PF 0000:6b:0.0 configured for `sym;asym` or `asym;sym`:
+
+		=========== ============ =========== ============ ===========
+		PCI BDF/VF  RP0/service  RP1/service RP2/service  RP3/service
+		=========== ============ =========== ============ ===========
+		0000:6b:0.1 RP 0 asym    RP 1 sym    RP 2 asym    RP 3 sym
+		0000:6b:0.2 RP 4 asym    RP 5 sym    RP 6 asym    RP 7 sym
+		0000:6b:0.3 RP 8 asym    RP 9 sym    RP10 asym    RP11 sym
+		...         ...          ...         ...          ...
+		=========== ============ =========== ============ ===========
+
+		All VFs follow the same pattern.
+
+
+		The following table reports how ring pairs are mapped to VFs on
+		the PF 0000:6b:0.0 configured for `dc`:
+
+		=========== ============ =========== ============ ===========
+		PCI BDF/VF  RP0/service  RP1/service RP2/service  RP3/service
+		=========== ============ =========== ============ ===========
+		0000:6b:0.1 RP 0 dc      RP 1 dc     RP 2 dc      RP 3 dc
+		0000:6b:0.2 RP 4 dc      RP 5 dc     RP 6 dc      RP 7 dc
+		0000:6b:0.3 RP 8 dc      RP 9 dc     RP10 dc      RP11 dc
+		...         ...          ...         ...          ...
+		=========== ============ =========== ============ ===========
+
+		The mapping of a RP to a service can be retrieved using
+		``rp2srv`` from sysfs.
+		See Documentation/ABI/testing/sysfs-driver-qat for details.
+
+		This attribute is only available for qat_4xxx devices.
diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c
@@ -520,6 +520,7 @@ void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id)
 	hw_data->init_device = adf_gen4_init_device;
 	hw_data->reset_device = adf_reset_flr;
 	hw_data->admin_ae_mask = ADF_420XX_ADMIN_AE_MASK;
+	hw_data->num_rps = ADF_GEN4_MAX_RPS;
 	hw_data->fw_name = ADF_420XX_FW;
 	hw_data->fw_mmp_name = ADF_420XX_MMP;
 	hw_data->uof_get_name = uof_get_name_420xx;
diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c
@@ -421,6 +421,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id)
 	hw_data->init_device = adf_gen4_init_device;
 	hw_data->reset_device = adf_reset_flr;
 	hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK;
+	hw_data->num_rps = ADF_GEN4_MAX_RPS;
 	switch (dev_id) {
 	case ADF_402XX_PCI_DEVICE_ID:
 		hw_data->fw_name = ADF_402XX_FW;
diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h
@@ -278,6 +278,7 @@ struct adf_hw_device_data {
 	u8 num_logical_accel;
 	u8 num_engines;
 	u32 num_hb_ctrs;
+	u8 num_rps;
 };
 
 /* CSR write macro */
diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h
@@ -36,6 +36,7 @@
 #define ADF_GEN4_MSIX_RTTABLE_OFFSET(i)		(0x409000 + ((i) * 0x04))
 
 /* Bank and ring configuration */
+#define ADF_GEN4_MAX_RPS		64
 #define ADF_GEN4_NUM_RINGS_PER_BANK	2
 #define ADF_GEN4_NUM_BANKS_PER_VF	4
 #define ADF_GEN4_ETR_MAX_BANKS		64
diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c
@@ -9,6 +9,8 @@
 
 #define ADF_GEN4_TL_DEV_REG_OFF(reg) ADF_TL_DEV_REG_OFF(reg, gen4)
 
+#define ADF_GEN4_TL_RP_REG_OFF(reg) ADF_TL_RP_REG_OFF(reg, gen4)
+
 #define ADF_GEN4_TL_SL_UTIL_COUNTER(_name)	\
 	ADF_TL_COUNTER("util_" #_name,		\
 			ADF_TL_SIMPLE_COUNT,	\
@@ -101,11 +103,42 @@ static const struct adf_tl_dbg_counter sl_exec_counters[ADF_TL_SL_CNT_COUNT] = {
 	ADF_GEN4_TL_SL_EXEC_COUNTER(ath),
 };
 
+/* Ring pair counters. */
+static const struct adf_tl_dbg_counter rp_counters[] = {
+	/* PCIe partial transactions. */
+	ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT,
+		       ADF_GEN4_TL_RP_REG_OFF(reg_tl_pci_trans_cnt)),
+	/* Get to put latency average[ns]. */
+	ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG,
+			       ADF_GEN4_TL_RP_REG_OFF(reg_tl_gp_lat_acc),
+			       ADF_GEN4_TL_RP_REG_OFF(reg_tl_ae_put_cnt)),
+	/* PCIe write bandwidth[Mbps]. */
+	ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS,
+		       ADF_GEN4_TL_RP_REG_OFF(reg_tl_bw_in)),
+	/* PCIe read bandwidth[Mbps]. */
+	ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS,
+		       ADF_GEN4_TL_RP_REG_OFF(reg_tl_bw_out)),
+	/* Message descriptor DevTLB hit rate. */
+	ADF_TL_COUNTER(AT_GLOB_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT,
+		       ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_hit)),
+	/* Message descriptor DevTLB miss rate. */
+	ADF_TL_COUNTER(AT_GLOB_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT,
+		       ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_miss)),
+	/* Payload DevTLB hit rate. */
+	ADF_TL_COUNTER(AT_PAYLD_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT,
+		       ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_hit)),
+	/* Payload DevTLB miss rate. */
+	ADF_TL_COUNTER(AT_PAYLD_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT,
+		       ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_miss)),
+};
+
 void adf_gen4_init_tl_data(struct adf_tl_hw_data *tl_data)
 {
 	tl_data->layout_sz = ADF_GEN4_TL_LAYOUT_SZ;
 	tl_data->slice_reg_sz = ADF_GEN4_TL_SLICE_REG_SZ;
+	tl_data->rp_reg_sz = ADF_GEN4_TL_RP_REG_SZ;
 	tl_data->num_hbuff = ADF_GEN4_TL_NUM_HIST_BUFFS;
+	tl_data->max_rp = ADF_GEN4_TL_MAX_RP_NUM;
 	tl_data->msg_cnt_off = ADF_GEN4_TL_MSG_CNT_OFF;
 	tl_data->cpp_ns_per_cycle = ADF_GEN4_CPP_NS_PER_CYCLE;
 	tl_data->bw_units_to_bytes = ADF_GEN4_TL_BW_HW_UNITS_TO_BYTES;
@@ -114,5 +147,7 @@ void adf_gen4_init_tl_data(struct adf_tl_hw_data *tl_data)
 	tl_data->num_dev_counters = ARRAY_SIZE(dev_counters);
 	tl_data->sl_util_counters = sl_util_counters;
 	tl_data->sl_exec_counters = sl_exec_counters;
+	tl_data->rp_counters = rp_counters;
+	tl_data->num_rp_counters = ARRAY_SIZE(rp_counters);
 }
 EXPORT_SYMBOL_GPL(adf_gen4_init_tl_data);
diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h
@@ -21,6 +21,9 @@ struct adf_tl_hw_data;
 /* Max number of HW resources of one type. */
 #define ADF_GEN4_TL_MAX_SLICES_PER_TYPE		24
 
+/* Max number of simultaneously monitored ring pairs. */
+#define ADF_GEN4_TL_MAX_RP_NUM			4
+
 /**
  * struct adf_gen4_tl_slice_data_regs - HW slice data as populated by FW.
  * @reg_tm_slice_exec_cnt: Slice execution count.
@@ -92,18 +95,52 @@ struct adf_gen4_tl_device_data_regs {
 	struct adf_gen4_tl_slice_data_regs wcp_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE];
 };
 
+/**
+ * struct adf_gen4_tl_ring_pair_data_regs - This structure stores Ring Pair
+ * telemetry counter values as are being populated periodically by device.
+ * @reg_tl_gp_lat_acc: get-put latency accumulator
+ * @reserved: reserved
+ * @reg_tl_pci_trans_cnt: PCIe partial transactions
+ * @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings
+ * @reg_tl_bw_in: PCIe write bandwidth
+ * @reg_tl_bw_out: PCIe read bandwidth
+ * @reg_tl_at_glob_devtlb_hit: Message descriptor DevTLB hit rate
+ * @reg_tl_at_glob_devtlb_miss: Message descriptor DevTLB miss rate
+ * @reg_tl_at_payld_devtlb_hit: Payload DevTLB hit rate
+ * @reg_tl_at_payld_devtlb_miss: Payload DevTLB miss rate
+ * @reg_tl_re_cnt: ring empty time samples count
+ * @reserved1: reserved
+ */
+struct adf_gen4_tl_ring_pair_data_regs {
+	__u64 reg_tl_gp_lat_acc;
+	__u64 reserved;
+	__u32 reg_tl_pci_trans_cnt;
+	__u32 reg_tl_ae_put_cnt;
+	__u32 reg_tl_bw_in;
+	__u32 reg_tl_bw_out;
+	__u32 reg_tl_at_glob_devtlb_hit;
+	__u32 reg_tl_at_glob_devtlb_miss;
+	__u32 reg_tl_at_payld_devtlb_hit;
+	__u32 reg_tl_at_payld_devtlb_miss;
+	__u32 reg_tl_re_cnt;
+	__u32 reserved1;
+};
+
+#define ADF_GEN4_TL_RP_REG_SZ sizeof(struct adf_gen4_tl_ring_pair_data_regs)
+
 /**
  * struct adf_gen4_tl_layout - This structure represents entire telemetry
  * counters data: Device + 4 Ring Pairs as are being populated periodically
  * by device.
  * @tl_device_data_regs: structure of device telemetry registers
- * @reserved1: reserved
+ * @tl_ring_pairs_data_regs: array of ring pairs telemetry registers
  * @reg_tl_msg_cnt: telemetry messages counter
  * @reserved: reserved
  */
 struct adf_gen4_tl_layout {
 	struct adf_gen4_tl_device_data_regs tl_device_data_regs;
-	__u32 reserved1[14];
+	struct adf_gen4_tl_ring_pair_data_regs
+			tl_ring_pairs_data_regs[ADF_GEN4_TL_MAX_RP_NUM];
 	__u32 reg_tl_msg_cnt;
 	__u32 reserved;
 };
diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.c b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c
@@ -33,7 +33,9 @@ static int validate_tl_data(struct adf_tl_hw_data *tl_data)
 	if (!tl_data->dev_counters ||
 	    TL_IS_ZERO(tl_data->num_dev_counters) ||
 	    !tl_data->sl_util_counters ||
-	    !tl_data->sl_exec_counters)
+	    !tl_data->sl_exec_counters ||
+	    !tl_data->rp_counters ||
+	    TL_IS_ZERO(tl_data->num_rp_counters))
 		return -EOPNOTSUPP;
 
 	return 0;
@@ -53,11 +55,17 @@ static int adf_tl_alloc_mem(struct adf_accel_dev *accel_dev)
 	if (!telemetry)
 		return -ENOMEM;
 
+	telemetry->rp_num_indexes = kmalloc_array(tl_data->max_rp,
+						  sizeof(*telemetry->rp_num_indexes),
+						  GFP_KERNEL);
+	if (!telemetry->rp_num_indexes)
+		goto err_free_tl;
+
 	telemetry->regs_hist_buff = kmalloc_array(tl_data->num_hbuff,
 						  sizeof(*telemetry->regs_hist_buff),
 						  GFP_KERNEL);
 	if (!telemetry->regs_hist_buff)
-		goto err_free_tl;
+		goto err_free_rp_indexes;
 
 	telemetry->regs_data = dma_alloc_coherent(dev, regs_sz,
 						  &telemetry->regs_data_p,
@@ -86,6 +94,8 @@ static int adf_tl_alloc_mem(struct adf_accel_dev *accel_dev)
 
 err_free_regs_hist_buff:
 	kfree(telemetry->regs_hist_buff);
+err_free_rp_indexes:
+	kfree(telemetry->rp_num_indexes);
 err_free_tl:
 	kfree(telemetry);
 
@@ -107,6 +117,7 @@ static void adf_tl_free_mem(struct adf_accel_dev *accel_dev)
 			  telemetry->regs_data_p);
 
 	kfree(telemetry->regs_hist_buff);
+	kfree(telemetry->rp_num_indexes);
 	kfree(telemetry);
 	accel_dev->telemetry = NULL;
 }
@@ -196,7 +207,8 @@ int adf_tl_run(struct adf_accel_dev *accel_dev, int state)
 	int ret;
 
 	ret = adf_send_admin_tl_start(accel_dev, telemetry->regs_data_p,
-				      layout_sz, NULL, &telemetry->slice_cnt);
+				      layout_sz, telemetry->rp_num_indexes,
+				      &telemetry->slice_cnt);
 	if (ret) {
 		dev_err(dev, "failed to start telemetry\n");
 		return ret;
@@ -213,8 +225,10 @@ int adf_tl_run(struct adf_accel_dev *accel_dev, int state)
 int adf_tl_init(struct adf_accel_dev *accel_dev)
 {
 	struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev);
+	u8 max_rp = GET_TL_DATA(accel_dev).max_rp;
 	struct device *dev = &GET_DEV(accel_dev);
 	struct adf_telemetry *telemetry;
+	unsigned int i;
 	int ret;
 
 	ret = validate_tl_data(tl_data);
@@ -234,6 +248,9 @@ int adf_tl_init(struct adf_accel_dev *accel_dev)
 	mutex_init(&telemetry->regs_hist_lock);
 	INIT_DELAYED_WORK(&telemetry->work_ctx, tl_work_handler);
 
+	for (i = 0; i < max_rp; i++)
+		telemetry->rp_num_indexes[i] = ADF_TL_RP_REGS_DISABLED;
+
 	return 0;
 }
 
diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.h b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h
@@ -23,17 +23,23 @@ struct dentry;
 /* Interval within timer interrupt should be handled. Value in milliseconds. */
 #define ADF_TL_TIMER_INT_MS		(ADF_TL_DATA_WR_INTERVAL_MS / 2)
 
+#define ADF_TL_RP_REGS_DISABLED		(0xff)
+
 struct adf_tl_hw_data {
 	size_t layout_sz;
 	size_t slice_reg_sz;
+	size_t rp_reg_sz;
 	size_t msg_cnt_off;
 	const struct adf_tl_dbg_counter *dev_counters;
 	const struct adf_tl_dbg_counter *sl_util_counters;
 	const struct adf_tl_dbg_counter *sl_exec_counters;
+	const struct adf_tl_dbg_counter *rp_counters;
 	u8 num_hbuff;
 	u8 cpp_ns_per_cycle;
 	u8 bw_units_to_bytes;
 	u8 num_dev_counters;
+	u8 num_rp_counters;
+	u8 max_rp;
 };
 
 struct adf_telemetry {
@@ -50,6 +56,7 @@ struct adf_telemetry {
 	 */
 	void **regs_hist_buff;
 	struct dentry *dbg_dir;
+	u8 *rp_num_indexes;
 	/**
 	 * @regs_hist_lock: protects from race conditions between write and read
 	 * to the copies referenced by @regs_hist_buff
diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c
diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h
