ALSA: usb-audio: Cap upper limits of buffer/period bytes for implicit fb

In the implicit feedback mode, some parameters are tied between both
playback and capture streams.  One of the tied parameters is the
period size, and this can be a problem if the device has different
number of channels to both streams.  Assume that an application opens
a playback stream that has an implicit feedback from a capture stream,
and it allocates up to the max period and buffer size as much as
possible.  When the capture device supports only more channels than
the playback, the minimum period and buffer sizes become larger than
the sizes the playback stream took.  That is, the minimum size will be
over the max size the driver limits, and PCM core sees as if no
available configuration is found, returning -EINVAL mercilessly.

For avoiding this problem, we have to look through the counter part of
audioformat list for each sync ep, and checks the channels.  If more
channels are found there, we reduce the max period and buffer sizes
accordingly.

You may wonder that the patch adds only the evaluation of channels
between streams, and what about other parameters?  Both the format and
the rate are tied in the implicit fb mode, hence they are always
identical.

BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=215792
Fixes: 5a6c3e1 ("ALSA: usb-audio: Add hw constraint for implicit fb sync")
Cc: <stable@vger.kernel.org>
Link: https://lore.kernel.org/r/20220407211657.15087-1-tiwai@suse.de
Signed-off-by: Takashi Iwai <tiwai@suse.de>

Author: tiwai

sound/usb/pcm.c

@@ -659,6 +659,9 @@ static int snd_usb_pcm_prepare(struct snd_pcm_substream *substream)
 #define hwc_debug(fmt, args...) do { } while(0)
 #endif
 
+#define MAX_BUFFER_BYTES	(1024 * 1024)
+#define MAX_PERIOD_BYTES	(512 * 1024)
+
 static const struct snd_pcm_hardware snd_usb_hardware =
 {
 	.info =			SNDRV_PCM_INFO_MMAP |
@@ -669,9 +672,9 @@ static const struct snd_pcm_hardware snd_usb_hardware =
 				SNDRV_PCM_INFO_PAUSE,
 	.channels_min =		1,
 	.channels_max =		256,
-	.buffer_bytes_max =	1024 * 1024,
+	.buffer_bytes_max =	MAX_BUFFER_BYTES,
 	.period_bytes_min =	64,
-	.period_bytes_max =	512 * 1024,
+	.period_bytes_max =	MAX_PERIOD_BYTES,
 	.periods_min =		2,
 	.periods_max =		1024,
 };
@@ -971,6 +974,78 @@ static int hw_rule_periods_implicit_fb(struct snd_pcm_hw_params *params,
 				      ep->cur_buffer_periods);
 }
 
+/* get the adjusted max buffer (or period) bytes that can fit with the
+ * paired format for implicit fb
+ */
+static unsigned int
+get_adjusted_max_bytes(struct snd_usb_substream *subs,
+		       struct snd_usb_substream *pair,
+		       struct snd_pcm_hw_params *params,
+		       unsigned int max_bytes,
+		       bool reverse_map)
+{
+	const struct audioformat *fp, *pp;
+	unsigned int rmax = 0, r;
+
+	list_for_each_entry(fp, &subs->fmt_list, list) {
+		if (!fp->implicit_fb)
+			continue;
+		if (!reverse_map &&
+		    !hw_check_valid_format(subs, params, fp))
+			continue;
+		list_for_each_entry(pp, &pair->fmt_list, list) {
+			if (pp->iface != fp->sync_iface ||
+			    pp->altsetting != fp->sync_altsetting ||
+			    pp->ep_idx != fp->sync_ep_idx)
+				continue;
+			if (reverse_map &&
+			    !hw_check_valid_format(pair, params, pp))
+				break;
+			if (!reverse_map && pp->channels > fp->channels)
+				r = max_bytes * fp->channels / pp->channels;
+			else if (reverse_map && pp->channels < fp->channels)
+				r = max_bytes * pp->channels / fp->channels;
+			else
+				r = max_bytes;
+			rmax = max(rmax, r);
+			break;
+		}
+	}
+	return rmax;
+}
+
+/* Reduce the period or buffer bytes depending on the paired substream;
+ * when a paired configuration for implicit fb has a higher number of channels,
+ * we need to reduce the max size accordingly, otherwise it may become unusable
+ */
+static int hw_rule_bytes_implicit_fb(struct snd_pcm_hw_params *params,
+				     struct snd_pcm_hw_rule *rule)
+{
+	struct snd_usb_substream *subs = rule->private;
+	struct snd_usb_substream *pair;
+	struct snd_interval *it;
+	unsigned int max_bytes;
+	unsigned int rmax;
+
+	pair = &subs->stream->substream[!subs->direction];
+	if (!pair->ep_num)
+		return 0;
+
+	if (rule->var == SNDRV_PCM_HW_PARAM_PERIOD_BYTES)
+		max_bytes = MAX_PERIOD_BYTES;
+	else
+		max_bytes = MAX_BUFFER_BYTES;
+
+	rmax = get_adjusted_max_bytes(subs, pair, params, max_bytes, false);
+	if (!rmax)
+		rmax = get_adjusted_max_bytes(pair, subs, params, max_bytes, true);
+	if (!rmax)
+		return 0;
+
+	it = hw_param_interval(params, rule->var);
+	return apply_hw_params_minmax(it, 0, rmax);
+}
+
 /*
  * set up the runtime hardware information.
  */
@@ -1085,6 +1160,16 @@ static int setup_hw_info(struct snd_pcm_runtime *runtime, struct snd_usb_substre
 				  SNDRV_PCM_HW_PARAM_PERIODS, -1);
 	if (err < 0)
 		return err;
+	err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_BUFFER_BYTES,
+				  hw_rule_bytes_implicit_fb, subs,
+				  SNDRV_PCM_HW_PARAM_BUFFER_BYTES, -1);
+	if (err < 0)
+		return err;
+	err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_PERIOD_BYTES,
+				  hw_rule_bytes_implicit_fb, subs,
+				  SNDRV_PCM_HW_PARAM_PERIOD_BYTES, -1);
+	if (err < 0)
+		return err;
 
 	list_for_each_entry(fp, &subs->fmt_list, list) {
 		if (fp->implicit_fb) {