Skip to content

Mistral: per-key rate limiting, retry/backoff and improved error handling for streaming requests #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Android-PowerUser merged 4 commits into from
Mar 30, 2026
+117 −47
Merged

Mistral: per-key rate limiting, retry/backoff and improved error handling for streaming requests #65

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
61d16f6
Adjust Mistral retry scheduling for per-key 1.1s cooldown
Android-PowerUser Mar 30, 2026
3cbeac1
Update app/src/main/kotlin/com/google/ai/sample/feature/multimodal/Ph…
Android-PowerUser Mar 30, 2026
e89c450
Update app/src/main/kotlin/com/google/ai/sample/feature/multimodal/Ph…
Android-PowerUser Mar 30, 2026
d139fcc
[skip ci] Fix Mistral API retry logic and error handling
amazon-q-developer[bot] Mar 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 117 additions & 47 deletions app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ import kotlinx.serialization.modules.subclass
import com.google.ai.sample.webrtc.WebRTCSender
import com.google.ai.sample.webrtc.SignalingClient
import org.webrtc.IceCandidate
import kotlin.math.max

class PhotoReasoningViewModel(
application: Application,
Expand Down Expand Up @@ -182,9 +183,11 @@ class PhotoReasoningViewModel(
// to avoid re-executing already-executed commands
private var incrementalCommandCount = 0

// Mistral rate limiting: track last request time to enforce 1-second minimum interval
private var lastMistralRequestTimeMs = 0L
private val MISTRAL_MIN_INTERVAL_MS = 1000L
// Mistral rate limiting per API key (1.1 seconds between requests with same key)
private val mistralNextAllowedRequestAtMsByKey = mutableMapOf<String, Long>()
private var lastMistralTokenTimeMs = 0L
private var lastMistralTokenKey: String? = null
private val MISTRAL_MIN_INTERVAL_MS = 1100L

// Accumulated full text during streaming for incremental command parsing
private var streamingAccumulatedText = StringBuilder()
Expand Down Expand Up @@ -1052,12 +1055,6 @@ private fun reasonWithMistral(
resetStreamingCommandState()

viewModelScope.launch(Dispatchers.IO) {
// Rate limiting: nur die verbleibende Zeit warten
val elapsed = System.currentTimeMillis() - lastMistralRequestTimeMs
if (lastMistralRequestTimeMs > 0 && elapsed < MISTRAL_MIN_INTERVAL_MS) {
delay(MISTRAL_MIN_INTERVAL_MS - elapsed)
}

try {
val currentModel = com.google.ai.sample.GenerativeAiViewModelFactory.getCurrentModel()
val genSettings = com.google.ai.sample.util.GenerationSettingsPreferences.loadSettings(context, currentModel.modelName)
Expand Down Expand Up @@ -1126,60 +1123,133 @@ private fun reasonWithMistral(
.addHeader("Authorization", "Bearer $key")
.build()

var currentKey = initialApiKey
var response = client.newCall(buildRequest(currentKey)).execute()
lastMistralRequestTimeMs = System.currentTimeMillis()
val availableKeys = apiKeyManager.getApiKeys(ApiProvider.MISTRAL)
.filter { it.isNotBlank() }
.distinct()
if (availableKeys.isEmpty()) {
throw IOException("Mistral API key not found.")
}

if (response.code == 429) {
response.close()
apiKeyManager.markKeyAsFailed(currentKey, ApiProvider.MISTRAL)
val nextKey = apiKeyManager.switchToNextAvailableKey(ApiProvider.MISTRAL)
if (nextKey != null && nextKey != currentKey) {
// Anderer Key verfugbar -> sofort wechseln wie bisher
currentKey = nextKey
val elapsed2 = System.currentTimeMillis() - lastMistralRequestTimeMs
if (elapsed2 < MISTRAL_MIN_INTERVAL_MS) delay(MISTRAL_MIN_INTERVAL_MS - elapsed2)
response = client.newCall(buildRequest(currentKey)).execute()
lastMistralRequestTimeMs = System.currentTimeMillis()
} else {
// Kein anderer Key -> 5 Sekunden lang sofort wiederholen
apiKeyManager.resetFailedKeys(ApiProvider.MISTRAL)
// Validate that we have at least one key before proceeding
require(availableKeys.isNotEmpty()) { "No valid Mistral API keys available after filtering" }

fun markKeyCooldown(key: String, referenceTimeMs: Long) {
val nextAllowedAt = referenceTimeMs + MISTRAL_MIN_INTERVAL_MS
val existing = mistralNextAllowedRequestAtMsByKey[key] ?: 0L
mistralNextAllowedRequestAtMsByKey[key] = max(existing, nextAllowedAt)
}

fun remainingWaitForKeyMs(key: String, nowMs: Long): Long {
val nextAllowedAt = mistralNextAllowedRequestAtMsByKey[key] ?: 0L
return (nextAllowedAt - nowMs).coerceAtLeast(0L)
}

fun isRetryableMistralFailure(code: Int): Boolean {
return code == 429 || code >= 500
}

var response: okhttp3.Response? = null
var selectedKeyForResponse: String? = null
var consecutiveFailures = 0
var blockedKeysThisRound = mutableSetOf<String>()

val maxAttempts = availableKeys.size * 2 + 3 // Allow cycling through all keys at least twice
while (response == null && consecutiveFailures < maxAttempts) {
if (stopExecutionFlag.get()) break

val now = System.currentTimeMillis()
val keyPool = availableKeys.filter { it !in blockedKeysThisRound }.ifEmpty {
blockedKeysThisRound.clear()
availableKeys
}

val keyWithLeastWait = keyPool.minByOrNull { remainingWaitForKeyMs(it, now) } ?: availableKeys.first()
val waitMs = remainingWaitForKeyMs(keyWithLeastWait, now)
if (waitMs > 0L) {
delay(waitMs)
}

val selectedKey = keyWithLeastWait
selectedKeyForResponse = selectedKey

try {
val attemptResponse = client.newCall(buildRequest(selectedKey)).execute()
val requestEndMs = System.currentTimeMillis()
markKeyCooldown(selectedKey, requestEndMs)

if (attemptResponse.isSuccessful) {
response = attemptResponse
break
}

val isRetryable = isRetryableMistralFailure(attemptResponse.code)
if (!isRetryable) {
val errBody = attemptResponse.body?.string()
attemptResponse.close()
throw IllegalStateException("Mistral Error ${attemptResponse.code}: $errBody")
}

attemptResponse.close()
blockedKeysThisRound.add(selectedKey)
consecutiveFailures++
withContext(Dispatchers.Main) {
replaceAiMessageText("Rate limit erreicht. Wiederhole...", isPending = true)
replaceAiMessageText(
"Mistral temporär nicht verfügbar (Versuch $consecutiveFailures/$maxAttempts). Wiederhole...",
isPending = true
)
}
val retryDeadline = System.currentTimeMillis() + 5000L
var retryResponse: okhttp3.Response? = null
while (System.currentTimeMillis() < retryDeadline) {
if (stopExecutionFlag.get()) break
val retryResp = client.newCall(buildRequest(currentKey)).execute()
lastMistralRequestTimeMs = System.currentTimeMillis()
if (retryResp.code != 429) {
retryResponse = retryResp
break
}
retryResp.close()
} catch (e: IOException) {
val requestEndMs = System.currentTimeMillis()
markKeyCooldown(selectedKey, requestEndMs)
blockedKeysThisRound.add(selectedKey)
consecutiveFailures++
if (consecutiveFailures >= 5) {
throw IOException("Mistral request failed after 5 attempts: ${e.message}", e)
}
if (retryResponse == null || stopExecutionFlag.get()) {
throw IOException("Mistral rate limit: Kein Erfolg innerhalb von 5 Sekunden.")
withContext(Dispatchers.Main) {
replaceAiMessageText(
if (consecutiveFailures >= maxAttempts) {
throw IOException("Mistral request failed after $maxAttempts attempts: ${e.message}", e)
)
}
response = retryResponse
}
Comment on lines +1201 to 1215
Copy link
Copy Markdown

@amazon-q-developer amazon-q-developer bot Mar 30, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛑 Logic Error: After closing a failed response, the code continues to use selectedKey in error handling without checking if the request actually failed due to network issues versus retryable API errors. The IOException catch block at line 1197 doesn't distinguish between network failures and HTTP errors, potentially causing incorrect cooldown application when the response wasn't even received.

"Mistral Netzwerkfehler (Versuch $consecutiveFailures/$maxAttempts). Wiederhole...",

if (stopExecutionFlag.get()) {
throw IOException("Mistral request aborted.")
}

if (!response.isSuccessful) {
val errBody = response.body?.string()
response.close()
throw IOException("Mistral Error ${response.code}: $errBody")
val finalResponse = response ?: throw IOException("Mistral request failed after 5 attempts.")

if (!finalResponse.isSuccessful) {
val errBody = finalResponse.body?.string()
finalResponse.close()
val finalResponse = response ?: throw IOException("Mistral request failed after $maxAttempts attempts.")
}

val body = response.body ?: throw IOException("Empty response body from Mistral")
val body = finalResponse.body ?: throw IOException("Empty response body from Mistral")
val aiResponseText = openAiStreamParser.parse(body) { accText ->
selectedKeyForResponse?.let { key ->
lastMistralTokenKey = key
lastMistralTokenTimeMs = System.currentTimeMillis()
markKeyCooldown(key, lastMistralTokenTimeMs)
} ?: run {
Log.w(TAG, "selectedKeyForResponse is null during streaming callback")
}
withContext(Dispatchers.Main) {
replaceAiMessageText(accText, isPending = true)
processCommandsIncrementally(accText)
}
}
response.close()
finalResponse.close()
selectedKeyForResponse?.let { key ->
val reference = if (lastMistralTokenKey == key && lastMistralTokenTimeMs > 0L) {
lastMistralTokenTimeMs
} else {
System.currentTimeMillis()
}
markKeyCooldown(key, reference)
}

withContext(Dispatchers.Main) {
_uiState.value = PhotoReasoningUiState.Success(aiResponseText)
Expand Down