Add CSV file batch-import support

smaugfm · May 23, 2022 · 3a7da7a · 3a7da7a
1 parent 8673b98
commit 3a7da7a
Show file tree

Hide file tree

Showing 22 changed files with 383 additions and 53 deletions.
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -96,6 +96,7 @@ dependencies {
     implementation("com.github.elbekD:kt-telegram-bot:1.4.1")
     implementation("com.github.ajalt.clikt:clikt:3.4.0")
     implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.2")
+    implementation("de.brudaswen.kotlinx.serialization:kotlinx-serialization-csv:1.1.0")
     implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.3.1")
     implementation("io.ktor:ktor-server-core:$ktor")
     implementation("io.ktor:ktor-server-netty:$ktor")

diff --git a/detekt.yml b/detekt.yml
@@ -14,26 +14,28 @@ config:
   excludes: ''
 
 exceptions:
+  SwallowedException:
+    ignoredExceptionTypes: [ 'YnabRateLimitException' ]
   TooGenericExceptionCaught:
     active: false
 
 naming:
   ConstructorParameterNaming:
     active: true
-    excludes: ['**/models.kt']
+    excludes: [ '**/models.kt' ]
 
 complexity:
   LargeClass:
     active: true
-    excludes: ['**/MCC.kt']
+    excludes: [ '**/MCC.kt' ]
 
 style:
   MagicNumber:
     active: true
-    excludes: ['**/MCC.kt', '**Test.kt']
+    excludes: [ '**/MCC.kt', '**Test.kt' ]
   MaxLineLength:
     active: true
-    excludes: ['**/MCC.kt']
+    excludes: [ '**/MCC.kt' ]
   ReturnCount:
     active: true
-    max: 3
+    max: 3
diff --git a/src/main/kotlin/com/github/smaugfm/YnabMonoApp.kt b/src/main/kotlin/com/github/smaugfm/YnabMonoApp.kt
@@ -4,8 +4,10 @@ import com.github.smaugfm.apis.MonoApis
 import com.github.smaugfm.apis.TelegramApi
 import com.github.smaugfm.workflows.CreateTransaction
 import com.github.smaugfm.workflows.HandleCallback
+import com.github.smaugfm.workflows.HandleCsv
 import com.github.smaugfm.workflows.ProcessError
-import com.github.smaugfm.workflows.SendMessage
+import com.github.smaugfm.workflows.RetryWithRateLimit
+import com.github.smaugfm.workflows.SendTransactionCreatedMessage
 import io.ktor.util.error
 import mu.KotlinLogging
 import org.koin.core.component.KoinComponent
@@ -19,8 +21,10 @@ class YnabMonoApp : KoinComponent {
     val monoApis by inject<MonoApis>()
 
     val createTransaction by inject<CreateTransaction>()
-    val sendMessage by inject<SendMessage>()
+    val retryWithRateLimit by inject<RetryWithRateLimit>()
+    val sendTransactionCreatedMessage by inject<SendTransactionCreatedMessage>()
     val handleCallback by inject<HandleCallback>()
+    val handleCsv by inject<HandleCsv>()
     val processError by inject<ProcessError>()
 
     suspend fun run(setWebhook: Boolean, monoWebhookUrl: URI, webhookPort: Int) {
@@ -35,17 +39,21 @@ class YnabMonoApp : KoinComponent {
         val webhookJob = monoApis.listenWebhooks(
             monoWebhookUrl,
             webhookPort,
-        ) handler@{
+        ) { responseData ->
             try {
-                val newTransaction = createTransaction(it) ?: return@handler
-                sendMessage(it, newTransaction)
+                retryWithRateLimit(responseData.account) retry@{
+                    val newTransaction = createTransaction(responseData) ?: return@retry
+                    sendTransactionCreatedMessage(responseData, newTransaction)
+                }
             } catch (e: Throwable) {
                 logger.error(e)
                 processError()
             }
         }
-        val telegramJob = telegramApi.listenForCallbacks {
-            handleCallback(it)
+        val telegramJob = telegramApi.start(
+            handleCallback::invoke,
+        ) { chatId, file ->
+            handleCsv(chatId, file)
         }
         logger.info { "Listening for Monobank webhooks and Telegram callbacks..." }
         webhookJob.join()

diff --git a/src/main/kotlin/com/github/smaugfm/YnabMonoCommand.kt b/src/main/kotlin/com/github/smaugfm/YnabMonoCommand.kt
@@ -14,8 +14,11 @@ import com.github.smaugfm.models.settings.Settings
 import com.github.smaugfm.util.DEFAULT_HTTP_PORT
 import com.github.smaugfm.workflows.CreateTransaction
 import com.github.smaugfm.workflows.HandleCallback
+import com.github.smaugfm.workflows.HandleCsv
 import com.github.smaugfm.workflows.ProcessError
-import com.github.smaugfm.workflows.SendMessage
+import com.github.smaugfm.workflows.RetryWithRateLimit
+import com.github.smaugfm.workflows.SendHTMLMessageToTelegram
+import com.github.smaugfm.workflows.SendTransactionCreatedMessage
 import com.github.smaugfm.workflows.TransformStatementToYnabTransaction
 import kotlinx.coroutines.runBlocking
 import mu.KotlinLogging
@@ -60,9 +63,12 @@ class YnabMonoCommand : CliktCommand() {
                             )
                         }
                         single { CreateTransaction(get(), get(), get()) }
-                        single { SendMessage(get(), get()) }
+                        single { SendHTMLMessageToTelegram(get(), get()) }
+                        single { RetryWithRateLimit(get()) }
+                        single { SendTransactionCreatedMessage(get(), get()) }
                         single { ProcessError(get(), get()) }
-                        single { HandleCallback(get(), get(), get()) }
+                        single { HandleCallback(get(), get(), get(), get()) }
+                        single { HandleCsv(get(), get(), get()) }
                     }
                 )
             }

diff --git a/src/main/kotlin/com/github/smaugfm/apis/MonoApi.kt b/src/main/kotlin/com/github/smaugfm/apis/MonoApi.kt
@@ -7,8 +7,8 @@ import com.github.smaugfm.models.MonoStatementItem
 import com.github.smaugfm.models.MonoStatusResponse
 import com.github.smaugfm.models.MonoUserInfo
 import com.github.smaugfm.models.MonoWebHookRequest
+import com.github.smaugfm.util.logError
 import com.github.smaugfm.util.makeJson
-import com.github.smaugfm.util.requestCatching
 import io.ktor.application.call
 import io.ktor.client.HttpClient
 import io.ktor.client.features.defaultRequest
@@ -57,7 +57,9 @@ class MonoApi(private val token: String) {
     private suspend inline fun <reified T : Any> catching(
         method: KFunction<Any>,
         block: () -> T,
-    ): T = requestCatching<T, MonoErrorResponse>("Monobank", logger, method.name, json, block)
+    ): T = logError<T, MonoErrorResponse>("Monobank", logger, method.name, json, block) {
+        // do nothing
+    }
 
     @Suppress("unused")
     suspend fun fetchUserInfo(): MonoUserInfo =

diff --git a/src/main/kotlin/com/github/smaugfm/apis/TelegramApi.kt b/src/main/kotlin/com/github/smaugfm/apis/TelegramApi.kt
@@ -1,6 +1,7 @@
 package com.github.smaugfm.apis
 
 import com.elbekD.bot.Bot
+import com.elbekD.bot.http.await
 import com.elbekD.bot.types.CallbackQuery
 import com.elbekD.bot.types.InlineKeyboardMarkup
 import com.elbekD.bot.types.ReplyKeyboard
@@ -12,13 +13,15 @@ import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.future.asDeferred
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
 import mu.KotlinLogging
+import java.net.URL
 
 private val logger = KotlinLogging.logger {}
 
 class TelegramApi(
     private val scope: CoroutineScope,
-    settings: Settings
+    private val settings: Settings
 ) {
     private val bot: Bot =
         Bot.createPolling(settings.telegramBotUsername, settings.telegramBotToken)
@@ -77,10 +80,29 @@ class TelegramApi(
     }
 
     @OptIn(ExperimentalCoroutinesApi::class)
-    fun listenForCallbacks(callback: suspend (CallbackQuery) -> Unit): Job {
+    fun start(
+        callbackHandler: suspend (CallbackQuery) -> Unit,
+        csvFileHandler: suspend (Long, String) -> Unit,
+    ): Job {
         bot.onCallbackQuery {
             logger.debug { "Received callbackQuery.\n\t$it" }
-            callback(it)
+            callbackHandler(it)
+        }
+        bot.onMessage { msg ->
+            val doc = msg.document
+            if (doc != null && doc.file_name?.endsWith(".csv") == true) {
+                val file = bot.getFile(doc.file_id).await()
+                val url = URL(
+                    "https://api.telegram.org/" +
+                        "file/${settings.telegramBotToken}/${file.file_path}"
+                )
+                csvFileHandler(
+                    msg.chat.id,
+                    withContext(Dispatchers.IO) {
+                        url.openStream().readAllBytes().toString(Charsets.UTF_8)
+                    }
+                )
+            }
         }
         return scope.launch(context = Dispatchers.IO) {
             bot.start()

diff --git a/src/main/kotlin/com/github/smaugfm/apis/YnabApi.kt b/src/main/kotlin/com/github/smaugfm/apis/YnabApi.kt
@@ -16,8 +16,9 @@ import com.github.smaugfm.models.YnabTransactionResponse
 import com.github.smaugfm.models.YnabTransactionResponseWithServerKnowledge
 import com.github.smaugfm.models.YnabTransactionsResponse
 import com.github.smaugfm.models.settings.Settings
+import com.github.smaugfm.util.YnabRateLimitException
+import com.github.smaugfm.util.logError
 import com.github.smaugfm.util.makeJson
-import com.github.smaugfm.util.requestCatching
 import io.ktor.client.HttpClient
 import io.ktor.client.features.json.JsonFeature
 import io.ktor.client.features.json.serializer.KotlinxSerializer
@@ -60,12 +61,20 @@ class YnabApi(settings: Settings) {
     private suspend inline fun <reified T : Any> catching(
         method: KFunction<Any>,
         block: () -> T,
-    ): T = requestCatching<T, YnabErrorResponse>("YNAB", logger, method.name, json, block)
+    ): T = logError<T, YnabErrorResponse>("YNAB", logger, method.name, json, block) {
+        if (it.error.id == "429") {
+            throw YnabRateLimitException()
+        }
+    }
 
     private suspend inline fun <reified T : Any> catchingNoLogging(
         method: KFunction<Any>,
         block: () -> T,
-    ): T = requestCatching<T, YnabErrorResponse>("YNAB", null, method.name, json, block)
+    ): T = logError<T, YnabErrorResponse>("YNAB", null, method.name, json, block) {
+        if (it.error.id == "429") {
+            throw YnabRateLimitException()
+        }
+    }
 
     suspend fun getAccounts(): List<YnabAccount> =
         catching(this::getAccounts) {

diff --git a/src/main/kotlin/com/github/smaugfm/models/MonobankModels.kt b/src/main/kotlin/com/github/smaugfm/models/MonobankModels.kt
@@ -1,11 +1,14 @@
 package com.github.smaugfm.models
 
 import com.github.smaugfm.models.serializers.CurrencyAsIntSerializer
+import com.github.smaugfm.models.serializers.CurrencyAsStringSerializer
+import com.github.smaugfm.models.serializers.HumanReadableDateSerializer
 import com.github.smaugfm.models.serializers.InstantAsLongSerializer
-import com.github.smaugfm.util.IErrorFormattable
+import com.github.smaugfm.util.ErrorFormattable
 import kotlinx.datetime.Instant
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
+import java.time.LocalDateTime
 import java.util.Currency
 
 typealias MonoAccountId = String
@@ -51,7 +54,7 @@ data class MonoCurrencyInfo(
 @Serializable
 data class MonoErrorResponse(
     val errorDescription: String,
-) : IErrorFormattable {
+) : ErrorFormattable {
     override fun formatError() = errorDescription
 }
 
@@ -109,3 +112,19 @@ data class MonoWebHookResponseData(
     val account: MonoAccountId,
     val statementItem: MonoStatementItem,
 )
+
+@Serializable
+data class MonoExportCsvRow(
+    @Serializable(with = HumanReadableDateSerializer::class)
+    val time: LocalDateTime,
+    val description: String,
+    val mcc: Int,
+    val amount: Double,
+    val operationAmount: Double,
+    @Serializable(with = CurrencyAsStringSerializer::class)
+    val currency: Currency,
+    val exchangeRate: Double,
+    val comission: Double,
+    val cashbackAmount: Double,
+    val balance: Double,
+)
diff --git a/src/main/kotlin/com/github/smaugfm/models/YnabModels.kt b/src/main/kotlin/com/github/smaugfm/models/YnabModels.kt
@@ -3,7 +3,7 @@
 package com.github.smaugfm.models
 
 import com.github.smaugfm.models.serializers.LocalDateAsISOSerializer
-import com.github.smaugfm.util.IErrorFormattable
+import com.github.smaugfm.util.ErrorFormattable
 import kotlinx.datetime.LocalDate
 import kotlinx.serialization.Serializable
 
@@ -280,7 +280,7 @@ data class YnabSubTransaction(
 @Serializable
 data class YnabErrorResponse(
     val error: YnabErrorDetail,
-) : IErrorFormattable {
+) : ErrorFormattable {
     override fun formatError() = with(error) { "$name: $detail" }
 }
 

diff --git a/src/main/kotlin/com/github/smaugfm/models/serializers/HumanReadableDateSerializer.kt b/src/main/kotlin/com/github/smaugfm/models/serializers/HumanReadableDateSerializer.kt
@@ -0,0 +1,28 @@
+package com.github.smaugfm.models.serializers
+
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.descriptors.PrimitiveKind
+import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+
+class HumanReadableDateSerializer : KSerializer<LocalDateTime> {
+    private val formatter = DateTimeFormatter.ofPattern("dd.MM.yyyy HH:mm:ss")
+
+    override val descriptor = PrimitiveSerialDescriptor(
+        this::class.qualifiedName!!,
+        PrimitiveKind.LONG
+    )
+
+    override fun deserialize(decoder: Decoder): LocalDateTime =
+        LocalDateTime.parse(
+            decoder.decodeString(),
+            formatter
+        )
+
+    override fun serialize(encoder: Encoder, value: LocalDateTime) {
+        encoder.encodeString(formatter.format(value))
+    }
+}
diff --git a/src/main/kotlin/com/github/smaugfm/models/settings/Mappings.kt b/src/main/kotlin/com/github/smaugfm/models/settings/Mappings.kt
@@ -45,6 +45,12 @@ data class Mappings(
                 logger.error { "Could not find Telegram chatID for Monobank account $monoAcc" }
         }
 
+    fun getMonoAccIdByTelegramChatId(chatId: Long) =
+        monoAccToTelegram.inverse[chatId].also {
+            if (it == null)
+                logger.error { "Could not find Monobank account for Telegram chatID $chatId" }
+        }
+
     fun getMonoAccAlias(string: MonoAccountId): String? {
         return monoAccAliases[string].also {
             if (it == null)

diff --git a/.../github/smaugfm/util/IErrorFormattable.kt → ...m/github/smaugfm/util/ErrorFormattable.kt b/.../github/smaugfm/util/IErrorFormattable.kt → ...m/github/smaugfm/util/ErrorFormattable.kt
@@ -1,5 +1,5 @@
 package com.github.smaugfm.util
 
-interface IErrorFormattable {
+interface ErrorFormattable {
     fun formatError(): String
 }
diff --git a/src/main/kotlin/com/github/smaugfm/util/Flow.kt b/src/main/kotlin/com/github/smaugfm/util/Flow.kt
@@ -0,0 +1,32 @@
+package com.github.smaugfm.util
+
+import kotlinx.coroutines.InternalCoroutinesApi
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.collect
+import kotlinx.coroutines.flow.flow
+
+@OptIn(InternalCoroutinesApi::class)
+fun <T> Flow<T>.throttle(periodMillis: Long): Flow<T> {
+    require(periodMillis > 0) { "period should be positive" }
+    return flow {
+        var lastTime = 0L
+        collect {
+            val currentTime = System.currentTimeMillis()
+            if (currentTime - lastTime >= periodMillis) {
+                lastTime = currentTime
+                emit(it)
+            }
+        }
+    }
+}
+
+fun <T> Flow<T>.chunked(size: Int): Flow<List<T>> = flow {
+    val chunkedList = mutableListOf<T>()
+    collect {
+        chunkedList.add(it)
+        if (chunkedList.size == size) {
+            emit(chunkedList.toList())
+            chunkedList.clear()
+        }
+    }
+}