major refactor

build.gradle.kts

@@ -13,6 +13,7 @@ repositories {
 }
 
 dependencies {
+    implementation(kotlin("reflect"))
     implementation("com.squareup.okhttp3:okhttp:5.0.0-alpha.11")
     implementation("org.jsoup:jsoup:1.15.4")
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.0-Beta")

src/main/kotlin/io/github/yamin8000/twitterscrapper/Main.kt

@@ -1,45 +1,13 @@
 package io.github.yamin8000.twitterscrapper
 
-import com.github.ajalt.mordant.rendering.BorderType
-import com.github.ajalt.mordant.rendering.TextColors
-import com.github.ajalt.mordant.table.table
-import io.github.yamin8000.twitterscrapper.util.Constants.DOWNLOAD_FOLDER
-import io.github.yamin8000.twitterscrapper.util.Constants.DOWNLOAD_PATH
-import io.github.yamin8000.twitterscrapper.util.Constants.askStyle
-import io.github.yamin8000.twitterscrapper.util.Constants.infoStyle
-import io.github.yamin8000.twitterscrapper.util.Constants.mainMenu
-import io.github.yamin8000.twitterscrapper.util.Constants.menuStyle
-import io.github.yamin8000.twitterscrapper.util.Constants.t
+import io.github.yamin8000.twitterscrapper.modules.MainModule
+import io.github.yamin8000.twitterscrapper.modules.crawler.Crawler
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 
-private val config = Config()
 fun main() {
-    val lines = mainMenu.split("\n")
-    t.println(table {
-        borderType = BorderType.ROUNDED
-        borderStyle = TextColors.brightBlue
-        body { lines.forEach { row(menuStyle(it)) } }
-    })
-
-    val menu = readlnOrNull()
-    if (menu != null) {
-        when (menu) {
-            "1" -> crawler()
-            "2" -> fetchTweets()
-            "3" -> settings()
-        }
-    }
-}
-
-private fun crawler() {
-    val crawler = Crawler()
-    runBlocking {
-        withContext(Dispatchers.Default) {
-            crawler.crawl()
-        }
-    }
+    MainModule().run()
 }
 
 fun fetchTweets() {
@@ -49,14 +17,4 @@ fun fetchTweets() {
             crawler.crawl()
         }
     }
-}
-
-fun settings() {
-    t.println(infoStyle("current download folder is: $DOWNLOAD_PATH"))
-    val prompt = readlnOrNull() ?: "n"
-    if (prompt == "y") {
-        t.println(askStyle("enter new download path"))
-        DOWNLOAD_PATH = readlnOrNull() ?: DOWNLOAD_PATH
-        config.updateConfigFile(DOWNLOAD_FOLDER to DOWNLOAD_PATH)
-    }
-}
+}

src/main/kotlin/io/github/yamin8000/twitterscrapper/helpers/ConsoleHelper.kt

@@ -0,0 +1,155 @@
+package io.github.yamin8000.twitterscrapper.helpers
+
+import com.github.ajalt.mordant.rendering.*
+import com.github.ajalt.mordant.terminal.Terminal
+import io.github.yamin8000.twitterscrapper.helpers.ConsoleHelper.table
+import java.util.*
+import kotlin.reflect.full.memberProperties
+import kotlin.reflect.KVisibility
+
+@OptIn(com.github.ajalt.mordant.terminal.ExperimentalTerminalApi::class)
+object ConsoleHelper {
+
+    private val tableSubjectStyle = TextColors.blue + TextStyles.bold
+
+    private val affirmatives = listOf(
+        "y",
+        "yes",
+        "true",
+        "1",
+        "yep",
+        "yeah",
+        "yup",
+        "yuh",
+        "بله",
+        "آره",
+        "باشه",
+        "نعم",
+        "да",
+        "давай",
+        "давайте",
+        "si",
+        "oui",
+        "ja",
+        "ok",
+        "okay"
+    )
+
+    val t = Terminal()
+    val resultStyle = TextColors.green
+    val infoStyle = TextColors.brightMagenta + TextStyles.bold
+    val errorStyle = TextColors.red + TextStyles.bold
+    val askStyle = TextColors.cyan + TextStyles.bold
+    val warningStyle = TextColors.yellow + TextStyles.bold
+    val menuStyle = TextColors.blue + TextStyles.bold
+
+    private const val integerInputFailure = "Please enter a number only, try again!"
+
+    /**
+     * Prompts the user for an [Integer] input with the given optional [message],
+     * the number must be between [range] if included otherwise any [Integer] is acceptable.
+     * Eventually return the input as [Int]
+     */
+    fun readInteger(message: String? = null, range: IntRange? = null): Int {
+        if (message != null) t.println(askStyle(message))
+        return try {
+            val input = readCleanLine()
+            if (input.isNotBlank() && input.all { it.isDigit() }) {
+                val number = input.toInt()
+                if (number.isInRange(range)) number
+                else readIntegerAfterFailure("Input is out of range.", message, range)
+            } else readIntegerAfterFailure(integerInputFailure, message, range)
+        } catch (exception: NumberFormatException) {
+            readIntegerAfterFailure(integerInputFailure, message, range)
+        }
+    }
+
+    /**
+     * Extension function for checking if [Int] is in the given [range]
+     */
+    private fun Int.isInRange(range: IntRange?) = range == null || this in range
+
+    /**
+     * If the input is not valid (refer to [readInteger]), prompt the user again for an [Integer] input
+     */
+    private fun readIntegerAfterFailure(error: String, message: String?, range: IntRange?): Int {
+        t.println(errorStyle(error))
+        return readInteger(message, range)
+    }
+
+    /**
+     * Prompts the user for a [Boolean] input with the given optional [message],
+     * and eventually return the input as [Boolean]
+     */
+    fun readBoolean(message: String? = null): Boolean {
+        return try {
+            if (message != null) t.println(askStyle(message))
+            readCleanLine().lowercase(Locale.getDefault()) in affirmatives
+        } catch (exception: Exception) {
+            false
+        }
+    }
+
+    /**
+     * Prompts the user to press enter to continue
+     */
+    fun pressEnterToContinue(message: String = "continue...") {
+        t.println((TextColors.yellow on TextColors.black)("Press enter to $message"))
+        readCleanLine()
+    }
+
+    /**
+     * Prompts the user for entering multiple [String] values,
+     * and eventually return a [List] of [String]
+     */
+    fun readMultipleStrings(field: String): List<String> {
+        t.println(askStyle("Please enter ${infoStyle("$field/${field}s")}"))
+        t.println(askStyle("If there are more than one ${infoStyle(field)} separate them using a comma (${infoStyle(",")})"))
+        t.print(askStyle("Example: "))
+        t.println("${randHsv()("Jackie")},${randHsv()("Elon")},${randHsv()("Taylor")},${randHsv()("Gary")}")
+        val input = readCleanLine().split(",").map { it.trim() }
+        return if (input.isValid()) {
+            t.println(errorStyle("Please enter at least one $field."))
+            readMultipleStrings(field)
+        } else input
+    }
+
+    /**
+     * Prompts the user for a single [String] value,
+     * and eventually return the input as [String]
+     */
+    fun readSingleString(field: String): String {
+        t.println(askStyle("Please enter ") + infoStyle(field))
+        return readCleanLine().ifBlank {
+            t.println(errorStyle("Input cannot be empty, try again!"))
+            this.readSingleString(field)
+        }
+    }
+
+    fun <T> T.table() = buildString {
+        this@table!!::class.memberProperties.forEach {
+            if (it.visibility == KVisibility.PUBLIC) {
+                appendLine(it.name)
+                appendLine("${it.getter.call(this@table)}")
+            }
+        }
+    }
+
+    fun <T> T.printTable() {
+        this.table().split('\n').forEachIndexed { index, line ->
+            if (index % 2 == 0)
+                t.println(tableSubjectStyle(line))
+            else t.println(infoStyle(line))
+        }
+    }
+
+    /**
+     * Reads a line from input and [String.trim] it,
+     * if the input is null then returns an empty [String]
+     */
+    private fun readCleanLine(): String = readlnOrNull() ?: "".trim()
+
+    private fun List<String>.isValid() = !(this.isEmpty() || this.all { it.isNotEmpty() })
+
+    private fun randHsv() = t.colors.hsv((0..360).random(), 1, 1)
+}

src/main/kotlin/io/github/yamin8000/twitterscrapper/helpers/UserHelper.kt

@@ -0,0 +1,59 @@
+package io.github.yamin8000.twitterscrapper.helpers
+
+import io.github.yamin8000.twitterscrapper.web.get
+import io.github.yamin8000.twitterscrapper.helpers.ConsoleHelper.errorStyle
+import io.github.yamin8000.twitterscrapper.helpers.ConsoleHelper.t
+import io.github.yamin8000.twitterscrapper.model.User
+import io.github.yamin8000.twitterscrapper.util.Constants.instances
+import io.github.yamin8000.twitterscrapper.util.Utility.sanitizeUsername
+import org.jsoup.Jsoup
+
+object UserHelper {
+    @Throws(Exception::class)
+    suspend fun getUser(
+        username: String,
+        base: String = instances.first()
+    ): User? {
+        val response = get("${base}${username.sanitizeUsername()}")
+        return if (response.isSuccessful) parseUser(response.body.string())
+        else getUserFailedRequest(username, response.code)
+    }
+
+    private suspend fun getUserFailedRequest(
+        username: String,
+        httpCode: Int
+    ): User? {
+        val temp = instances.drop(0)
+        if (temp.isNotEmpty()) return getUser(username, temp.first())
+        else throw Exception("Fetching info for user: $username failed with $httpCode")
+    }
+
+    private fun parseUser(
+        html: String
+    ): User? {
+        return try {
+            val doc = Jsoup.parse(html)
+            User(
+                username = doc.selectFirst("a[class^=profile-card-username]")?.attr("title") ?: "",
+                fullname = doc.selectFirst("a[class^=profile-card-fullname]")?.attr("title") ?: "",
+                isVerified = doc.selectFirst("div[class^=profile-card-tabs-name] span[class^=icon-ok verified-icon]") != null,
+                bio = doc.selectFirst("div[class^=profile-bio] > p")?.text() ?: "",
+                location = doc.select("div[class^=profile-location] > span").getOrNull(1)?.text() ?: "",
+                joinDate = doc.selectFirst("div[class^=profile-joindate] > span[title]")?.attr("title") ?: "",
+                avatar = instances.first().dropLast(1) + doc.selectFirst("a[class^=profile-card-avatar]")?.attr("href"),
+                banner = instances.first().dropLast(1) + doc.selectFirst("div[class^=profile-banner] a")?.attr("href"),
+                tweets = doc.selectFirst("li[class^=posts] span[class^=profile-stat-num]")?.text().sanitizeNum(),
+                following = doc.selectFirst("li[class^=following] > span[class^=profile-stat-num]")?.text()
+                    .sanitizeNum(),
+                followers = doc.selectFirst("li[class^=followers] > span[class^=profile-stat-num]")?.text()
+                    .sanitizeNum(),
+                likes = doc.selectFirst("li[class^=likes] span[class^=profile-stat-num]")?.text().sanitizeNum()
+            )
+        } catch (e: Exception) {
+            t.println(errorStyle(e.stackTraceToString()))
+            null
+        }
+    }
+
+    private fun String?.sanitizeNum() = this?.filter { it != ',' }?.toIntOrNull() ?: 0
+}

src/main/kotlin/io/github/yamin8000/twitterscrapper/model/Tweet.kt

@@ -0,0 +1,17 @@
+package io.github.yamin8000.twitterscrapper.model
+
+data class Tweet(
+    val content: String,
+    val date: String,
+    val link: String,
+    val contentType: TweetContentType,
+    val user: User,
+    val stats: TweetStats,
+    val isRetweet: Boolean,
+    val isThreaded: Boolean,
+    val isPinned: Boolean = false,
+    val replies: List<Tweet> = listOf(),
+    val originalTweeter: User? = null,
+    val quote: Tweet? = null,
+    val thread: String? = null,
+)

src/main/kotlin/io/github/yamin8000/twitterscrapper/model/TweetContentType.kt

@@ -0,0 +1,5 @@
+package io.github.yamin8000.twitterscrapper.model
+
+enum class TweetContentType {
+    TextTweet, Mixed
+}

src/main/kotlin/io/github/yamin8000/twitterscrapper/model/TweetStats.kt

@@ -0,0 +1,8 @@
+package io.github.yamin8000.twitterscrapper.model
+
+data class TweetStats(
+    val replies: Int = 0,
+    val retweets: Int = 0,
+    val quotes: Int = 0,
+    val likes: Int = 0,
+)

src/main/kotlin/io/github/yamin8000/twitterscrapper/model/User.kt

@@ -0,0 +1,16 @@
+package io.github.yamin8000.twitterscrapper.model
+
+data class User(
+    val username: String = "",
+    val fullname: String = "",
+    val isVerified: Boolean = false,
+    val bio: String = "",
+    val location: String = "",
+    val joinDate: String = "",
+    val avatar: String = "",
+    val banner: String = "",
+    val tweets: Int? = null,
+    val following: Int? = null,
+    val followers: Int? = null,
+    val likes: Int? = null
+)

src/main/kotlin/io/github/yamin8000/twitterscrapper/modules/BaseModule.kt

@@ -0,0 +1,28 @@
+package io.github.yamin8000.twitterscrapper.modules
+
+import com.github.ajalt.mordant.rendering.BorderType
+import com.github.ajalt.mordant.rendering.TextColors
+import com.github.ajalt.mordant.rendering.TextStyles
+import com.github.ajalt.mordant.table.table
+import io.github.yamin8000.twitterscrapper.helpers.ConsoleHelper.readInteger
+import io.github.yamin8000.twitterscrapper.helpers.ConsoleHelper.t
+
+open class BaseModule(private val menuText: String) {
+
+    private val style = TextColors.blue + TextStyles.bold
+
+    open fun run(): Int {
+        val subMenus = showMenu()
+        return readInteger(range = 0 until subMenus)
+    }
+
+    fun showMenu(): Int {
+        val lines = menuText.split("\n")
+        t.println(table {
+            borderType = BorderType.ROUNDED
+            borderStyle = TextColors.brightBlue
+            body { lines.forEach { row(style(it)) } }
+        })
+        return lines.size
+    }
+}