更新阅读解析库

hectorqin · Apr 8, 2022 · bb8d13e · bb8d13e
1 parent e20b1b6
commit bb8d13e
Show file tree

Hide file tree

Showing 53 changed files with 4,666 additions and 1,542 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -3,7 +3,7 @@ name: Publish Docker Multi-Platform Images
 on:
   push:
     tags:
-      - 'v*.*.*'
+      - 'v**'
 
 jobs:
   docker:
@@ -42,7 +42,7 @@ jobs:
           # generate Docker tags based on the following events/attributes
           tags: |
             type=semver,pattern={{version}}
-            latest
+            type=raw,value=latest,enable=${{ !contains(version, 'beta') }}
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1

diff --git a/README.md b/README.md
@@ -501,7 +501,7 @@ lastIndex 是上次搜索结果中返回的字段，默认为 0，可以传入 `
 
 ## 问题
 
-- 部分使用了 `Javascript` 的书源可能会报错
+- 部分使用了 `Javascript` 的书源可能会报错，不支持 `cookie` 和 `cache` 功能
 - 不支持使用了 `webview` 的书源
 - 不支持书源登录功能
 

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -6,8 +6,8 @@ import io.github.fvarrui.javapackager.model.WindowsConfig
 import de.undercouch.gradle.tasks.download.Download
 
 buildscript {
-    val kotlin_version: String by extra
-    extra["kotlin_version"] = "1.3.50"
+    val kotlin_version: String by extra{"1.5.21"}
+    // extra["kotlin_version"] = "1.5.21"
     repositories {
 	    mavenLocal()
         mavenCentral()
@@ -76,47 +76,63 @@ val compileOnly by configurations.getting {
 }
 
 dependencies {
-    val kotlin_version: String by extra
+    val kotlin_version: String by extra{"1.5.21"}
+    // val kotlin_version: String by extra
     implementation("org.springframework.boot:spring-boot-starter")
     testImplementation("org.springframework.boot:spring-boot-starter-test")
     implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version")
-    implementation("org.jsoup:jsoup:1.14.1")
-    implementation("com.google.code.gson:gson:2.8.5")
+    // vertx
     implementation("io.vertx:vertx-core:3.8.1")
     implementation("io.vertx:vertx-lang-kotlin:3.8.1")
     implementation("io.vertx:vertx-lang-kotlin-coroutines:3.8.1")
     implementation("io.vertx:vertx-web:3.8.1")
     implementation("io.vertx:vertx-web-client:3.8.1")
-//    implementation("io.vertx:vertx-rx-java2:3.8.1")
+
+    // json
+    implementation("com.google.code.gson:gson:2.8.5")
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.9.+")
+
+    // log
     implementation("io.github.microutils:kotlin-logging:1.6.24")
-    implementation("com.google.guava:guava:28.0-jre")
-    implementation("com.squareup.okhttp3:okhttp:4.1.1")
     implementation("uk.org.lidalia:sysout-over-slf4j:1.0.2")
-    implementation("io.vertx:vertx-mysql-client:3.8.1")
-    //Retrofit
+
+    implementation("com.google.guava:guava:28.0-jre")
+
+    // 网络
+    implementation("com.squareup.okhttp3:okhttp:4.9.1")
     implementation("com.squareup.okhttp3:logging-interceptor:4.1.0")
+    // Retrofit
     implementation("com.squareup.retrofit2:retrofit:2.6.1")
-
-    implementation("com.github.gedoor:rhino-android:1.3")
-    implementation("cn.wanghaomiao:JsoupXpath:2.3.2")
-    implementation("com.jayway.jsonpath:json-path:2.4.0")
     implementation("com.julienviet:retrofit-vertx:1.1.3")
-    implementation("com.squareup.okhttp3:logging-interceptor:4.2.1")
 
-    // implementation("xmlpull:xmlpull:1.1.3.1")
+    //JS rhino
+    implementation("com.github.gedoor:rhino-android:1.6")
+
+    // 规则相关
+    implementation("org.jsoup:jsoup:1.14.1")
+    implementation("cn.wanghaomiao:JsoupXpath:2.5.0")
+    implementation("com.jayway.jsonpath:json-path:2.6.0")
+
+    // xml
     implementation("org.xmlpull:xmlpull:1.1.4.0")
     implementation("com.github.stefanhaustein:kxml2:2.5.0")
+
+    // 转换繁体
+    // implementation("com.github.liuyueyi.quick-chinese-transfer:quick-transfer-core:0.2.1")
 }
 
-val compileKotlin: KotlinCompile by tasks
-val compileTestKotlin: KotlinCompile by tasks
+// val compileKotlin: KotlinCompile by tasks
+// val compileTestKotlin: KotlinCompile by tasks
 
-compileKotlin.kotlinOptions {
-    jvmTarget = "1.8"
-}
-compileTestKotlin.kotlinOptions {
-    jvmTarget = "1.8"
+// compileKotlin.kotlinOptions {
+//     jvmTarget = "1.8"
+// }
+// compileTestKotlin.kotlinOptions {
+//     jvmTarget = "1.8"
+// }
+
+tasks.withType<KotlinCompile> {
+    kotlinOptions.jvmTarget = "1.8"
 }
 
 application {

diff --git a/cli.gradle b/cli.gradle
@@ -1,5 +1,5 @@
 buildscript {
-    ext.kotlin_version = '1.3.50'
+    ext.kotlin_version = '1.5.21'
     repositories {
         mavenCentral()
     }
@@ -38,33 +38,44 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter'
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
-    implementation "org.jsoup:jsoup:1.14.1"
-    implementation 'com.google.code.gson:gson:2.8.5'
+    // vertx
     implementation "io.vertx:vertx-core:3.8.1"
     implementation "io.vertx:vertx-lang-kotlin:3.8.1"
     implementation "io.vertx:vertx-lang-kotlin-coroutines:3.8.1"
     implementation 'io.vertx:vertx-web:3.8.1'
     implementation 'io.vertx:vertx-web-client:3.8.1'
-//    implementation 'io.vertx:vertx-rx-java2:3.8.1'
+
+    // json
+    implementation "com.google.code.gson:gson:2.8.5"
     implementation "com.fasterxml.jackson.module:jackson-module-kotlin:2.9.+"
-    implementation 'io.github.microutils:kotlin-logging:1.6.24'
-    implementation 'com.google.guava:guava:28.0-jre'
-    implementation("com.squareup.okhttp3:okhttp:4.1.1")
-    implementation 'uk.org.lidalia:sysout-over-slf4j:1.0.2'
-    implementation 'io.vertx:vertx-mysql-client:3.8.1'
-    //Retrofit
-    implementation 'com.squareup.okhttp3:logging-interceptor:4.1.0'
-    implementation 'com.squareup.retrofit2:retrofit:2.6.1'
 
-    implementation 'com.github.gedoor:rhino-android:1.3'
-    implementation 'cn.wanghaomiao:JsoupXpath:2.3.2'
-    implementation 'com.jayway.jsonpath:json-path:2.4.0'
-    implementation 'com.julienviet:retrofit-vertx:1.1.3'
-    implementation("com.squareup.okhttp3:logging-interceptor:4.2.1")
+    // log
+    implementation "io.github.microutils:kotlin-logging:1.6.24"
+    implementation "uk.org.lidalia:sysout-over-slf4j:1.0.2"
+
+    implementation "com.google.guava:guava:28.0-jre"
+
+    // 网络
+    implementation "com.squareup.okhttp3:okhttp:4.9.1"
+    implementation "com.squareup.okhttp3:logging-interceptor:4.1.0"
+    // Retrofit
+    implementation "com.squareup.retrofit2:retrofit:2.6.1"
+    implementation "com.julienviet:retrofit-vertx:1.1.3"
+
+    //JS rhino
+    implementation "com.github.gedoor:rhino-android:1.6"
+
+    // 规则相关
+    implementation "org.jsoup:jsoup:1.14.1"
+    implementation "cn.wanghaomiao:JsoupXpath:2.5.0"
+    implementation "com.jayway.jsonpath:json-path:2.6.0"
+
+    // xml
+    implementation "org.xmlpull:xmlpull:1.1.4.0"
+    implementation "com.github.stefanhaustein:kxml2:2.5.0"
 
-    // implementation 'xmlpull:xmlpull:1.1.3.1'
-    implementation("org.xmlpull:xmlpull:1.1.4.0")
-    implementation("com.github.stefanhaustein:kxml2:2.5.0")
+    // 转换繁体
+    // implementation "com.github.liuyueyi.quick-chinese-transfer:quick-transfer-core:0.2.1"
 }
 compileKotlin {
     kotlinOptions {

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/src/main/java/com/htmake/reader/api/YueduApi.kt b/src/main/java/com/htmake/reader/api/YueduApi.kt
@@ -1806,7 +1806,7 @@ class YueduApi : RestVerticle() {
         if (!bookInfo.isLocalBook() && bookSource.isNullOrEmpty()) {
             return returnData.setErrorMsg("未配置书源")
         }
-        if (chapterUrl.isNullOrEmpty()) {
+        if (chapterInfo == null || chapterUrl.isNullOrEmpty()) {
             return returnData.setErrorMsg("获取章节链接失败")
         }
 
@@ -1853,7 +1853,7 @@ class YueduApi : RestVerticle() {
             content = bookContent
         } else {
             try {
-                content = WebBook(bookSource ?: "", false).getBookContent(chapterUrl, nextChapterUrl)
+                content = WebBook(bookSource ?: "", false).getBookContent(bookInfo, chapterInfo, nextChapterUrl)
             } catch(e: Exception) {
                 if (!bookSource.isNullOrEmpty()) {
                     var bookSourceObject = asJsonObject(bookSource)?.mapTo(BookSource::class.java)

diff --git a/src/main/java/com/htmake/reader/init/appCtx.kt b/src/main/java/com/htmake/reader/init/appCtx.kt
@@ -0,0 +1,10 @@
+package com.htmake.reader.init
+
+import com.htmake.reader.utils.getWorkDir
+
+// 处理 appCtx
+object appCtx {
+    val cacheDir: String by lazy {
+        getWorkDir("storage", "cache")
+    }
+}
diff --git a/src/main/java/io/legado/app/constant/AppPattern.kt b/src/main/java/io/legado/app/constant/AppPattern.kt
@@ -3,7 +3,8 @@ package io.legado.app.constant
 import java.util.regex.Pattern
 
 object AppPattern {
-    val JS_PATTERN: Pattern = Pattern.compile("(<js>[\\w\\W]*?</js>|@js:[\\w\\W]*$)", Pattern.CASE_INSENSITIVE)
+    val JS_PATTERN: Pattern =
+        Pattern.compile("<js>([\\w\\W]+?)</js>|@js:([\\w\\W]*)", Pattern.CASE_INSENSITIVE)
     val EXP_PATTERN: Pattern = Pattern.compile("\\{\\{([\\w\\W]*?)\\}\\}")
 
     // val JS_PATTERN: Pattern = Pattern.compile("<js>([\\w\\W]*?)</js>|@js:([\\w\\W]*)", Pattern.CASE_INSENSITIVE)
@@ -12,10 +13,7 @@ object AppPattern {
     val imgPattern: Pattern = Pattern.compile("<img[^>]*src=\"([^\"]*(?:\"[^>]+\\})?)\"[^>]*>")
 
     val nameRegex = Regex("\\s+作\\s*者.*|\\s+\\S+\\s+著")
-    val authorRegex = Regex("^\\s*作\\s*者[:：\\s]+|\\s+著")
+    val authorRegex = Regex("^.*?作\\s*者[:：\\s]*|\\s+著")
     val fileNameRegex = Regex("[\\\\/:*?\"<>|.]")
     val splitGroupRegex = Regex("[,;，；]")
-
-    val bdRegex = Regex("(\\p{P})+")
-    val rnRegex = Regex("[\\r\\n]")
 }
diff --git a/src/main/java/io/legado/app/data/entities/BaseBook.kt b/src/main/java/io/legado/app/data/entities/BaseBook.kt
@@ -4,6 +4,9 @@ import io.legado.app.model.analyzeRule.RuleDataInterface
 import io.legado.app.utils.splitNotBlank
 
 interface BaseBook : RuleDataInterface {
+    var name: String
+    var author: String
+    var bookUrl: String
     var kind: String?
     var wordCount: String?
 

diff --git a/src/main/java/io/legado/app/data/entities/Book.kt b/src/main/java/io/legado/app/data/entities/Book.kt
@@ -19,12 +19,12 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 
 @JsonIgnoreProperties("variableMap", "infoHtml", "tocHtml", "config", "rootDir", "readConfig", "localBook", "epub", "epubRootDir", "onLineTxt", "localTxt", "umd", "realAuthor", "unreadChapterNum", "folderName", "localFile", "kindList")
 data class Book(
-        var bookUrl: String = "",                   // 详情页Url(本地书源存储完整文件路径)
+        override var bookUrl: String = "",                   // 详情页Url(本地书源存储完整文件路径)
         var tocUrl: String = "",                    // 目录页Url (toc=table of Contents)
         var origin: String = BookType.local,        // 书源URL(默认BookType.local)
         var originName: String = "",                //书源名称
-        var name: String = "",                   // 书籍名称(书源获取)
-        var author: String = "",                 // 作者名称(书源获取)
+        override var name: String = "",                   // 书籍名称(书源获取)
+        override var author: String = "",                 // 作者名称(书源获取)
         override var kind: String? = null,                    // 分类信息(书源获取)
         var customTag: String? = null,              // 分类信息(用户修改)
         var coverUrl: String? = null,               // 封面Url(书源获取)

diff --git a/src/main/java/io/legado/app/data/entities/BookChapter.kt b/src/main/java/io/legado/app/data/entities/BookChapter.kt
@@ -3,18 +3,20 @@ package io.legado.app.data.entities
 
 import io.legado.app.utils.GSON
 import io.legado.app.utils.fromJsonObject
+import io.legado.app.utils.MD5Utils
+import io.legado.app.model.analyzeRule.AnalyzeUrl
+import io.legado.app.utils.NetworkUtils
 
 data class BookChapter(
         var url: String = "",               // 章节地址
         var title: String = "",              // 章节标题
+        var baseUrl: String = "",           //用来拼接相对url
         var bookUrl: String = "",           // 书籍地址
         var index: Int = 0,                 // 章节序号
         var resourceUrl: String? = null,    // 音频真实URL
         var tag: String? = null,            //
         var start: Long? = null,            // 章节起始位置
         var end: Long? = null,               // 章节终止位置
-        var isVip: Boolean = false,         // 是否VIP
-        var isPay: Boolean = false,         // 是否已购买
         var startFragmentId: String? = null,  //EPUB书籍当前章节的fragmentId
         var endFragmentId: String? = null,    //EPUB书籍下一章节的fragmentId
         var variable: String? = null        //变量
@@ -35,5 +37,23 @@ data class BookChapter(
         variable = GSON.toJson(variableMap)
     }
 
+    override fun hashCode() = url.hashCode()
+
+    override fun equals(other: Any?): Boolean {
+        if (other is BookChapter) {
+            return other.url == url
+        }
+        return false
+    }
+
+    fun getAbsoluteURL():String{
+        val urlMatcher = AnalyzeUrl.paramPattern.matcher(url)
+        val urlBefore = if(urlMatcher.find())url.substring(0,urlMatcher.start()) else url
+        val urlAbsoluteBefore = NetworkUtils.getAbsoluteURL(baseUrl,urlBefore)
+        return if(urlBefore.length == url.length) urlAbsoluteBefore else urlAbsoluteBefore + ',' + url.substring(urlMatcher.end())
+    }
+
+
+    fun getFileName(): String = String.format("%05d-%s.nb", index, MD5Utils.md5Encode16(title))
 }
 
diff --git a/src/main/java/io/legado/app/data/entities/BookSource.kt b/src/main/java/io/legado/app/data/entities/BookSource.kt
@@ -64,8 +64,7 @@ data class BookSource(
     @Throws(Exception::class)
     fun getHeaderMap(): Map<String, String> {
         val headerMap = HashMap<String, String>()
-//        headerMap[AppConst.UA_NAME] = App.INSTANCE.getPrefString("user_agent") ?: userAgent
-        //todo UA
+        // TODO UA
         headerMap[AppConst.UA_NAME] = userAgent
         header?.let {
             val header1 = when {

diff --git a/src/main/java/io/legado/app/data/entities/Cache.kt b/src/main/java/io/legado/app/data/entities/Cache.kt
@@ -0,0 +1,9 @@
+package io.legado.app.data.entities
+
+// @Entity(tableName = "caches", indices = [(Index(value = ["key"], unique = true))])
+data class Cache(
+    // @PrimaryKey
+    val key: String = "",
+    var value: String? = null,
+    var deadline: Long = 0L
+)
diff --git a/src/main/java/io/legado/app/data/entities/Cookie.kt b/src/main/java/io/legado/app/data/entities/Cookie.kt
@@ -0,0 +1,8 @@
+package io.legado.app.data.entities
+
+// @Entity(tableName = "cookies", indices = [(Index(value = ["url"], unique = true))])
+data class Cookie(
+    // @PrimaryKey
+    var url: String = "",
+    var cookie: String = ""
+)