fix: remove correnct line in duplicated

unit-mesh · Aug 9, 2023 · 6758260 · 6758260
1 parent bf0e1a4
commit 6758260
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 5 deletions.
diff --git a/src/main/kotlin/cc/unitmesh/devti/llms/tokenizer/Tokenizer.kt b/src/main/kotlin/cc/unitmesh/devti/llms/tokenizer/Tokenizer.kt
@@ -3,4 +3,5 @@ package cc.unitmesh.devti.llms.tokenizer
 interface Tokenizer {
     fun getMaxLength(): Int
     fun count(string: String): Int
+    fun tokenize(chunk: String): List<Int>
 }
diff --git a/src/main/kotlin/cc/unitmesh/devti/llms/tokenizer/TokenizerImpl.kt b/src/main/kotlin/cc/unitmesh/devti/llms/tokenizer/TokenizerImpl.kt
@@ -15,6 +15,9 @@ class TokenizerImpl(private val maxTokenLength: Int = 8192) : Tokenizer {
     override fun getMaxLength(): Int = maxTokenLength
 
     override fun count(string: String): Int = encoding.countTokens(string)
+    override fun tokenize(chunk: String): List<Int> {
+        return encoding.encode(chunk, maxTokenLength).tokens
+    }
 
     companion object {
         val INSTANCE = ApplicationManager.getApplication().getService(TokenizerImpl::class.java)

diff --git a/src/main/kotlin/com/intellij/temporary/similar/chunks/SimilarChunksWithPaths.kt b/src/main/kotlin/com/intellij/temporary/similar/chunks/SimilarChunksWithPaths.kt
@@ -102,13 +102,12 @@ class SimilarChunksWithPaths(private var chunkSize: Int = 60, private var maxRel
     }
 
     private fun getMostRecentFiles(element: PsiElement): List<VirtualFile> {
-        val fileType: FileType? = element.containingFile?.fileType
-        if (element.containingFile == null || fileType == null) {
-            return emptyList()
-        }
+        val fileType: FileType = element.containingFile?.fileType ?: return emptyList()
+
         val recentFiles: List<VirtualFile> = EditorHistoryManager.getInstance(element.project).fileList.filter { file ->
-            file.isValid && file.fileType == fileType
+            file.isValid && file.fileType == fileType && file != element.containingFile.virtualFile
         }
+
         val start = (recentFiles.size - maxRelevantFiles + 1).coerceAtLeast(0)
         val end = (recentFiles.size - 1).coerceAtLeast(0)
         return recentFiles.subList(start, end)