diff --git a/src/main/kotlin/app/extractors/CExtractor.kt b/src/main/kotlin/app/extractors/CExtractor.kt index 10d1cc47..f68539d1 100644 --- a/src/main/kotlin/app/extractors/CExtractor.kt +++ b/src/main/kotlin/app/extractors/CExtractor.kt @@ -19,17 +19,17 @@ class CExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() + val imports = mutableSetOf() val regex = Regex("""#include\s+["<](\w+)[/\w+]*\.\w+[">]""") fileContent.forEach { val res = regex.find(it) if (res != null) { val lineLib = res.groupValues.last() - libraries.add(lineLib) + imports.add(lineLib) } } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/CSharpExtractor.kt b/src/main/kotlin/app/extractors/CSharpExtractor.kt index 5ff978c9..65f9d6ea 100644 --- a/src/main/kotlin/app/extractors/CSharpExtractor.kt +++ b/src/main/kotlin/app/extractors/CSharpExtractor.kt @@ -12,6 +12,7 @@ class CSharpExtractor : ExtractorInterface { companion object { val LANGUAGE_NAME = "cs" val FILE_EXTS = listOf("cs") + val LIBRARIES = ExtractorInterface.getLibraries("cs") } override fun extract(files: List): List { @@ -20,27 +21,21 @@ class CSharpExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() - - // TODO(anatoly): Load file statically. - val csLibraries = File("data/libraries/cs_libraries.txt") - .inputStream().bufferedReader() - .readLines() - .toSet() + val imports = mutableSetOf() val regex = Regex("""using\s+(\w+[.\w+]*)""") fileContent.forEach { val res = regex.find(it) if (res != null) { val importedName = res.groupValues[1] - csLibraries.forEach { library -> + LIBRARIES.forEach { library -> if (importedName.startsWith(library)) { - libraries.add(library) + imports.add(library) } } } } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/CppExtractor.kt b/src/main/kotlin/app/extractors/CppExtractor.kt index 60686777..29828cbd 100644 --- a/src/main/kotlin/app/extractors/CppExtractor.kt +++ b/src/main/kotlin/app/extractors/CppExtractor.kt @@ -19,17 +19,17 @@ class CppExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() + val imports = mutableSetOf() val regex = Regex("""#include\s+["<](\w+)[/\w+]*\.\w+[">]""") fileContent.forEach { val res = regex.find(it) if (res != null) { val lineLib = res.groupValues.last() - libraries.add(lineLib) + imports.add(lineLib) } } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/ExtractorInterface.kt b/src/main/kotlin/app/extractors/ExtractorInterface.kt index f2b8b9c7..165c923d 100644 --- a/src/main/kotlin/app/extractors/ExtractorInterface.kt +++ b/src/main/kotlin/app/extractors/ExtractorInterface.kt @@ -8,6 +8,14 @@ import app.model.DiffFile import app.model.CommitStats interface ExtractorInterface { + companion object { + fun getLibraries(name: String): Set { + return ExtractorInterface::class.java.classLoader + .getResourceAsStream("data/libraries/${name}_libraries.txt") + .bufferedReader().readLines().toSet() + } + } + fun extract(files: List): List { files.map { file -> file.old.imports = extractImports(file.old.content) @@ -29,4 +37,6 @@ interface ExtractorInterface { fun extractImports(fileContent: List): List { return listOf() } + + } diff --git a/src/main/kotlin/app/extractors/GoExtractor.kt b/src/main/kotlin/app/extractors/GoExtractor.kt index ec98cfee..bd2862a4 100644 --- a/src/main/kotlin/app/extractors/GoExtractor.kt +++ b/src/main/kotlin/app/extractors/GoExtractor.kt @@ -19,26 +19,26 @@ class GoExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() + val imports = mutableSetOf() val singleImportRegex = Regex("""import\s+"(\w+)"""") fileContent.forEach { val res = singleImportRegex.find(it) if (res != null) { val lineLib = res.groupValues.last() - libraries.add(lineLib) + imports.add(lineLib) } } val multipleImportRegex = Regex("""import[\s\t\n]+\((.+?)\)""", RegexOption.DOT_MATCHES_ALL) val contentJoined = fileContent.joinToString(separator = "") multipleImportRegex.findAll(contentJoined).forEach { matchResult -> - libraries.addAll(matchResult.groupValues.last() + imports.addAll(matchResult.groupValues.last() .split(Regex("""(\t+|\n+|\s+)""")) .filter { it.isNotEmpty() } .map { it -> it.replace("\"", "") }) } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/JavaExtractor.kt b/src/main/kotlin/app/extractors/JavaExtractor.kt index 051dc68e..f36d5ead 100644 --- a/src/main/kotlin/app/extractors/JavaExtractor.kt +++ b/src/main/kotlin/app/extractors/JavaExtractor.kt @@ -12,6 +12,7 @@ class JavaExtractor : ExtractorInterface { companion object { val LANGUAGE_NAME = "java" val FILE_EXTS = listOf("java") + val LIBRARIES = ExtractorInterface.getLibraries("java") } val KEYWORDS = listOf("abstract", "continue", "for", "new", "switch", @@ -56,27 +57,21 @@ class JavaExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() - - // TODO(anatoly): Load file statically. - val javaLibraries = File("data/libraries/java_libraries.txt") - .inputStream().bufferedReader() - .readLines() - .toSet() + val imports = mutableSetOf() val regex = Regex("""import\s+(\w+[.\w+]*)""") fileContent.forEach { val res = regex.find(it) if (res != null) { val importedName = res.groupValues[1] - javaLibraries.forEach { library -> + LIBRARIES.forEach { library -> if (importedName.startsWith(library)) { - libraries.add(library) + imports.add(library) } } } } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/JavascriptExtractor.kt b/src/main/kotlin/app/extractors/JavascriptExtractor.kt index 914533da..a5d3a2df 100644 --- a/src/main/kotlin/app/extractors/JavascriptExtractor.kt +++ b/src/main/kotlin/app/extractors/JavascriptExtractor.kt @@ -6,12 +6,12 @@ package app.extractors import app.model.CommitStats import app.model.DiffFile -import java.io.File class JavascriptExtractor : ExtractorInterface { companion object { val LANGUAGE_NAME = "js" val FILE_EXTS = listOf("js") + val LIBRARIES = ExtractorInterface.getLibraries("js") } override fun extract(files: List): List { @@ -20,20 +20,14 @@ class JavascriptExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() - - // TODO(anatoly): Load file statically. - val jsLibraries = File("data/libraries/js_libraries.txt") - .inputStream().bufferedReader() - .readLines() - .toSet() + val imports = mutableSetOf() val splitRegex = Regex("""\s+|,|;|:|\\*|\n|\(|\)|\\[|]|\{|}|\+|=|\.|>|<|#|@|\$""") val fileTokens = fileContent.joinToString(separator = " ") .split(splitRegex) - libraries.addAll(fileTokens.filter { token -> token in jsLibraries }) + imports.addAll(fileTokens.filter { token -> token in LIBRARIES }) - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/ObjectiveCExtractor.kt b/src/main/kotlin/app/extractors/ObjectiveCExtractor.kt index 5f16b09e..4eda9282 100644 --- a/src/main/kotlin/app/extractors/ObjectiveCExtractor.kt +++ b/src/main/kotlin/app/extractors/ObjectiveCExtractor.kt @@ -19,7 +19,7 @@ class ObjectiveCExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() + val imports = mutableSetOf() val sharpImportIncludeRegex = Regex("""#(import|include)\s+[">](\w+)[/\w+]*\.\w+[">]""") @@ -30,10 +30,10 @@ class ObjectiveCExtractor : ExtractorInterface { atImportRegex.findAll(it) if (res.toList().isNotEmpty()) { val lineLib = res.toList().map { it.groupValues }.last().last() - libraries.add(lineLib) + imports.add(lineLib) } } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/PhpExtractor.kt b/src/main/kotlin/app/extractors/PhpExtractor.kt index 60b91215..66832489 100644 --- a/src/main/kotlin/app/extractors/PhpExtractor.kt +++ b/src/main/kotlin/app/extractors/PhpExtractor.kt @@ -19,7 +19,7 @@ class PhpExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() + val imports = mutableSetOf() val useRegex = Regex("""use\s+(\w+)[\\\w+]*""") val requireIncludeRegex = Regex("""(require|require_once|include|""" + @@ -28,10 +28,10 @@ class PhpExtractor : ExtractorInterface { val res = useRegex.findAll(it) + requireIncludeRegex.findAll(it) if (res.toList().isNotEmpty()) { val lineLib = res.toList().map { it.groupValues }.last().last() - libraries.add(lineLib) + imports.add(lineLib) } } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/PythonExtractor.kt b/src/main/kotlin/app/extractors/PythonExtractor.kt index 604027f3..139cc9f2 100644 --- a/src/main/kotlin/app/extractors/PythonExtractor.kt +++ b/src/main/kotlin/app/extractors/PythonExtractor.kt @@ -19,7 +19,7 @@ class PythonExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() + val imports = mutableSetOf() val regex = Regex("""(from\s+(\w+)[.\w+]*\s+import|import\s+(\w+[,\s*\w+]*))""") @@ -28,10 +28,10 @@ class PythonExtractor : ExtractorInterface { if (res != null) { val lineLibs = res.groupValues.last { it != "" } .split(Regex(""",\s*""")) - libraries.addAll(lineLibs) + imports.addAll(lineLibs) } } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/RubyExtractor.kt b/src/main/kotlin/app/extractors/RubyExtractor.kt index 98adaf27..5c9518df 100644 --- a/src/main/kotlin/app/extractors/RubyExtractor.kt +++ b/src/main/kotlin/app/extractors/RubyExtractor.kt @@ -19,17 +19,17 @@ class RubyExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() + val imports = mutableSetOf() val regex = Regex("""(require\s+'(\w+)'|load\s+'(\w+)\.\w+')""") fileContent.forEach { val res = regex.find(it) if (res != null) { val lineLib = res.groupValues.last { it -> it != "" } - libraries.add(lineLib) + imports.add(lineLib) } } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/extractors/SwiftExtractor.kt b/src/main/kotlin/app/extractors/SwiftExtractor.kt index 95d21848..742f2c29 100644 --- a/src/main/kotlin/app/extractors/SwiftExtractor.kt +++ b/src/main/kotlin/app/extractors/SwiftExtractor.kt @@ -19,17 +19,17 @@ class SwiftExtractor : ExtractorInterface { } override fun extractImports(fileContent: List): List { - val libraries = mutableSetOf() + val imports = mutableSetOf() val regex = Regex("""import\s+(\w+)""") fileContent.forEach { val res = regex.find(it) if (res != null) { val lineLib = res.groupValues[1] - libraries.add(lineLib) + imports.add(lineLib) } } - return libraries.toList() + return imports.toList() } } diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index f3a1a05f..e9cc1d81 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -95,6 +95,7 @@ class CommitHasher(private val localRepo: LocalRepo, .subscribe({ commitsBundle -> // OnNext. postCommitsToServer(commitsBundle) // Send ready commits. }, { e -> // OnError. + Logger.error("Hashing error: " + e.message) throwables.add(e) // TODO(anatoly): Top-class handling errors. }) } diff --git a/data/libraries/cs_libraries.txt b/src/main/resources/data/libraries/cs_libraries.txt similarity index 100% rename from data/libraries/cs_libraries.txt rename to src/main/resources/data/libraries/cs_libraries.txt diff --git a/data/libraries/java_libraries.txt b/src/main/resources/data/libraries/java_libraries.txt similarity index 100% rename from data/libraries/java_libraries.txt rename to src/main/resources/data/libraries/java_libraries.txt diff --git a/data/libraries/js_libraries.txt b/src/main/resources/data/libraries/js_libraries.txt similarity index 100% rename from data/libraries/js_libraries.txt rename to src/main/resources/data/libraries/js_libraries.txt