From 370eda5f4d4b6499afa37d7f49a531aebc37bab1 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Thu, 23 Nov 2017 23:17:42 +0300 Subject: [PATCH 1/3] feat: new path for config and data files --- .../kotlin/app/config/FileConfigurator.kt | 32 ++++------------ .../app/extractors/ExtractorInterface.kt | 36 +++++++----------- src/main/kotlin/app/hashers/CodeLongevity.kt | 23 +++++------- src/main/kotlin/app/utils/FileHelper.kt | 37 ++++++++++++++++++- 4 files changed, 66 insertions(+), 62 deletions(-) diff --git a/src/main/kotlin/app/config/FileConfigurator.kt b/src/main/kotlin/app/config/FileConfigurator.kt index fa2b48e3..c90ebad2 100644 --- a/src/main/kotlin/app/config/FileConfigurator.kt +++ b/src/main/kotlin/app/config/FileConfigurator.kt @@ -5,8 +5,8 @@ package app.config import app.Logger import app.model.LocalRepo -import app.model.Repo import app.model.User +import app.utils.FileHelper import app.utils.Options import app.utils.PasswordHelper import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility @@ -20,7 +20,6 @@ import java.io.IOException import java.nio.file.Files import java.nio.file.InvalidPathException import java.nio.file.NoSuchFileException -import java.nio.file.Paths import java.util.UUID /** @@ -30,7 +29,7 @@ class FileConfigurator : Configurator { /** * Persistent configuration file name. */ - private val CONFIG_FILE_NAME = ".sourcerer" + private val CONFIG_FILE_NAME = "config.yaml" // Config levels are presented in priority decreasing order. @@ -40,7 +39,7 @@ class FileConfigurator : Configurator { private var current: Config = Config() /** - * Persistent configuration saved in [userDir] in YAML format. + * Persistent configuration saved in config file in YAML format. */ private var persistent: Config = Config() @@ -65,17 +64,6 @@ class FileConfigurator : Configurator { */ private var user: User = User() - /** - * User directory path is where persistent config stored. - */ - private val userDir = try { - System.getProperty("user.home") - } - catch (e: SecurityException) { - Logger.error(e, "Cannot access user directory") - null - } - /** * Jackson's ObjectMapper. */ @@ -216,19 +204,15 @@ class FileConfigurator : Configurator { } /** - * Loads [persistent] configuration from config file stored in [userDir]. + * Loads [persistent] configuration from config file. */ override fun loadFromFile() { - if (userDir == null) { - return - } - // Сonfig initialization in case an exception is thrown. var loadConfig = Config() try { - loadConfig = Files.newBufferedReader(Paths.get(userDir, - CONFIG_FILE_NAME)).use { + loadConfig = Files.newBufferedReader(FileHelper + .getPath(CONFIG_FILE_NAME)).use { mapper.readValue(it, Config::class.java) } } catch (e: IOException) { @@ -253,11 +237,11 @@ class FileConfigurator : Configurator { } /** - * Saves [persistent] configuration to config file stored in [userDir]. + * Saves [persistent] configuration to config file. */ override fun saveToFile() { try { - Files.newBufferedWriter(Paths.get(userDir, CONFIG_FILE_NAME)).use { + Files.newBufferedWriter(FileHelper.getPath(CONFIG_FILE_NAME)).use { mapper.writeValue(it, persistent) } } catch (e: IOException) { diff --git a/src/main/kotlin/app/extractors/ExtractorInterface.kt b/src/main/kotlin/app/extractors/ExtractorInterface.kt index bb67495f..bd536441 100644 --- a/src/main/kotlin/app/extractors/ExtractorInterface.kt +++ b/src/main/kotlin/app/extractors/ExtractorInterface.kt @@ -8,18 +8,18 @@ import app.BuildConfig import app.Logger import app.model.DiffFile import app.model.CommitStats +import app.utils.FileHelper import java.io.InputStream import java.io.FileOutputStream -import java.nio.file.Files -import java.nio.file.Paths import org.apache.http.client.methods.HttpGet import org.apache.http.impl.client.HttpClientBuilder -import java.io.File interface ExtractorInterface { companion object { private val librariesCache = hashMapOf>() private val classifiersCache = hashMapOf() + private val modelsDir = "models" + private val pbExt = ".pb" private fun getResource(path: String): InputStream { return ExtractorInterface::class.java.classLoader @@ -45,21 +45,17 @@ interface ExtractorInterface { return importToLibrary } - private fun downloadModel(name: String, outputDir: String) { + private fun downloadModel(name: String) { val url = BuildConfig.LIBRARY_MODELS_URL + "$name.pb" - val outputPath = "$outputDir/$name.pb" - - if (Files.notExists(Paths.get(outputDir))) { - Files.createDirectories(Paths.get(outputDir)) - } + val file = FileHelper.getFile(name + pbExt, modelsDir) val builder = HttpClientBuilder.create() val client = builder.build() try { client.execute(HttpGet(url)).use { response -> val entity = response.entity if (entity != null) { - FileOutputStream(outputPath).use { outstream -> + FileOutputStream(file).use { outstream -> entity.writeTo(outstream) outstream.flush() outstream.close() @@ -78,22 +74,16 @@ interface ExtractorInterface { return classifiersCache[name]!! } - val pbDir = ".sourcerer/data/pb" - val pbPath = "$pbDir/$name.pb" - - if (Files.notExists(Paths.get(pbDir))) { - Files.createDirectories(Paths.get(pbDir)) - } - - if (Files.notExists(Paths.get(pbPath))) { - Logger.info { "Downloading $name.pb" } - downloadModel(name, pbDir) - Logger.info { "Downloaded $name.pb" } + val fileName = name + pbExt + if (FileHelper.isNotExists(fileName, modelsDir)) { + Logger.info { "Downloading " + fileName } + downloadModel(name) + Logger.info { "Downloaded " + fileName } } - Logger.info {"Loading $name evaluator" } + Logger.info { "Loading $name evaluator" } - val bytesArray = File(pbPath).readBytes() + val bytesArray = FileHelper.getFile(fileName, modelsDir).readBytes() val classifier = Classifier(bytesArray) classifiersCache.put(name, classifier) diff --git a/src/main/kotlin/app/hashers/CodeLongevity.kt b/src/main/kotlin/app/hashers/CodeLongevity.kt index 255bfbf9..078ea535 100644 --- a/src/main/kotlin/app/hashers/CodeLongevity.kt +++ b/src/main/kotlin/app/hashers/CodeLongevity.kt @@ -9,6 +9,7 @@ import app.api.Api import app.model.Author import app.model.Repo import app.model.Fact +import app.utils.FileHelper import app.utils.RepoHelper import io.reactivex.Observable import org.eclipse.jgit.diff.DiffFormatter @@ -152,8 +153,7 @@ class CodeLongevity(private val serverRepo: Repo, catch(e: Exception) { throw Exception("No branch") } val df = DiffFormatter(DisabledOutputStream.INSTANCE) - val storageDir = ".sourcerer/data/longevity" - val storagePath = "$storageDir/${serverRepo.rehash}" + val dataPath = FileHelper.getPath(serverRepo.rehash, "longevity") init { df.setRepository(repo) @@ -221,12 +221,13 @@ class CodeLongevity(private val serverRepo: Repo, */ fun scan() : CodeLineAges? { var storedHead: RevCommit? = null - var ageData: CodeLineAges = CodeLineAges() + var ageData = CodeLineAges() // Load existing age data if any. Expected format: commit id and // CodeLineAges structure following it. try { - val iStream = ObjectInputStream(FileInputStream(storagePath)) + val file = dataPath.toFile() + val iStream = ObjectInputStream(FileInputStream(file)) val storedHeadId = iStream.readUTF() Logger.debug { "Stored repo head: $storedHeadId" } storedHead = revWalk.parseCommit(repo.resolve(storedHeadId)) @@ -267,19 +268,15 @@ class CodeLongevity(private val serverRepo: Repo, } // Store ages for subsequent runs. - if (Files.notExists(Paths.get(storageDir))) { - Files.createDirectories(Paths.get(storageDir)) - } try { - val oStream = ObjectOutputStream(FileOutputStream(storagePath)) + val file = dataPath.toFile() + val oStream = ObjectOutputStream(FileOutputStream(file)) oStream.writeUTF(head.getName()) oStream.writeObject(ageData) } catch(e: Exception) { - Logger.error( - e, - "Failed to save longevity data. CAUTION: data will be recomputed on a next run." - ) + Logger.error(e, "Failed to save longevity data. CAUTION: data " + + "will be recomputed on a next run.") } return ageData } @@ -288,7 +285,7 @@ class CodeLongevity(private val serverRepo: Repo, * Clears the stored age data if any. */ fun dropSavedData() { - File(storagePath).delete() + dataPath.toFile().delete() } /** diff --git a/src/main/kotlin/app/utils/FileHelper.kt b/src/main/kotlin/app/utils/FileHelper.kt index 0ec2a3e8..5c80fa2c 100644 --- a/src/main/kotlin/app/utils/FileHelper.kt +++ b/src/main/kotlin/app/utils/FileHelper.kt @@ -3,12 +3,45 @@ package app.utils +import java.io.File +import java.net.URLDecoder +import java.nio.file.Files import java.nio.file.Paths +import java.nio.file.Path object FileHelper { + private val dirName = "data" + private val jarPath = getJarPath() + private val settingsPath = jarPath.resolve(dirName) + + fun getPath(name: String, vararg parts: String): Path { + val path = settingsPath.resolve(Paths.get("", *parts)) + if (Files.notExists(path)) { + Files.createDirectories(path) + } + return path.resolve(name) + } + + fun getFile(name: String, vararg parts: String): File { + return getPath(name, *parts).toFile() + } + + fun isNotExists(name:String, vararg parts: String): Boolean { + return Files.notExists(getPath(name, *parts)) + } + fun getFileExtension(path: String): String { val fileName = Paths.get(path).fileName.toString() - return fileName.substringAfterLast( - delimiter = '.', missingDelimiterValue = "") + return fileName.substringAfterLast(delimiter = '.', + missingDelimiterValue = "") + } + + fun getJarPath(): Path { + val fullPathString = URLDecoder.decode(FileHelper::class.java + .protectionDomain.codeSource.location.toURI().path, "UTF-8") + val fullPath = Paths.get(fullPathString) + val root = fullPath.root + // Removing jar filename. + return root.resolve(fullPath.subpath(0, fullPath.nameCount - 1)) } } From eadcd83e7f5b45c3b426651ce1ab71dd53f70fdc Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Thu, 23 Nov 2017 23:34:19 +0300 Subject: [PATCH 2/3] wip: update install script --- src/install/install | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/install/install b/src/install/install index cb75f9f6..4c06317f 100644 --- a/src/install/install +++ b/src/install/install @@ -5,7 +5,7 @@ echo "Installing sourcerer app.." SERVER=$SERVER_EXT DOWNLOAD_URL=$SERVER/app/download -JAR_DIR=/usr/local/lib +JAR_DIR=/usr/local/lib/sourcerer SCRIPT_DIR=/usr/local/bin if [ -f $SCRIPT_DIR/sourcerer ] ; then @@ -16,6 +16,7 @@ if [ -f $SCRIPT_DIR/sourcerer ] ; then fi fi +mkdir -p $JAR_DIR curl -s $DOWNLOAD_URL > $JAR_DIR/sourcerer.jar cat < $SCRIPT_DIR/sourcerer @@ -28,7 +29,7 @@ if [ "\$1" = "--uninstall" ] ; then fi rm $SCRIPT_DIR/sourcerer - rm $JAR_DIR/sourcerer.jar + rm -r $JAR_DIR echo "Done!" exit From 05c818cd32bc65b6b19f38207ee8656b0c2ff45e Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Fri, 24 Nov 2017 15:30:36 +0300 Subject: [PATCH 3/3] chore: pr fix --- src/main/kotlin/app/extractors/ExtractorInterface.kt | 2 +- src/main/kotlin/app/utils/FileHelper.kt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/app/extractors/ExtractorInterface.kt b/src/main/kotlin/app/extractors/ExtractorInterface.kt index bd536441..0430e6c5 100644 --- a/src/main/kotlin/app/extractors/ExtractorInterface.kt +++ b/src/main/kotlin/app/extractors/ExtractorInterface.kt @@ -75,7 +75,7 @@ interface ExtractorInterface { } val fileName = name + pbExt - if (FileHelper.isNotExists(fileName, modelsDir)) { + if (FileHelper.notExists(fileName, modelsDir)) { Logger.info { "Downloading " + fileName } downloadModel(name) Logger.info { "Downloaded " + fileName } diff --git a/src/main/kotlin/app/utils/FileHelper.kt b/src/main/kotlin/app/utils/FileHelper.kt index 5c80fa2c..668348f1 100644 --- a/src/main/kotlin/app/utils/FileHelper.kt +++ b/src/main/kotlin/app/utils/FileHelper.kt @@ -9,6 +9,9 @@ import java.nio.file.Files import java.nio.file.Paths import java.nio.file.Path +/* + * Wrapper around Java Path and File classes to work the sourcerer's files. + */ object FileHelper { private val dirName = "data" private val jarPath = getJarPath() @@ -26,7 +29,7 @@ object FileHelper { return getPath(name, *parts).toFile() } - fun isNotExists(name:String, vararg parts: String): Boolean { + fun notExists(name:String, vararg parts: String): Boolean { return Files.notExists(getPath(name, *parts)) }