Skip to content

feat: refactor config and files storing (APP-154) #69

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 26, 2017
Merged

feat: refactor config and files storing (APP-154) #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
370eda5
feat: new path for config and data files
astansler Nov 23, 2017
eadcd83
wip: update install script
astansler Nov 23, 2017
05c818c
chore: pr fix
astansler Nov 24, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/install/install
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ echo "Installing sourcerer app.."

SERVER=$SERVER_EXT
DOWNLOAD_URL=$SERVER/app/download
JAR_DIR=/usr/local/lib
JAR_DIR=/usr/local/lib/sourcerer
SCRIPT_DIR=/usr/local/bin

if [ -f $SCRIPT_DIR/sourcerer ] ; then
Expand All @@ -16,6 +16,7 @@ if [ -f $SCRIPT_DIR/sourcerer ] ; then
fi
fi

mkdir -p $JAR_DIR
curl -s $DOWNLOAD_URL > $JAR_DIR/sourcerer.jar

cat <<EOF > $SCRIPT_DIR/sourcerer
Expand All @@ -28,7 +29,7 @@ if [ "\$1" = "--uninstall" ] ; then
fi

rm $SCRIPT_DIR/sourcerer
rm $JAR_DIR/sourcerer.jar
rm -r $JAR_DIR

echo "Done!"
exit
Expand Down
32 changes: 8 additions & 24 deletions src/main/kotlin/app/config/FileConfigurator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ package app.config

import app.Logger
import app.model.LocalRepo
import app.model.Repo
import app.model.User
import app.utils.FileHelper
import app.utils.Options
import app.utils.PasswordHelper
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility
Expand All @@ -20,7 +20,6 @@ import java.io.IOException
import java.nio.file.Files
import java.nio.file.InvalidPathException
import java.nio.file.NoSuchFileException
import java.nio.file.Paths
import java.util.UUID

/**
Expand All @@ -30,7 +29,7 @@ class FileConfigurator : Configurator {
/**
* Persistent configuration file name.
*/
private val CONFIG_FILE_NAME = ".sourcerer"
private val CONFIG_FILE_NAME = "config.yaml"

// Config levels are presented in priority decreasing order.

Expand All @@ -40,7 +39,7 @@ class FileConfigurator : Configurator {
private var current: Config = Config()

/**
* Persistent configuration saved in [userDir] in YAML format.
* Persistent configuration saved in config file in YAML format.
*/
private var persistent: Config = Config()

Expand All @@ -65,17 +64,6 @@ class FileConfigurator : Configurator {
*/
private var user: User = User()

/**
* User directory path is where persistent config stored.
*/
private val userDir = try {
System.getProperty("user.home")
}
catch (e: SecurityException) {
Logger.error(e, "Cannot access user directory")
null
}

/**
* Jackson's ObjectMapper.
*/
Expand Down Expand Up @@ -216,19 +204,15 @@ class FileConfigurator : Configurator {
}

/**
* Loads [persistent] configuration from config file stored in [userDir].
* Loads [persistent] configuration from config file.
*/
override fun loadFromFile() {
if (userDir == null) {
return
}

// Сonfig initialization in case an exception is thrown.
var loadConfig = Config()

try {
loadConfig = Files.newBufferedReader(Paths.get(userDir,
CONFIG_FILE_NAME)).use {
loadConfig = Files.newBufferedReader(FileHelper
.getPath(CONFIG_FILE_NAME)).use {
mapper.readValue(it, Config::class.java)
}
} catch (e: IOException) {
Expand All @@ -253,11 +237,11 @@ class FileConfigurator : Configurator {
}

/**
* Saves [persistent] configuration to config file stored in [userDir].
* Saves [persistent] configuration to config file.
*/
override fun saveToFile() {
try {
Files.newBufferedWriter(Paths.get(userDir, CONFIG_FILE_NAME)).use {
Files.newBufferedWriter(FileHelper.getPath(CONFIG_FILE_NAME)).use {
mapper.writeValue(it, persistent)
}
} catch (e: IOException) {
Expand Down
36 changes: 13 additions & 23 deletions src/main/kotlin/app/extractors/ExtractorInterface.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,18 @@ import app.BuildConfig
import app.Logger
import app.model.DiffFile
import app.model.CommitStats
import app.utils.FileHelper
import java.io.InputStream
import java.io.FileOutputStream
import java.nio.file.Files
import java.nio.file.Paths
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClientBuilder
import java.io.File

interface ExtractorInterface {
companion object {
private val librariesCache = hashMapOf<String, Set<String>>()
private val classifiersCache = hashMapOf<String, Classifier>()
private val modelsDir = "models"
private val pbExt = ".pb"

private fun getResource(path: String): InputStream {
return ExtractorInterface::class.java.classLoader
Expand All @@ -45,21 +45,17 @@ interface ExtractorInterface {
return importToLibrary
}

private fun downloadModel(name: String, outputDir: String) {
private fun downloadModel(name: String) {
val url = BuildConfig.LIBRARY_MODELS_URL + "$name.pb"
val outputPath = "$outputDir/$name.pb"

if (Files.notExists(Paths.get(outputDir))) {
Files.createDirectories(Paths.get(outputDir))
}

val file = FileHelper.getFile(name + pbExt, modelsDir)
val builder = HttpClientBuilder.create()
val client = builder.build()
try {
client.execute(HttpGet(url)).use { response ->
val entity = response.entity
if (entity != null) {
FileOutputStream(outputPath).use { outstream ->
FileOutputStream(file).use { outstream ->
entity.writeTo(outstream)
outstream.flush()
outstream.close()
Expand All @@ -78,22 +74,16 @@ interface ExtractorInterface {
return classifiersCache[name]!!
}

val pbDir = ".sourcerer/data/pb"
val pbPath = "$pbDir/$name.pb"

if (Files.notExists(Paths.get(pbDir))) {
Files.createDirectories(Paths.get(pbDir))
}

if (Files.notExists(Paths.get(pbPath))) {
Logger.info { "Downloading $name.pb" }
downloadModel(name, pbDir)
Logger.info { "Downloaded $name.pb" }
val fileName = name + pbExt
if (FileHelper.notExists(fileName, modelsDir)) {
Logger.info { "Downloading " + fileName }
downloadModel(name)
Logger.info { "Downloaded " + fileName }
}

Logger.info {"Loading $name evaluator" }
Logger.info { "Loading $name evaluator" }

val bytesArray = File(pbPath).readBytes()
val bytesArray = FileHelper.getFile(fileName, modelsDir).readBytes()
val classifier = Classifier(bytesArray)
classifiersCache.put(name, classifier)

Expand Down
23 changes: 10 additions & 13 deletions src/main/kotlin/app/hashers/CodeLongevity.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import app.api.Api
import app.model.Author
import app.model.Repo
import app.model.Fact
import app.utils.FileHelper
import app.utils.RepoHelper
import io.reactivex.Observable
import org.eclipse.jgit.diff.DiffFormatter
Expand Down Expand Up @@ -152,8 +153,7 @@ class CodeLongevity(private val serverRepo: Repo,
catch(e: Exception) { throw Exception("No branch") }

val df = DiffFormatter(DisabledOutputStream.INSTANCE)
val storageDir = ".sourcerer/data/longevity"
val storagePath = "$storageDir/${serverRepo.rehash}"
val dataPath = FileHelper.getPath(serverRepo.rehash, "longevity")

init {
df.setRepository(repo)
Expand Down Expand Up @@ -221,12 +221,13 @@ class CodeLongevity(private val serverRepo: Repo,
*/
fun scan() : CodeLineAges? {
var storedHead: RevCommit? = null
var ageData: CodeLineAges = CodeLineAges()
var ageData = CodeLineAges()

// Load existing age data if any. Expected format: commit id and
// CodeLineAges structure following it.
try {
val iStream = ObjectInputStream(FileInputStream(storagePath))
val file = dataPath.toFile()
val iStream = ObjectInputStream(FileInputStream(file))
val storedHeadId = iStream.readUTF()
Logger.debug { "Stored repo head: $storedHeadId" }
storedHead = revWalk.parseCommit(repo.resolve(storedHeadId))
Expand Down Expand Up @@ -267,19 +268,15 @@ class CodeLongevity(private val serverRepo: Repo,
}

// Store ages for subsequent runs.
if (Files.notExists(Paths.get(storageDir))) {
Files.createDirectories(Paths.get(storageDir))
}
try {
val oStream = ObjectOutputStream(FileOutputStream(storagePath))
val file = dataPath.toFile()
val oStream = ObjectOutputStream(FileOutputStream(file))
oStream.writeUTF(head.getName())
oStream.writeObject(ageData)
}
catch(e: Exception) {
Logger.error(
e,
"Failed to save longevity data. CAUTION: data will be recomputed on a next run."
)
Logger.error(e, "Failed to save longevity data. CAUTION: data " +
"will be recomputed on a next run.")
}
return ageData
}
Expand All @@ -288,7 +285,7 @@ class CodeLongevity(private val serverRepo: Repo,
* Clears the stored age data if any.
*/
fun dropSavedData() {
File(storagePath).delete()
dataPath.toFile().delete()
}

/**
Expand Down
40 changes: 38 additions & 2 deletions src/main/kotlin/app/utils/FileHelper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,48 @@

package app.utils

import java.io.File
import java.net.URLDecoder
import java.nio.file.Files
import java.nio.file.Paths
import java.nio.file.Path

/*
* Wrapper around Java Path and File classes to work the sourcerer's files.
*/
object FileHelper {
private val dirName = "data"
private val jarPath = getJarPath()
private val settingsPath = jarPath.resolve(dirName)

fun getPath(name: String, vararg parts: String): Path {
val path = settingsPath.resolve(Paths.get("", *parts))
if (Files.notExists(path)) {
Files.createDirectories(path)
}
return path.resolve(name)
}

fun getFile(name: String, vararg parts: String): File {
return getPath(name, *parts).toFile()
}

fun notExists(name:String, vararg parts: String): Boolean {
return Files.notExists(getPath(name, *parts))
}

fun getFileExtension(path: String): String {
val fileName = Paths.get(path).fileName.toString()
return fileName.substringAfterLast(
delimiter = '.', missingDelimiterValue = "")
return fileName.substringAfterLast(delimiter = '.',
missingDelimiterValue = "")
}

fun getJarPath(): Path {
val fullPathString = URLDecoder.decode(FileHelper::class.java
.protectionDomain.codeSource.location.toURI().path, "UTF-8")
val fullPath = Paths.get(fullPathString)
val root = fullPath.root
// Removing jar filename.
return root.resolve(fullPath.subpath(0, fullPath.nameCount - 1))
Copy link
Contributor

@asurkov asurkov Nov 24, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

might be unexpected behavior for the user, if the user moved a jar file, and we put all our stuff next to it, @sergey-surkov is it ok?

Copy link
Member Author

@astansler astansler Nov 24, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why would the user want to do it? We use bash script that points to a specific location of the JAR to run sourcerer, all will be broken anyway.

}
}