diff --git a/.gitignore b/.gitignore
index 928ac4c2..f45d90bc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,10 +1,14 @@
.DS_Store
.gradle/
.idea/
+.sourcerer
build/
sourcerer-app.iml
sourcerer-app.ipr
sourcerer-app.iws
+app.iml
+app.ipr
+app.iws
/confluence/target
/dependencies
/dist
@@ -16,4 +20,5 @@ sourcerer-app.iws
/ultimate/dependencies
/ultimate/ideaSDK
/ultimate/out
-/ultimate/tmp
\ No newline at end of file
+/ultimate/tmp
+tmp_repo
diff --git a/README.md b/README.md
index 26a55ecd..1b859fd2 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,27 @@
-# sourcerer-app
-Sourcerer hashes your git repos into intelligent engineering profiles.
+
+
+ 
+
+ Sourcerer App
+
+
+
+
+ Sourcerer app creates an intelligent profile for software engineers.
+ Get early access on Sourcerer.
+
+
+
+Prerequirements
+=================
+
+* Sourcerer alpha/beta access
+* Linux or macOS
+* Java Platform ([JRE](http://www.oracle.com/technetwork/java/javase/downloads/jre8-downloads-2133155.html) for Linux or [JDK](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) for macOS)
Install/uninstall
=================
-Sourcerer requires Java Platform installed on the system.
-* Proceed with JRE(http://www.oracle.com/technetwork/java/javase/downloads/jre8-downloads-2133155.html) if on Linux,
-* or with JDK(http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) if on OS X.
-
To install sourcerer run the following command:
```
diff --git a/build.gradle b/build.gradle
index e9c81745..8c6e9c69 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,15 +1,15 @@
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
buildscript {
- ext.kotlin_version = '1.1.4-2'
+ ext.kotlin_version = '1.1.51'
repositories {
mavenCentral()
jcenter()
}
dependencies {
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
- classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.1"
- classpath "org.junit.platform:junit-platform-gradle-plugin:1.0.0-RC2"
+ classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.3"
+ classpath "org.junit.platform:junit-platform-gradle-plugin:1.0.1"
}
}
@@ -17,8 +17,6 @@ plugins {
id 'de.fuerstenau.buildconfig' version '1.1.8'
}
-
-
apply plugin: "idea"
apply plugin: "java"
apply plugin: "kotlin"
@@ -27,19 +25,11 @@ apply plugin: "com.google.protobuf"
apply plugin: "org.junit.platform.gradle.plugin"
buildConfig {
- ext.environment = project.hasProperty('env') &&
- env == 'production' ? 'prod' : 'dev'
-
clsName = 'BuildConfig'
packageName = 'app'
// API.
- def apiBasePath = 'https://staging.eng.sourcerer.io/api/commit'
- if (ext.environment == 'prod') {
- apiBasePath = 'https://sourcerer.io/api/commit'
- }
- apiBasePath = project.hasProperty('api') ? api : apiBasePath
-
+ def apiBasePath = project.hasProperty('api') ? api : 'https://sourcerer.io/api/commit'
buildConfigField 'String', 'API_BASE_PATH', apiBasePath
// Common.
@@ -49,11 +39,21 @@ buildConfig {
buildConfigField 'int', 'VERSION_CODE', '1'
buildConfigField 'String', 'VERSION', '0.0.1'
+ // Logging.
+ buildConfigField 'String', 'LOG_LEVEL', project.hasProperty('log') ? log : 'info'
+
// Google Analytics.
buildConfigField 'String', 'GA_BASE_PATH', 'http://www.google-analytics.com'
buildConfigField 'String', 'GA_TRACKING_ID', 'UA-107129190-2'
buildConfigField 'boolean', 'IS_GA_ENABLED', 'true'
+ // Logging.
+ buildConfigField 'String', 'SENTRY_DSN', 'https://0263d6473bd24a9ba40e25aa5fb0a242:c5451dc815074bff8ce3fb9f0851f2f5@sentry.io/233260'
+ buildConfigField 'boolean', 'PRINT_STACK_TRACE', 'false'
+
+ // Models storage path.
+ buildConfigField 'String', 'LIBRARY_MODELS_URL', 'https://storage.googleapis.com/sourcerer-app/library-models/v1/'
+
buildConfig
}
@@ -65,39 +65,49 @@ junitPlatform {
}
}
+task cleanData {
+ delete 'build/libs/data'
+ delete 'build/kotlin/data'
+}
+
+test.dependsOn cleanData
+
mainClassName = "app.MainKt"
repositories {
mavenCentral()
jcenter()
maven { url "http://dl.bintray.com/jetbrains/spek" }
+ flatDir {
+ dirs 'libs'
+ }
}
dependencies {
compile "org.jetbrains.kotlin:kotlin-stdlib-jre8:$kotlin_version"
compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
compile "com.beust:jcommander:1.72"
- compile 'com.google.protobuf:protobuf-java:3.0.0'
- compile group: 'commons-codec', name: 'commons-codec', version: '1.5'
+ compile 'com.google.protobuf:protobuf-java:3.4.0'
+ compile 'commons-codec:commons-codec:1.5'
compile 'com.fasterxml.jackson.core:jackson-databind:2.8.9'
compile 'com.fasterxml.jackson.module:jackson-module-kotlin:2.8.9'
compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.9'
- compile "io.reactivex.rxjava2:rxjava:2.1.1"
- compile 'com.github.kittinunf.fuel:fuel:1.9.0'
- compile 'com.github.kittinunf.fuel:fuel-rxjava:1.9.0'
- compile group: 'org.eclipse.jgit', name: 'org.eclipse.jgit',
- version: '4.8.0.201706111038-r'
- compile "org.slf4j:slf4j-nop:1.7.2"
+ compile 'io.reactivex.rxjava2:rxjava:2.1.6'
+ compile 'com.github.kittinunf.fuel:fuel:1.11.0'
+ compile 'com.github.kittinunf.fuel:fuel-rxjava:1.11.0'
+ compile 'org.eclipse.jgit:org.eclipse.jgit:4.9.0.201710071750-r'
+ compile 'org.slf4j:slf4j-nop:1.7.2'
+ compile 'io.sentry:sentry:1.6.0'
testCompile 'org.jetbrains.kotlin:kotlin-test'
- testCompile 'org.jetbrains.spek:spek-api:1.1.4'
- testCompile 'org.junit.platform:junit-platform-runner:1.0.0-RC2'
- testRuntime 'org.jetbrains.spek:spek-junit-platform-engine:1.1.4'
+ testCompile 'org.jetbrains.spek:spek-api:1.1.5'
+ testCompile 'org.junit.platform:junit-platform-runner:1.0.1'
+ testRuntime 'org.jetbrains.spek:spek-junit-platform-engine:1.1.5'
}
protobuf {
protoc {
- artifact = "com.google.protobuf:protoc:3.0.0"
+ artifact = "com.google.protobuf:protoc:3.4.0"
}
}
diff --git a/deploy/Jenkinsfile b/deploy/Jenkinsfile
index 6385668c..86703db0 100644
--- a/deploy/Jenkinsfile
+++ b/deploy/Jenkinsfile
@@ -1,6 +1,6 @@
podTemplate(label: 'build-pod-sourcerer-app',
containers: [
- containerTemplate(name: 'jnlp', image: 'gcr.io/cloud-solutions-images/jenkins-k8s-slave', args: '${computer.jnlpmac} ${computer.name}'),
+ containerTemplate(name: 'jnlp', image: 'gcr.io/sourcerer-1377/jenkins-slave:v4', args: '${computer.jnlpmac} ${computer.name}'),
containerTemplate(name: 'gradle', image: 'gcr.io/sourcerer-1377/gradle:4.2.0', ttyEnabled: true, command: 'tail -f /dev/null')
],
envVars: [
@@ -12,11 +12,15 @@ podTemplate(label: 'build-pod-sourcerer-app',
]
) {
node('build-pod-sourcerer-app') {
- def namespace = 'staging'
- def benv = 'development'
+ def namespace = 'sandbox'
+ def plog = 'debug'
+
if (env.BRANCH_NAME == 'master') {
- benv = 'production'
namespace = 'production'
+ plog = 'info'
+ } else if (env.BRANCH_NAME == 'develop') {
+ namespace = 'staging'
+ plog = 'info'
}
stage('checkout') {
@@ -25,7 +29,7 @@ podTemplate(label: 'build-pod-sourcerer-app',
stage('build jar and test') {
container('gradle') {
- sh("ENV=${benv} ./do.sh build_jar_inside")
+ sh("LOG=${plog} NAMESPACE=${namespace} ./do.sh build_jar_inside")
}
}
diff --git a/deploy/production_env.sh b/deploy/production_env.sh
new file mode 100644
index 00000000..5f4be620
--- /dev/null
+++ b/deploy/production_env.sh
@@ -0,0 +1 @@
+export REPLICAS="2"
diff --git a/deploy/sandbox_env.sh b/deploy/sandbox_env.sh
new file mode 100644
index 00000000..065d597f
--- /dev/null
+++ b/deploy/sandbox_env.sh
@@ -0,0 +1 @@
+export REPLICAS="1"
diff --git a/deploy/sourcerer-app-staging.yaml b/deploy/sourcerer-app-staging.yaml
deleted file mode 100644
index d8b831f2..00000000
--- a/deploy/sourcerer-app-staging.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-kind: Service
-metadata:
- name: sourcerer-app
- labels:
- app: sourcerer-app
-spec:
- type: ClusterIP
- ports:
- - port: 80
- targetPort: 80
- selector:
- app: sourcerer-app
----
-apiVersion: apps/v1beta1
-kind: Deployment
-metadata:
- name: sourcerer-app
-spec:
- replicas: 1
- template:
- metadata:
- labels:
- app: sourcerer-app
- spec:
- containers:
- - name: sourcerer-app
- image: gcr.io/sourcerer-1377/sourcerer-app:CONTAINER_TAG
- imagePullPolicy: Always
- resources:
- requests:
- cpu: 100m
- memory: 100Mi
diff --git a/deploy/sourcerer-app-production.yaml b/deploy/sourcerer-app.yaml
similarity index 85%
rename from deploy/sourcerer-app-production.yaml
rename to deploy/sourcerer-app.yaml
index 25e08d6a..f25aee41 100644
--- a/deploy/sourcerer-app-production.yaml
+++ b/deploy/sourcerer-app.yaml
@@ -16,7 +16,7 @@ kind: Deployment
metadata:
name: sourcerer-app
spec:
- replicas: 2
+ replicas: $REPLICAS
template:
metadata:
labels:
@@ -24,7 +24,7 @@ spec:
spec:
containers:
- name: sourcerer-app
- image: gcr.io/sourcerer-1377/sourcerer-app:CONTAINER_TAG
+ image: gcr.io/sourcerer-1377/sourcerer-app:$CONTAINER_TAG
imagePullPolicy: Always
resources:
requests:
diff --git a/deploy/staging_env.sh b/deploy/staging_env.sh
new file mode 100644
index 00000000..065d597f
--- /dev/null
+++ b/deploy/staging_env.sh
@@ -0,0 +1 @@
+export REPLICAS="1"
diff --git a/do.sh b/do.sh
index 2acc09db..958dd401 100755
--- a/do.sh
+++ b/do.sh
@@ -4,6 +4,8 @@
#----- Helpers -----#
#-------------------#
+set -x
+
usage() {
echo "$0 [COMMAND] [ARGUMENTS]"
echo " Commands:"
@@ -22,8 +24,8 @@ shift
ARGUMENTS=${@}
TAG="${CONTAINER_TAG:-latest}"
-NAMESPACE="${NAMESPACE:-staging}"
-ENV="${ENV:-development}"
+NAMESPACE="${NAMESPACE:-sandbox}"
+LOG="${LOG:-debug}"
VOLUME="${BUILD_VOLUME:-$PWD}"
PROJECT=sourcerer-app
PORT=3182
@@ -36,12 +38,22 @@ GRADLE_VERSION=4.2.0
# run only inside build container
build_jar_inside() {
- gradle -Penv=$ENV build
+ if [ "$NAMESPACE" == "sandbox" ]; then
+ API="https://sandbox.eng.sourcerer.io/api/commit"
+ elif [ "$NAMESPACE" == "staging" ]; then
+ API="https://staging.eng.sourcerer.io/api/commit"
+ elif [ "$NAMESPACE" == "local" ]; then
+ API="http://localhost:3181"
+ else
+ API="https://sourcerer.io/api/commit"
+ fi
+ gradle -Plog=$LOG -Papi=$API build
}
build_jar() {
- docker run -i -v $VOLUME:/home/gradle/app --workdir=/home/gradle/app -e ENV=$ENV \
- gradle:$GRADLE_VERSION \
+ docker run -i -v $VOLUME:/home/gradle/app --workdir=/home/gradle/app \
+ -e LOG=$LOG -e NAMESPACE=$NAMESPACE \
+ gradle:$GRADLE_VERSION \
./do.sh build_jar_inside
}
@@ -50,7 +62,8 @@ build_prod_inside() {
}
deploy() {
- sed "s#CONTAINER_TAG#$TAG#" ./deploy/sourcerer-app-$NAMESPACE.yaml > /tmp/deploy.yaml
+ source ./deploy/${NAMESPACE}_env.sh
+ envsubst < ./deploy/sourcerer-app.yaml > /tmp/deploy.yaml
kubectl --namespace=$NAMESPACE apply -f /tmp/deploy.yaml
}
diff --git a/src/install/install b/src/install/install
index cb75f9f6..2e094e94 100644
--- a/src/install/install
+++ b/src/install/install
@@ -5,8 +5,8 @@ echo "Installing sourcerer app.."
SERVER=$SERVER_EXT
DOWNLOAD_URL=$SERVER/app/download
-JAR_DIR=/usr/local/lib
-SCRIPT_DIR=/usr/local/bin
+SCRIPT_DIR=$HOME/.sourcerer
+JAR_DIR=$SCRIPT_DIR
if [ -f $SCRIPT_DIR/sourcerer ] ; then
read -p "Previous version of sourcerer is detected. Reinstall it? [Y/n] " yesno < /dev/tty
@@ -16,6 +16,8 @@ if [ -f $SCRIPT_DIR/sourcerer ] ; then
fi
fi
+mkdir -p $SCRIPT_DIR
+mkdir -p $JAR_DIR
curl -s $DOWNLOAD_URL > $JAR_DIR/sourcerer.jar
cat < $SCRIPT_DIR/sourcerer
@@ -27,8 +29,9 @@ if [ "\$1" = "--uninstall" ] ; then
exit
fi
+ rm -f /usr/local/bin/sourcerer
rm $SCRIPT_DIR/sourcerer
- rm $JAR_DIR/sourcerer.jar
+ rm -r $JAR_DIR
echo "Done!"
exit
@@ -45,10 +48,24 @@ if ! which java > /dev/null ; then
exit 1
fi
-java -jar $JAR_DIR/sourcerer.jar "\$@"
+# Java 9 requires additional parameters.
+version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}')
+if [[ "$version" > "9." ]]; then # Version example for Java 9: 9.0.1
+ java --add-modules java.activation --add-opens java.base/java.nio=ALL-UNNAMED -jar $JAR_DIR/sourcerer.jar "\$@"
+else
+ java -jar $JAR_DIR/sourcerer.jar "\$@"
+fi
EOF
chmod +x $SCRIPT_DIR/sourcerer
-echo "Done!"
-echo "Run sourcerer to start hashing your repos!"
+
+{
+ rm -f /usr/local/bin/sourcerer 2> /dev/null &&
+ ln -s $SCRIPT_DIR/sourcerer /usr/local/bin/sourcerer 2> /dev/null &&
+ echo "Done!" &&
+ echo "Run sourcerer to start hashing your repos!"
+} || {
+ echo 'We installed app to $HOME/.sourcerer/sourcerer.'
+ echo 'You can add it to $PATH or ln it to /usr/local/bin'
+}
diff --git a/src/main/kotlin/app/Analytics.kt b/src/main/kotlin/app/Analytics.kt
index ecbe740e..96b0da01 100644
--- a/src/main/kotlin/app/Analytics.kt
+++ b/src/main/kotlin/app/Analytics.kt
@@ -51,7 +51,7 @@ object Analytics {
* - t: Hit Type - type of event
* - dp: Document Path - virtual url
*/
- private fun trackEvent(event: String, params: List = listOf()) {
+ fun trackEvent(event: String, params: List = listOf()) {
if (!IS_ENABLED || (username.isEmpty() && uuid.isEmpty())) {
return
}
@@ -75,44 +75,16 @@ object Analytics {
post(params + defaultParams.filter { !params.contains(it) }
+ idParams).responseString()
} catch (e: Throwable) {
- Logger.error("Error while sending error report", e, logOnly = true)
+ Logger.error(e, "Error while sending GA report", logOnly = true)
}
}
- fun trackStart() {
- trackEvent("start")
- }
-
- fun trackAuth() {
- trackEvent("auth")
- }
-
- fun trackConfigSetup() {
- trackEvent("config/setup")
- }
-
- fun trackConfigChanged() {
- trackEvent("config/changed")
- }
-
- fun trackHashingRepoSuccess() {
- trackEvent("hashing/repo/success")
- }
-
- fun trackHashingSuccess() {
- trackEvent("hashing/success")
- }
-
- fun trackError(e: Throwable? = null, code: String = "") {
- val url = if (e != null) getErrorUrl(e) else code
+ fun trackError(e: Throwable? = null) {
+ val url = if (e != null) getErrorUrl(e) else ""
val separator = if (url.isNotEmpty()) "/" else ""
trackEvent("error" + separator + url, listOf("t" to HIT_EXCEPTION))
}
- fun trackExit() {
- trackEvent("exit")
- }
-
private fun getErrorUrl(e: Throwable): String {
// Mapping for request exceptions.
when (e) {
diff --git a/src/main/kotlin/app/FactCodes.kt b/src/main/kotlin/app/FactCodes.kt
index ccc5834a..572e1c0e 100644
--- a/src/main/kotlin/app/FactCodes.kt
+++ b/src/main/kotlin/app/FactCodes.kt
@@ -1,11 +1,20 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
package app
object FactCodes {
- val COMMITS_DAY_WEEK = 1
- val COMMITS_DAY_TIME = 2
- val LINE_LONGEVITY = 3
- val LINE_LONGEVITY_REPO = 4
- val REPO_DATE_START = 5
- val REPO_DATE_END = 6
- val REPO_TEAM_SIZE = 7
+ val COMMIT_DAY_WEEK = 1 // Day of week fun fact and graph.
+ val COMMIT_DAY_TIME = 2 // Day time fun fact.
+ val COMMIT_LINE_NUM_AVG = 8 // Average number of lines per commit fun fact.
+ val COMMIT_NUM = 9 // Used for averaging COMMIT_LINE_NUM_AVG between repos.
+ // A map of line numbers to commits number. Used in a commit histogram.
+ val COMMIT_NUM_TO_LINE_NUM = 12
+ val LINE_LONGEVITY = 3 // Used for longevity graph.
+ val LINE_LONGEVITY_REPO = 4 // Used for longevity graph.
+ val LINE_LEN_AVG = 10 // Average length of line fun fact.
+ val LINE_NUM = 11 // Used for averaging LINE_LEN_AVG between repos.
+ val REPO_DATE_START = 5 // Repo summary info. Date of first contribution.
+ val REPO_DATE_END = 6 // Repo summary info. Date of last contribution.
+ val REPO_TEAM_SIZE = 7 // Repo summary info. Number of contributors.
}
diff --git a/src/main/kotlin/app/Logger.kt b/src/main/kotlin/app/Logger.kt
index 0b2d913f..ce29e243 100644
--- a/src/main/kotlin/app/Logger.kt
+++ b/src/main/kotlin/app/Logger.kt
@@ -3,77 +3,236 @@
package app
+import io.sentry.Sentry
+import io.sentry.context.Context
+import io.sentry.event.Breadcrumb
+import io.sentry.event.UserBuilder
+import io.sentry.event.BreadcrumbBuilder
+import java.util.*
+
+
/**
* Singleton class that logs events of different levels.
*/
object Logger {
+ object Events {
+ val START = "start"
+ val AUTH = "auth"
+ val CONFIG_SETUP = "config/setup"
+ val CONFIG_CHANGED = "config/changed"
+ val HASHING_REPO_SUCCESS = "hashing/repo/success"
+ val HASHING_SUCCESS = "hashing/success"
+ val EXIT = "exit"
+ }
+
/**
* Current log level. All that higher than this level will not be displayed.
*/
- const val LEVEL = 3
+ @kotlin.PublishedApi
+ internal val LEVEL : Int
/**
* Error level.
*/
- const val ERROR = 0
+ @kotlin.PublishedApi
+ internal const val ERROR = 0
/**
* Warning level.
*/
- const val WARN = 1
+ @kotlin.PublishedApi
+ internal const val WARN = 1
/**
* Information level.
*/
- const val INFO = 2
+ @kotlin.PublishedApi
+ internal const val INFO = 2
/**
* Debug level.
*/
- const val DEBUG = 3
+ @kotlin.PublishedApi
+ internal const val DEBUG = 3
+
+ /**
+ * Trace level. For extremely detailed and high volume debug logs.
+ */
+ @kotlin.PublishedApi
+ internal const val TRACE = 4
+
+ /**
+ * Print stack trace on error log.
+ */
+ private const val PRINT_STACK_TRACE = BuildConfig.PRINT_STACK_TRACE
+
+ /**
+ * Context of Sentry error reporting software for adding info.
+ */
+ private val sentryContext: Context
+
+ /**
+ * Username used for error reporting.
+ */
+ var username: String? = null
+ set(value) {
+ sentryContext.user = UserBuilder().setUsername(value).build()
+ Analytics.username = value ?: ""
+ }
+
+ var uuid: String? = null
+ set(value) {
+ Analytics.uuid = value ?: ""
+ }
+
+ init {
+ Sentry.init(BuildConfig.SENTRY_DSN)
+ sentryContext = Sentry.getContext()
+ addTags()
+ LEVEL = configLevelValue()
+ }
+
+ private fun configLevelValue() : Int {
+ val a = mapOf("trace" to TRACE, "debug" to DEBUG, "info" to INFO, "warn" to WARN, "error" to ERROR)
+ return a.getValue(BuildConfig.LOG_LEVEL)
+ }
+
+ /**
+ * Utils.
+ */
+ private fun Double.format(digits: Int, digitsFloating: Int) =
+ java.lang.String.format("%${digits}.${digitsFloating}f", this)
+
+ private fun generateIndent(num: Int): String {
+ return 0.rangeTo(num).fold("") { ind, _ -> ind + " " }
+ }
+
+ /**
+ * CLI messages and pretty printing.
+ */
+ fun print(message: Any, indentLine: Boolean = false) {
+ print(message.toString(), indentLine)
+ }
+
+ fun print(message: String, indentLine: Boolean = false) {
+ if (indentLine) {
+ println()
+ }
+ println(message)
+ }
+
+ fun printCommit(commitMessage: String, commitHash: String,
+ percents: Double) {
+ val percentsStr = percents.format(6, 2)
+ val hash = commitHash.substring(0, 7)
+ val messageTrim = if (commitMessage.length > 59) {
+ commitMessage.substring(0, 56).plus("...")
+ } else commitMessage
+ println(" [$percentsStr%] * $hash $messageTrim")
+ }
+
+ private val commitDetailIndent = generateIndent(10) + "|" +
+ generateIndent(8)
+ fun printCommitDetail(message: String) {
+ val messageTrim = if (message.length > 59) {
+ message.substring(0, 56).plus("...")
+ } else message
+ println(commitDetailIndent + messageTrim)
+ }
/**
* Log error message with exception info.
+ * Don't log private information with this method.
*
- * @property message the message for user and logs.
* @property e the exception if presented.
- * @property code the code of error if exception is not presented.
+ * @property message the message for user and logs.
+ * @property logOnly only log to console, no additional actions.
*/
- fun error(message: String, e: Throwable? = null, code: String = "",
- logOnly: Boolean = false) {
+ fun error(e: Throwable, message: String = "", logOnly: Boolean = false) {
+ val finalMessage = if (message.isNotBlank()) { message + ": " }
+ else { "" } + e.message
if (LEVEL >= ERROR) {
- println("[e] $message" + if (e != null) ": $e" else "")
+ println("[e] $finalMessage")
+ if (PRINT_STACK_TRACE) {
+ e.printStackTrace()
+ }
}
if (!logOnly) {
- Analytics.trackError(e = e, code = code)
- //TODO(anatoly): Add error tracking software.
+ Analytics.trackError(e)
+ Sentry.capture(e)
}
+ addBreadcrumb(finalMessage, Breadcrumb.Level.ERROR)
}
/**
- * Log warning message.
+ * Log warning message. Don't log private information with this method.
*/
- fun warn(message: String) {
+ inline fun warn(message: () -> String) {
+ val msg = message()
if (LEVEL >= WARN) {
- println("[w] $message.")
+ println("[w] $msg.")
}
+ addBreadcrumb(msg, Breadcrumb.Level.WARNING)
}
/**
- * Log information message.
+ * Log information message. Don't log private information with this method.
*/
- fun info(message: String) {
+ inline fun info(event: String = "", message: () -> String) {
+ val msg = message()
if (LEVEL >= INFO) {
- println("[i] $message.")
+ println("[i] $msg.")
+ }
+ if (event.isNotBlank()) {
+ Analytics.trackEvent(event)
}
+ addBreadcrumb(msg, Breadcrumb.Level.INFO)
}
/**
* Log debug message.
*/
- fun debug(message: String) {
+ inline fun debug(message: () -> String) {
if (LEVEL >= DEBUG) {
- println("[d] $message.")
+ println("[d] ${message()}.")
+ }
+ }
+
+ /**
+ * Log trace message.
+ */
+ inline fun trace(message: () -> String) {
+ if (LEVEL >= TRACE) {
+ println("[t] ${message()}.")
}
}
+
+ val isDebug: Boolean
+ inline get() = LEVEL >= DEBUG
+
+ @kotlin.PublishedApi
+ internal fun addBreadcrumb(message: String, level: Breadcrumb.Level) {
+ sentryContext.recordBreadcrumb(BreadcrumbBuilder()
+ .setMessage(message)
+ .setLevel(level)
+ .setTimestamp(Date())
+ .build())
+ }
+
+ private fun addTags() {
+ val default = "unavailable"
+ val osName = System.getProperty("os.name", default)
+ val osVersion = System.getProperty("os.version", default)
+ val javaVendor = System.getProperty("java.vendor", default)
+ val javaVersion = System.getProperty("java.version", default)
+
+ sentryContext.addTag("log-level", BuildConfig.LOG_LEVEL)
+ sentryContext.addTag("version", BuildConfig.VERSION)
+ sentryContext.addTag("version-code", BuildConfig.VERSION_CODE
+ .toString())
+ sentryContext.addTag("os-name", osName)
+ sentryContext.addTag("os-version", osVersion)
+ sentryContext.addTag("java-vendor", javaVendor)
+ sentryContext.addTag("java-version", javaVersion)
+ }
}
diff --git a/src/main/kotlin/app/Main.kt b/src/main/kotlin/app/Main.kt
index 72be7309..110f0cb5 100644
--- a/src/main/kotlin/app/Main.kt
+++ b/src/main/kotlin/app/Main.kt
@@ -19,8 +19,8 @@ import com.beust.jcommander.JCommander
import com.beust.jcommander.MissingCommandException
fun main(argv : Array) {
- Thread.setDefaultUncaughtExceptionHandler { _, e: Throwable? ->
- Logger.error("Uncaught exception", e)
+ Thread.setDefaultUncaughtExceptionHandler { _, e: Throwable ->
+ Logger.error(e, "Uncaught exception")
}
Main(argv)
}
@@ -30,8 +30,8 @@ class Main(argv: Array) {
private val api = ServerApi(configurator)
init {
- Analytics.uuid = configurator.getUuidPersistent()
- Analytics.trackStart()
+ Logger.uuid = configurator.getUuidPersistent()
+ Logger.info(Logger.Events.START) { "App started" }
val options = Options()
val commandAdd = CommandAdd()
@@ -64,13 +64,10 @@ class Main(argv: Array) {
else -> startUi()
}
} catch (e: MissingCommandException) {
- Logger.error(
- message = "No such command: ${e.unknownCommand}",
- code = "no-command"
- )
+ Logger.warn { "No such command: ${e.unknownCommand}" }
}
- Analytics.trackExit()
+ Logger.info(Logger.Events.EXIT) { "App finished" }
}
private fun startUi() {
@@ -84,12 +81,10 @@ class Main(argv: Array) {
localRepo.hashAllContributors = commandAdd.hashAll
configurator.addLocalRepoPersistent(localRepo)
configurator.saveToFile()
- println("Added git repository at $path.")
-
- Analytics.trackConfigChanged()
+ Logger.print("Added git repository at $path.")
+ Logger.info(Logger.Events.CONFIG_CHANGED) { "Config changed" }
} else {
- Logger.error(message = "No valid git repository found at $path.",
- code = "repo-invalid")
+ Logger.warn { "No valid git repository found at specified path" }
}
}
@@ -97,8 +92,7 @@ class Main(argv: Array) {
val (key, value) = commandOptions.pair
if (!arrayListOf("username", "password").contains(key)) {
- Logger.error(message = "No such key $key",
- code = "invalid-params")
+ Logger.warn { "No such key $key" }
return
}
@@ -109,7 +103,7 @@ class Main(argv: Array) {
configurator.saveToFile()
- Analytics.trackConfigChanged()
+ Logger.info(Logger.Events.CONFIG_CHANGED) { "Config changed" }
}
private fun doList() {
@@ -124,11 +118,11 @@ class Main(argv: Array) {
if (path != null) {
configurator.removeLocalRepoPersistent(LocalRepo(path))
configurator.saveToFile()
- println("Repository removed from tracking list.")
+ Logger.print("Repository removed from tracking list.")
- Analytics.trackConfigChanged()
+ Logger.info(Logger.Events.CONFIG_CHANGED) { "Config changed" }
} else {
- println("Repository not found in tracking list.")
+ Logger.print("Repository not found in tracking list.")
}
}
@@ -143,10 +137,12 @@ class Main(argv: Array) {
}
private fun showHelp(jc: JCommander) {
- println("Sourcerer hashes your git repositories into intelligent "
- + "engineering profiles. If you don't have an account, "
- + "please, proceed to http://sourcerer.io/register. More info at "
- + "http://sourcerer.io.")
+ Logger.print("Sourcerer hashes your git repositories into intelligent "
+ + "engineering profiles.")
+ Logger.print("If you don't have an account, please, proceed to " +
+ "https://sourcerer.io/join")
+ Logger.print("More info at https://sourcerer.io and " +
+ "https://github.com/sourcerer-io")
jc.usage() // Will show detailed info about usage based on annotations.
}
}
diff --git a/src/main/kotlin/app/api/Api.kt b/src/main/kotlin/app/api/Api.kt
index 5b58a2e5..ea33d98e 100644
--- a/src/main/kotlin/app/api/Api.kt
+++ b/src/main/kotlin/app/api/Api.kt
@@ -3,17 +3,24 @@
package app.api
+import app.model.Author
import app.model.Commit
import app.model.Fact
import app.model.Repo
import app.model.User
interface Api {
- fun authorize()
- fun getUser(): User
- fun getRepo(repoRehash: String): Repo
- fun postRepo(repo: Repo)
- fun postCommits(commitsList: List)
- fun deleteCommits(commitsList: List)
- fun postFacts(factsList: List)
+ companion object {
+ val OUT_OF_DATE = 1
+ }
+
+ fun authorize(): Result
+ fun getUser(): Result
+ fun postUser(user: User): Result
+ fun postRepo(repo: Repo): Result
+ fun postComplete(): Result
+ fun postCommits(commitsList: List): Result
+ fun deleteCommits(commitsList: List): Result
+ fun postFacts(factsList: List): Result
+ fun postAuthors(authorsList: List): Result
}
diff --git a/src/main/kotlin/app/api/ApiError.kt b/src/main/kotlin/app/api/ApiError.kt
new file mode 100644
index 00000000..1093cf1e
--- /dev/null
+++ b/src/main/kotlin/app/api/ApiError.kt
@@ -0,0 +1,75 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
+package app.api
+
+import app.Logger
+import app.model.Error
+import app.model.Errors
+import com.github.kittinunf.fuel.core.FuelError
+import com.google.protobuf.InvalidProtocolBufferException
+import java.nio.charset.Charset
+import java.security.InvalidParameterException
+
+class ApiError(exception: Exception) : Exception(exception.message) {
+ companion object {
+ private val AUTH_ERROR_CODES = listOf(401, 403)
+ }
+
+ // Response content.
+ var httpStatusCode: Int = 0
+ var httpResponseMessage: String = ""
+ var httpBodyMessage: String = ""
+
+ // Server errors from response.
+ var serverErrors = listOf()
+
+ // Type of errors.
+ var isParseError = false
+ var isAuthError: Boolean = false
+ get() = AUTH_ERROR_CODES.contains(httpStatusCode)
+
+ constructor(fuelError: FuelError) : this(fuelError as Exception) {
+ httpStatusCode = fuelError.response.statusCode
+ httpResponseMessage = fuelError.response.responseMessage
+ if (fuelError.response.headers["Content-Type"]
+ ?.contains("application/octet-stream") == true) {
+ try {
+ serverErrors = Errors(fuelError.response.data).errors
+ } catch (e: Exception) {
+ Logger.error(e, "Error while parsing errors from server")
+ }
+ } else {
+ httpBodyMessage = fuelError.response.data
+ .toString(Charset.defaultCharset())
+ }
+ }
+
+ constructor(parseException: InvalidProtocolBufferException) :
+ this(parseException as Exception) {
+ isParseError = true
+ }
+
+ constructor(parseException: InvalidParameterException) :
+ this(parseException as Exception) {
+ isParseError = true
+ }
+
+ fun isWithServerCode(serverErrorCode: Int): Boolean {
+ return serverErrors.find { error ->
+ error.code == serverErrorCode } != null
+ }
+}
+
+fun ApiError?.ifNotNullThrow() {
+ if (this != null) {
+ throw this
+ }
+}
+
+fun ApiError?.isWithServerCode(serverErrorCode: Int): Boolean {
+ if (this != null) {
+ return this.isWithServerCode(serverErrorCode)
+ }
+ return false
+}
diff --git a/src/main/kotlin/app/api/MockApi.kt b/src/main/kotlin/app/api/MockApi.kt
index 5ac03d23..667d38ad 100644
--- a/src/main/kotlin/app/api/MockApi.kt
+++ b/src/main/kotlin/app/api/MockApi.kt
@@ -4,6 +4,7 @@
package app.api
import app.Logger
+import app.model.Author
import app.model.Commit
import app.model.Repo
import app.model.Fact
@@ -17,43 +18,66 @@ class MockApi( // GET requests.
var receivedRepos: MutableList = mutableListOf()
var receivedAddedCommits: MutableList = mutableListOf()
var receivedFacts: MutableList = mutableListOf()
+ var receivedAuthors: MutableList = mutableListOf()
+ var receivedUsers: MutableList = mutableListOf()
+ var receivedComplete: Int = 0
// DELETE requests.
var receivedDeletedCommits: MutableList = mutableListOf()
- override fun authorize() {
- Logger.debug("MockApi: authorize request")
+ override fun authorize(): Result {
+ Logger.debug { "MockApi: authorize request" }
+ return Result()
}
- override fun getUser(): User {
- Logger.debug("MockApi: getUser request")
- return mockUser
+ override fun getUser(): Result {
+ Logger.debug { "MockApi: getUser request" }
+ return Result(mockUser)
}
- override fun getRepo(repoRehash: String): Repo {
- Logger.debug("MockApi: getRepo request")
- return mockRepo
+ override fun postUser(user: User): Result {
+ Logger.debug { "MockApi: postUser request" }
+ receivedUsers.add(user)
+ return Result()
}
- override fun postRepo(repo: Repo) {
- Logger.debug("MockApi: postRepo request ($repo)")
+ override fun postRepo(repo: Repo): Result {
+ Logger.debug { "MockApi: postRepo request" }
receivedRepos.add(repo)
+ return Result(mockRepo)
}
- override fun postCommits(commitsList: List) {
- Logger.debug("MockApi: postCommits request "
- + "(${commitsList.size} commits)")
+ override fun postComplete(): Result {
+ Logger.debug { "MockApi: postComplete request " }
+ receivedComplete++
+ return Result()
+ }
+
+ override fun postCommits(commitsList: List): Result {
+ Logger.debug {
+ "MockApi: postCommits request (${commitsList.size} commits)"
+ }
receivedAddedCommits.addAll(commitsList)
+ return Result()
}
- override fun deleteCommits(commitsList: List) {
- Logger.debug("MockApi: deleteCommits request "
- + "(${commitsList.size} commits)")
+ override fun deleteCommits(commitsList: List): Result {
+ Logger.debug {
+ "MockApi: deleteCommits request (${commitsList.size} commits)" }
receivedDeletedCommits.addAll(commitsList)
+ return Result()
}
- override fun postFacts(factsList: List) {
- Logger.debug("MockApi: postStats request (${factsList.size} stats)")
+ override fun postFacts(factsList: List): Result {
+ Logger.debug { "MockApi: postFacts request (${factsList.size} facts)" }
receivedFacts.addAll(factsList)
+ return Result()
+ }
+
+ override fun postAuthors(authorsList: List): Result {
+ Logger.debug { "MockApi: postAuthors request (${authorsList.size} " +
+ "stats)" }
+ receivedAuthors.addAll(authorsList)
+ return Result()
}
}
diff --git a/src/main/kotlin/app/api/Result.kt b/src/main/kotlin/app/api/Result.kt
new file mode 100644
index 00000000..9a56a26a
--- /dev/null
+++ b/src/main/kotlin/app/api/Result.kt
@@ -0,0 +1,19 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
+package app.api
+
+data class Result (val data: T? = null, val error: ApiError? = null) {
+ fun getOrThrow(): T {
+ if (error == null) {
+ return data!!
+ }
+ throw error
+ }
+
+ fun onErrorThrow() {
+ if (error != null) {
+ throw error
+ }
+ }
+}
diff --git a/src/main/kotlin/app/api/ServerApi.kt b/src/main/kotlin/app/api/ServerApi.kt
index 65ab05a0..25622c7e 100644
--- a/src/main/kotlin/app/api/ServerApi.kt
+++ b/src/main/kotlin/app/api/ServerApi.kt
@@ -6,13 +6,14 @@ package app.api
import app.BuildConfig
import app.Logger
import app.config.Configurator
+import app.model.Author
+import app.model.AuthorGroup
import app.model.Commit
import app.model.CommitGroup
import app.model.Fact
import app.model.FactGroup
import app.model.Repo
import app.model.User
-import app.utils.RequestException
import com.github.kittinunf.fuel.core.FuelManager
import com.github.kittinunf.fuel.core.Method
import com.github.kittinunf.fuel.core.Request
@@ -41,11 +42,10 @@ class ServerApi (private val configurator: Configurator) : Api {
}
private fun cookieResponseInterceptor() = { _: Request, res: Response ->
- val newToken = res.httpResponseHeaders[HEADER_SET_COOKIE]
+ val newToken = res.headers[HEADER_SET_COOKIE]
?.find { it.startsWith(KEY_TOKEN) }
if (newToken != null && newToken.isNotBlank()) {
- token = newToken.substringAfter(KEY_TOKEN)
- .substringBefore(';')
+ token = newToken.substringAfter(KEY_TOKEN).substringBefore(';')
}
res
}
@@ -76,15 +76,16 @@ class ServerApi (private val configurator: Configurator) : Api {
private fun createRequestGetToken(): Request {
return post("/auth").authenticate(username, password)
- .header(getVersionCodeHeader())
+ .header(getVersionCodeHeader())
}
private fun createRequestGetUser(): Request {
return get("/user")
}
- private fun createRequestGetRepo(repoRehash: String): Request {
- return get("/repo/$repoRehash")
+ private fun createRequestPostUser(user: User): Request {
+ return post("/user").header(getContentTypeHeader())
+ .body(user.serialize())
}
private fun createRequestPostRepo(repo: Repo): Request {
@@ -92,6 +93,10 @@ class ServerApi (private val configurator: Configurator) : Api {
.body(repo.serialize())
}
+ private fun createRequestPostComplete(): Request {
+ return post("/complete").header(getContentTypeHeader())
+ }
+
private fun createRequestPostCommits(commits: CommitGroup): Request {
return post("/commits").header(getContentTypeHeader())
.body(commits.serialize())
@@ -107,27 +112,34 @@ class ServerApi (private val configurator: Configurator) : Api {
.body(facts.serialize())
}
+ private fun createRequestPostAuthors(authors: AuthorGroup): Request {
+ return post("/authors").header(getContentTypeHeader())
+ .body(authors.serialize())
+ }
+
private fun makeRequest(request: Request,
requestName: String,
- parser: (ByteArray) -> T): T {
+ parser: (ByteArray) -> T): Result {
+ var error: ApiError? = null
+ var data: T? = null
+
try {
- Logger.debug("Request $requestName initialized")
+ Logger.debug { "Request $requestName initialized" }
val (_, res, result) = request.responseString()
val (_, e) = result
if (e == null) {
- Logger.debug("Request $requestName success")
- return parser(res.data)
+ Logger.debug { "Request $requestName success" }
+ data = parser(res.data)
} else {
- Logger.error("Request $requestName error", e)
- throw RequestException(e)
+ error = ApiError(e)
}
} catch (e: InvalidProtocolBufferException) {
- Logger.error("Request $requestName error while parsing", e)
- throw RequestException(e)
+ error = ApiError(e)
} catch (e: InvalidParameterException) {
- Logger.error("Request $requestName error while parsing", e)
- throw RequestException(e)
+ error = ApiError(e)
}
+
+ return Result(data, error)
}
private fun getVersionCodeHeader(): Pair {
@@ -138,43 +150,52 @@ class ServerApi (private val configurator: Configurator) : Api {
return Pair(HEADER_CONTENT_TYPE, HEADER_CONTENT_TYPE_PROTO)
}
- override fun authorize() {
+ override fun authorize(): Result {
return makeRequest(createRequestGetToken(), "getToken", {})
}
- override fun getUser(): User {
+ override fun getUser(): Result {
return makeRequest(createRequestGetUser(), "getUser",
{ body -> User(body) })
}
- override fun getRepo(repoRehash: String): Repo {
- if (repoRehash.isBlank()) {
+ override fun postUser(user: User): Result {
+ return makeRequest(createRequestPostUser(user), "postUser", {})
+ }
+
+ override fun postRepo(repo: Repo): Result {
+ if (repo.rehash.isBlank()) {
throw IllegalArgumentException()
}
- return makeRequest(createRequestGetRepo(repoRehash), "getRepo",
+ return makeRequest(createRequestPostRepo(repo), "getRepo",
{ body -> Repo(body) })
}
- override fun postRepo(repo: Repo) {
- makeRequest(createRequestPostRepo(repo),
- "postRepo", {})
+ override fun postComplete(): Result {
+ return makeRequest(createRequestPostComplete(),
+ "postComplete", {})
}
- override fun postCommits(commitsList: List) {
+ override fun postCommits(commitsList: List): Result {
val commits = CommitGroup(commitsList)
- makeRequest(createRequestPostCommits(commits),
- "postCommits", {})
+ return makeRequest(createRequestPostCommits(commits),
+ "postCommits", {})
}
- override fun deleteCommits(commitsList: List) {
+ override fun deleteCommits(commitsList: List): Result {
val commits = CommitGroup(commitsList)
- makeRequest(createRequestDeleteCommits(commits),
- "deleteCommits", {})
+ return makeRequest(createRequestDeleteCommits(commits),
+ "deleteCommits", {})
}
- override fun postFacts(factsList: List) {
+ override fun postFacts(factsList: List): Result {
val facts = FactGroup(factsList)
- makeRequest(createRequestPostFacts(facts), "postFacts", {})
+ return makeRequest(createRequestPostFacts(facts), "postFacts", {})
+ }
+
+ override fun postAuthors(authorsList: List): Result {
+ val authors = AuthorGroup(authorsList)
+ return makeRequest(createRequestPostAuthors(authors), "postAuthors", {})
}
}
diff --git a/src/main/kotlin/app/config/Configurator.kt b/src/main/kotlin/app/config/Configurator.kt
index 0cab1036..c9960ed0 100644
--- a/src/main/kotlin/app/config/Configurator.kt
+++ b/src/main/kotlin/app/config/Configurator.kt
@@ -4,7 +4,7 @@
package app.config
import app.model.LocalRepo
-import app.model.Repo
+import app.model.User
import app.utils.Options
interface Configurator {
@@ -13,7 +13,7 @@ interface Configurator {
fun getPassword(): String
fun isValidCredentials(): Boolean
fun getLocalRepos(): List
- fun getRepos(): List
+ fun getUser(): User
fun setUsernameCurrent(username: String)
fun setPasswordCurrent(password: String)
fun getUuidPersistent(): String
@@ -21,7 +21,7 @@ interface Configurator {
fun setPasswordPersistent(password: String)
fun addLocalRepoPersistent(localRepo: LocalRepo)
fun removeLocalRepoPersistent(localRepo: LocalRepo)
- fun setRepos(repos: List)
+ fun setUser(user: User)
fun isFirstLaunch(): Boolean
fun loadFromFile()
fun saveToFile()
diff --git a/src/main/kotlin/app/config/FileConfigurator.kt b/src/main/kotlin/app/config/FileConfigurator.kt
index 117e9b22..c90ebad2 100644
--- a/src/main/kotlin/app/config/FileConfigurator.kt
+++ b/src/main/kotlin/app/config/FileConfigurator.kt
@@ -5,7 +5,8 @@ package app.config
import app.Logger
import app.model.LocalRepo
-import app.model.Repo
+import app.model.User
+import app.utils.FileHelper
import app.utils.Options
import app.utils.PasswordHelper
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility
@@ -16,11 +17,9 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.kotlin.KotlinModule
import java.io.IOException
-import java.lang.IllegalStateException
import java.nio.file.Files
import java.nio.file.InvalidPathException
import java.nio.file.NoSuchFileException
-import java.nio.file.Paths
import java.util.UUID
/**
@@ -30,7 +29,7 @@ class FileConfigurator : Configurator {
/**
* Persistent configuration file name.
*/
- private val CONFIG_FILE_NAME = ".sourcerer"
+ private val CONFIG_FILE_NAME = "config.yaml"
// Config levels are presented in priority decreasing order.
@@ -40,7 +39,7 @@ class FileConfigurator : Configurator {
private var current: Config = Config()
/**
- * Persistent configuration saved in [userDir] in YAML format.
+ * Persistent configuration saved in config file in YAML format.
*/
private var persistent: Config = Config()
@@ -63,18 +62,7 @@ class FileConfigurator : Configurator {
/**
* Used to temporarily save list of repos that known by server.
*/
- private var repos: List = listOf()
-
- /**
- * User directory path is where persistent config stored.
- */
- private val userDir = try {
- System.getProperty("user.home")
- }
- catch (e: SecurityException) {
- Logger.error("Cannot access user directory", e)
- null
- }
+ private var user: User = User()
/**
* Jackson's ObjectMapper.
@@ -144,10 +132,10 @@ class FileConfigurator : Configurator {
}
/**
- * Gets list of temprorary saved repos.
+ * Gets temprorary saved user information.
*/
- override fun getRepos(): List {
- return repos
+ override fun getUser(): User {
+ return user
}
/**
@@ -200,10 +188,10 @@ class FileConfigurator : Configurator {
}
/**
- * Temporarily sets list of repos.
+ * Temporarily sets info about user.
*/
- override fun setRepos(repos: List) {
- this.repos = repos
+ override fun setUser(user: User) {
+ this.user = user
}
/**
@@ -216,62 +204,58 @@ class FileConfigurator : Configurator {
}
/**
- * Loads [persistent] configuration from config file stored in [userDir].
+ * Loads [persistent] configuration from config file.
*/
override fun loadFromFile() {
- if (userDir == null) {
- return
- }
-
// Сonfig initialization in case an exception is thrown.
var loadConfig = Config()
try {
- loadConfig = Files.newBufferedReader(Paths.get(userDir,
- CONFIG_FILE_NAME)).use {
+ loadConfig = Files.newBufferedReader(FileHelper
+ .getPath(CONFIG_FILE_NAME)).use {
mapper.readValue(it, Config::class.java)
}
} catch (e: IOException) {
- if(e is NoSuchFileException){
- Logger.info("No config file found")
+ if (e is NoSuchFileException){
+ Logger.warn { "No config file found" }
} else {
- Logger.error("Cannot access config file", e)
+ Logger.error(e, "Cannot access config file")
}
} catch (e: SecurityException) {
- Logger.error("Cannot access config file", e)
+ Logger.error(e, "Cannot access config file")
} catch (e: InvalidPathException) {
- Logger.error("Cannot access config file", e)
+ Logger.error(e, "Cannot access config file")
} catch (e: JsonParseException) {
- Logger.error("Cannot parse config file", e)
+ Logger.error(e, "Cannot parse config file")
} catch (e: JsonMappingException) {
- Logger.error("Cannot parse config file", e)
+ Logger.error(e, "Cannot parse config file")
} catch (e: IllegalStateException) {
- Logger.error("Cannot parse config file", e)
+ Logger.error(e, "Cannot parse config file")
}
persistent = loadConfig
}
/**
- * Saves [persistent] configuration to config file stored in [userDir].
+ * Saves [persistent] configuration to config file.
*/
override fun saveToFile() {
try {
- Files.newBufferedWriter(Paths.get(userDir, CONFIG_FILE_NAME)).use {
+ Files.newBufferedWriter(FileHelper.getPath(CONFIG_FILE_NAME)).use {
mapper.writeValue(it, persistent)
}
} catch (e: IOException) {
- Logger.error("Cannot save config file", e)
+ Logger.error(e, "Cannot save config file")
} catch (e: SecurityException) {
- Logger.error("Cannot save config file", e)
+ Logger.error(e, "Cannot save config file")
} catch (e: InvalidPathException) {
- Logger.error("Cannot save config file", e)
+ Logger.error(e, "Cannot save config file")
} catch (e: JsonParseException) {
- Logger.error("Cannot parse config file", e)
+ Logger.error(e, "Cannot parse config file")
} catch (e: JsonMappingException) {
- Logger.error("Cannot parse config file", e)
+ Logger.error(e, "Cannot parse config file")
} catch (e: IllegalStateException) {
- Logger.error("Cannot parse config file", e)
+ Logger.error(e, "Cannot parse config file")
}
}
diff --git a/src/main/kotlin/app/config/MockConfigurator.kt b/src/main/kotlin/app/config/MockConfigurator.kt
index 1b482c23..75c30bbf 100644
--- a/src/main/kotlin/app/config/MockConfigurator.kt
+++ b/src/main/kotlin/app/config/MockConfigurator.kt
@@ -4,14 +4,14 @@
package app.config
import app.model.LocalRepo
-import app.model.Repo
+import app.model.User
import app.utils.Options
class MockConfigurator(var mockUsername: String = "",
var mockPassword: String = "",
var mockIsValidCredentials: Boolean = true,
var mockIsFirstLaunch: Boolean = true,
- var mockRepos: MutableList = mutableListOf(),
+ var mockUser: User = User(),
var mockLocalRepos: MutableList =
mutableListOf(),
var uuid: String = "") : Configurator {
@@ -39,8 +39,8 @@ class MockConfigurator(var mockUsername: String = "",
return mockLocalRepos
}
- override fun getRepos(): List {
- return mockRepos
+ override fun getUser(): User {
+ return mockUser
}
override fun setUsernameCurrent(username: String) {
@@ -72,8 +72,8 @@ class MockConfigurator(var mockUsername: String = "",
mockPersistent.localRepos.remove(localRepo)
}
- override fun setRepos(repos: List) {
- mockRepos = repos.toMutableList()
+ override fun setUser(user: User) {
+ mockUser = user
}
override fun isFirstLaunch(): Boolean {
diff --git a/src/main/kotlin/app/extractors/CExtractor.kt b/src/main/kotlin/app/extractors/CExtractor.kt
index f68539d1..1f01d4a6 100644
--- a/src/main/kotlin/app/extractors/CExtractor.kt
+++ b/src/main/kotlin/app/extractors/CExtractor.kt
@@ -11,6 +11,9 @@ class CExtractor : ExtractorInterface {
companion object {
val LANGUAGE_NAME = "c"
val FILE_EXTS = listOf("c")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
}
override fun extract(files: List): List {
@@ -32,4 +35,18 @@ class CExtractor : ExtractorInterface {
return imports.toList()
}
+
+ override fun tokenize(line: String): List {
+ val importRegex = Regex("""^([^\n]*#include)\s[^\n]*""")
+ val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+ var newLine = importRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+ }
}
diff --git a/src/main/kotlin/app/extractors/CSharpExtractor.kt b/src/main/kotlin/app/extractors/CSharpExtractor.kt
index 95395eba..8f446255 100644
--- a/src/main/kotlin/app/extractors/CSharpExtractor.kt
+++ b/src/main/kotlin/app/extractors/CSharpExtractor.kt
@@ -9,9 +9,12 @@ import app.model.DiffFile
class CSharpExtractor : ExtractorInterface {
companion object {
- val LANGUAGE_NAME = "cs"
+ val LANGUAGE_NAME = "csharp"
val FILE_EXTS = listOf("cs")
val LIBRARIES = ExtractorInterface.getLibraries("cs")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
}
override fun extract(files: List): List {
@@ -37,4 +40,18 @@ class CSharpExtractor : ExtractorInterface {
return imports.toList()
}
+
+ override fun tokenize(line: String): List {
+ val importRegex = Regex("""^.*using\s+(\w+[.\w+]*)""")
+ val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+ var newLine = importRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+ }
}
diff --git a/src/main/kotlin/app/extractors/Classifier.kt b/src/main/kotlin/app/extractors/Classifier.kt
new file mode 100644
index 00000000..7c261ced
--- /dev/null
+++ b/src/main/kotlin/app/extractors/Classifier.kt
@@ -0,0 +1,59 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
+
+package app.extractors
+
+import app.ModelsProtos
+import com.google.protobuf.InvalidProtocolBufferException
+import java.security.InvalidParameterException
+
+class Classifier {
+ var tokens: List
+ var libraries: List
+ var idf: Map
+ var weights: Map>
+ var biases: Map
+
+ @Throws(InvalidParameterException::class)
+ constructor(proto: ModelsProtos.Classifier) {
+ tokens = proto.tokensList
+ libraries = proto.librariesList
+ idf = tokens.zip(proto.idfList).toMap()
+ weights = libraries.zip(proto.weightsList.partition(tokens.size)
+ .map {it: List -> tokens.zip(it).toMap()}).toMap()
+ biases = libraries.zip(proto.biasesList).toMap()
+ }
+
+ @Throws(InvalidProtocolBufferException::class)
+ constructor(bytes: ByteArray) : this(ModelsProtos.Classifier
+ .parseFrom(bytes))
+
+ fun evaluate(input: List): List {
+ val inputTokens = input.filter { it in tokens}
+ val tokensWithWeight = inputTokens.groupBy { it }
+ .map { (token, tokens) -> Pair(token, tokens.size * idf[token]!!) }
+ .toMap()
+ val norm = Math.sqrt(tokensWithWeight
+ .map { (_, tfidf) -> tfidf * tfidf }
+ .sum() + 1e-7)
+ val output = libraries.map {
+ Math.exp(tokensWithWeight
+ .map { (token, tfidf) -> tfidf / norm * weights[it]!![token]!! }
+ .sum() + biases[it]!!)
+ }
+ val norm2 = output.sum()
+ val probs = output.map { it / norm2 }
+
+ return probs
+ }
+
+ fun getCategories(): List {
+ return libraries
+ }
+
+ private fun List.partition(size: Int): List> {
+ return this.withIndex()
+ .groupBy { it.index / size }
+ .map { group -> group.value.map { it.value } }
+ }
+}
diff --git a/src/main/kotlin/app/extractors/CommonExtractor.kt b/src/main/kotlin/app/extractors/CommonExtractor.kt
new file mode 100644
index 00000000..d3036483
--- /dev/null
+++ b/src/main/kotlin/app/extractors/CommonExtractor.kt
@@ -0,0 +1,79 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
+package app.extractors
+
+import app.model.CommitStats
+import app.model.DiffFile
+
+class CommonExtractor : ExtractorInterface {
+ companion object {
+ val FILE_EXTS_MAP = lazy {
+ val reversedMap = mutableMapOf>()
+ reversedMap.put("actionscript", listOf("as"))
+ reversedMap.put("arduino", listOf("ino"))
+ reversedMap.put("assembly", listOf("asm", "s", "S"))
+ reversedMap.put("clojure", listOf("clj", "cljs", "cljc", "edn"))
+ reversedMap.put("cobol", listOf("cbl", "cob", "cpy"))
+ reversedMap.put("coffeescript", listOf("coffee", "litcoffee"))
+ reversedMap.put("cuda", listOf("cu", "cuh"))
+ reversedMap.put("d", listOf("d"))
+ reversedMap.put("emacslisp", listOf("el", "elc"))
+ reversedMap.put("erlang", listOf("erl", "hrl"))
+ reversedMap.put("forth", listOf("forth", "4TH"))
+ reversedMap.put("fortran", listOf("f", "for", "f90", "f95", "f03",
+ "f08", "f15"))
+ reversedMap.put("gradle", listOf("gradle"))
+ reversedMap.put("groovy", listOf("groovy"))
+ reversedMap.put("haskell", listOf("hs", "lhs"))
+ reversedMap.put("html", listOf("html", "htm"))
+ reversedMap.put("j", listOf("ijs"))
+ reversedMap.put("julia", listOf("jl"))
+ reversedMap.put("kotlin", listOf("kt"))
+ reversedMap.put("lisp", listOf("lisp", "lsp", "l"))
+ reversedMap.put("lua", listOf("lua"))
+ reversedMap.put("makefile", listOf("makefile"))
+ reversedMap.put("matlab", listOf("m", "mlx"))
+ reversedMap.put("maven", listOf("pom"))
+ reversedMap.put("ocaml", listOf("ml", "mli"))
+ reversedMap.put("pascal", listOf("pas"))
+ reversedMap.put("perl", listOf("pl", "PL"))
+ reversedMap.put("powershell", listOf("ps1", "psm1", "psd1"))
+ reversedMap.put("processing", listOf("pde"))
+ reversedMap.put("prolog", listOf("pl", "pro", "P"))
+ reversedMap.put("puppet", listOf("pp"))
+ reversedMap.put("r", listOf("r", "R"))
+ reversedMap.put("rust", listOf("rs"))
+ reversedMap.put("sas", listOf("sas"))
+ reversedMap.put("scala", listOf("scala", "sc"))
+ reversedMap.put("scheme", listOf("scm", "ss"))
+ reversedMap.put("shell", listOf("sh"))
+ reversedMap.put("sql", listOf("sql"))
+ reversedMap.put("tcl", listOf("tcl"))
+ reversedMap.put("tex", listOf("tex"))
+ reversedMap.put("typescript", listOf("ts", "tsx"))
+ reversedMap.put("verilog", listOf("v"))
+ reversedMap.put("vhdl", listOf("vhdl"))
+ reversedMap.put("viml", listOf("vim"))
+ reversedMap.put("visualbasic", listOf("bas"))
+ reversedMap.put("vue", listOf("vue"))
+
+ val map = hashMapOf()
+ reversedMap.forEach({ lang, exts ->
+ exts.forEach { ext -> map.put(ext, lang)}
+ })
+ map
+ }
+ }
+
+ override fun extract(files: List): List {
+ files.mapNotNull { file ->
+ val lang = FILE_EXTS_MAP.value[file.extension]
+ if (lang != null) {
+ file.language = lang
+ file
+ } else null
+ }
+ return super.extract(files)
+ }
+}
diff --git a/src/main/kotlin/app/extractors/CppExtractor.kt b/src/main/kotlin/app/extractors/CppExtractor.kt
index 29828cbd..2e40cb6f 100644
--- a/src/main/kotlin/app/extractors/CppExtractor.kt
+++ b/src/main/kotlin/app/extractors/CppExtractor.kt
@@ -11,6 +11,11 @@ class CppExtractor : ExtractorInterface {
companion object {
val LANGUAGE_NAME = "cpp"
val FILE_EXTS = listOf("cc", "cpp", "cxx", "c++")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
+ val MULTI_IMPORT_TO_LIB =
+ ExtractorInterface.getMultipleImportsToLibraryMap(LANGUAGE_NAME)
}
override fun extract(files: List): List {
@@ -30,6 +35,21 @@ class CppExtractor : ExtractorInterface {
}
}
- return imports.toList()
+ val libraries = imports.map { MULTI_IMPORT_TO_LIB.getOrDefault(it, it) }
+ return libraries
+ }
+
+ override fun tokenize(line: String): List {
+ val importRegex = Regex("""^([^\n]*#include)\s[^\n]*""")
+ val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+ var newLine = importRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
}
}
diff --git a/src/main/kotlin/app/extractors/EmptyExtractor.kt b/src/main/kotlin/app/extractors/EmptyExtractor.kt
deleted file mode 100644
index eeb4cb11..00000000
--- a/src/main/kotlin/app/extractors/EmptyExtractor.kt
+++ /dev/null
@@ -1,18 +0,0 @@
-// Copyright 2017 Sourcerer Inc. All Rights Reserved.
-// Author: Anatoly Kislov (anatoly@sourcerer.io)
-// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
-
-package app.extractors
-
-import app.model.CommitStats
-import app.model.DiffFile
-
-class EmptyExtractor : ExtractorInterface {
- override fun extract(files: List): List {
- return listOf()
- }
-
- override fun extractImports(fileContent: List): List {
- return listOf()
- }
-}
diff --git a/src/main/kotlin/app/extractors/Extractor.kt b/src/main/kotlin/app/extractors/Extractor.kt
index d90b537f..4fbc9d3b 100644
--- a/src/main/kotlin/app/extractors/Extractor.kt
+++ b/src/main/kotlin/app/extractors/Extractor.kt
@@ -10,7 +10,8 @@ import app.model.DiffFile
class Extractor : ExtractorInterface {
companion object {
val TYPE_LANGUAGE = 1
- val TYPE_KEYWORD = 2
+ val TYPE_LIBRARY = 2
+ val TYPE_KEYWORD = 3
val SEPARATOR = ">"
}
@@ -27,7 +28,7 @@ class Extractor : ExtractorInterface {
in GoExtractor.FILE_EXTS -> GoExtractor()
in ObjectiveCExtractor.FILE_EXTS -> ObjectiveCExtractor()
in SwiftExtractor.FILE_EXTS -> SwiftExtractor()
- else -> EmptyExtractor()
+ else -> CommonExtractor()
}
}
diff --git a/src/main/kotlin/app/extractors/ExtractorInterface.kt b/src/main/kotlin/app/extractors/ExtractorInterface.kt
index 165c923d..6fd89467 100644
--- a/src/main/kotlin/app/extractors/ExtractorInterface.kt
+++ b/src/main/kotlin/app/extractors/ExtractorInterface.kt
@@ -4,39 +4,220 @@
package app.extractors
+import app.BuildConfig
+import app.Logger
import app.model.DiffFile
import app.model.CommitStats
+import app.utils.FileHelper
+import java.io.InputStream
+import java.io.FileOutputStream
+import org.apache.http.client.methods.HttpGet
+import org.apache.http.impl.client.HttpClientBuilder
interface ExtractorInterface {
companion object {
- fun getLibraries(name: String): Set {
+ private val librariesCache = hashMapOf>()
+ private val classifiersCache = hashMapOf()
+ private val modelsDir = "models"
+ private val pbExt = ".pb"
+
+ private fun getResource(path: String): InputStream {
return ExtractorInterface::class.java.classLoader
- .getResourceAsStream("data/libraries/${name}_libraries.txt")
+ .getResourceAsStream(path)
+ }
+
+ fun getLibraries(name: String): Set {
+ if (librariesCache.containsKey(name)) {
+ return librariesCache[name]!!
+ }
+ val libraries = getResource("data/libraries/${name}_libraries.txt")
.bufferedReader().readLines().toSet()
+ librariesCache.put(name, libraries)
+ return libraries
+ }
+
+ fun getMultipleImportsToLibraryMap(name: String): Map {
+ val importToLibrary = getResource("data/imports/$name.txt")
+ .bufferedReader().readLines().map {
+ val mapping = it.split(":")
+ Pair(mapping[0], mapping[1])
+ }.toMap()
+ return importToLibrary
+ }
+
+ private fun downloadModel(name: String) {
+ val url = BuildConfig.LIBRARY_MODELS_URL + "$name.pb"
+
+ val file = FileHelper.getFile(name + pbExt, modelsDir)
+ val builder = HttpClientBuilder.create()
+ val client = builder.build()
+ try {
+ client.execute(HttpGet(url)).use { response ->
+ val entity = response.entity
+ if (entity != null) {
+ FileOutputStream(file).use { outstream ->
+ entity.writeTo(outstream)
+ outstream.flush()
+ outstream.close()
+ }
+ }
+
+ }
+ }
+ catch (e: Exception) {
+ Logger.error(e, "Failed to download $name model")
+ }
+ }
+
+ fun getLibraryClassifier(name: String): Classifier {
+ if (classifiersCache.containsKey(name)) {
+ return classifiersCache[name]!!
+ }
+
+ val fileName = name + pbExt
+ if (FileHelper.notExists(fileName, modelsDir)) {
+ Logger.info { "Downloading " + fileName }
+ downloadModel(name)
+ Logger.info { "Downloaded " + fileName }
+ }
+
+ Logger.info { "Loading $name evaluator" }
+
+ val bytesArray = FileHelper.getFile(fileName, modelsDir).readBytes()
+ val classifier = Classifier(bytesArray)
+ classifiersCache.put(name, classifier)
+
+ Logger.info { "$name evaluator ready" }
+
+ return classifier
}
}
fun extract(files: List): List {
+ val langStats = files.filter { file -> file.language.isNotBlank() }
+ .groupBy { file -> file.language }
+ .map { (language, files) -> CommitStats(
+ numLinesAdded = files.fold(0) { total, file ->
+ total + file.getAllAdded().size },
+ numLinesDeleted = files.fold(0) { total, file ->
+ total + file.getAllDeleted().size },
+ type = Extractor.TYPE_LANGUAGE,
+ tech = language)
+ }
+
files.map { file ->
file.old.imports = extractImports(file.old.content)
file.new.imports = extractImports(file.new.content)
file
}
- return files.filter { file -> file.language.isNotBlank() }
- .groupBy { file -> file.language }
- .map { (language, files) -> CommitStats(
- numLinesAdded = files.fold(0) { total, file ->
- total + file.getAllAdded().size },
- numLinesDeleted = files.fold(0) { total, file ->
- total + file.getAllDeleted().size },
- type = Extractor.TYPE_LANGUAGE,
- tech = language)}
+ val oldLibraryToCount = mutableMapOf()
+ val newLibraryToCount = mutableMapOf()
+ val oldFilesImports = files.fold(mutableSetOf()) { acc, file ->
+ acc.addAll(file.old.imports)
+ acc
+ }
+ val newFilesImports = files.fold(mutableSetOf()) { acc, file ->
+ acc.addAll(file.new.imports)
+ acc
+ }
+
+ // Skip library stats calculation if no imports found.
+ if (oldFilesImports.isEmpty() && newFilesImports.isEmpty()) {
+ return langStats
+ }
+
+ oldFilesImports.forEach { oldLibraryToCount[it] = 0}
+ newFilesImports.forEach { newLibraryToCount[it] = 0}
+
+ files.filter { file -> file.language.isNotBlank() }
+ .forEach { file ->
+ val oldFileLibraries = mutableListOf()
+ file.getAllDeleted().forEach {
+ val lineLibs = getLineLibraries(it, file.old.imports)
+ oldFileLibraries.addAll(lineLibs)
+ }
+ file.old.imports.forEach { import ->
+ val numLines = oldFileLibraries.count { it == import }
+ oldLibraryToCount[import] =
+ oldLibraryToCount[import] as Int + numLines
+ }
+
+ val newFileLibraries = mutableListOf()
+ file.getAllAdded().forEach {
+ val lineLibs = getLineLibraries(it, file.new.imports)
+ newFileLibraries.addAll(lineLibs)
+ }
+ file.new.imports.forEach { import ->
+ val numLines = newFileLibraries.count { it == import }
+ newLibraryToCount[import] =
+ newLibraryToCount[import] as Int + numLines
+ }
+ }
+
+ val allImports = mutableSetOf()
+ allImports.addAll(oldFilesImports + newFilesImports)
+
+ val libraryStats = allImports.map { CommitStats(
+ numLinesAdded = newLibraryToCount.getOrDefault(it, 0),
+ numLinesDeleted = oldLibraryToCount.getOrDefault(it, 0),
+ type = Extractor.TYPE_LIBRARY,
+ tech = it)
+ }.filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
+
+ return langStats + libraryStats
}
fun extractImports(fileContent: List): List {
return listOf()
}
+ fun tokenize(line: String): List {
+ val stringRegex = Regex("""(".+?"|'.+?')""")
+ val newLine = stringRegex.replace(line, "")
+ //TODO(lyaronskaya): multiline comment regex
+ val splitRegex =
+ Regex("""\s|,|;|\*|\n|\(|\)|\[|]|\{|}|\+|=|&|\$|!=|\.|>|<|#|@|:|\?|!""")
+ val tokens = splitRegex.split(newLine)
+ .filter { it.isNotBlank() && !it.contains('"') && !it.contains('\'')
+ && it != "-" && it != "@"}
+ return tokens
+ }
+
+ fun getLineLibraries(line: String, fileLibraries: List):
+ List {
+ return listOf()
+ }
+
+ fun getLineLibraries(line: String,
+ fileLibraries: List,
+ evaluator: Classifier,
+ languageLabel: String): List {
+ val probabilities = evaluator.evaluate(tokenize(line))
+ val libraries = evaluator.getCategories()
+
+ val maxProbability = probabilities.max() as Double
+ val maxProbabilityCategory =
+ libraries[probabilities.indexOf(maxProbability)]
+ val selectedCategories = libraries.filter {
+ probabilities[libraries.indexOf(it)] >= 0.2 * maxProbability
+ }
+
+ if (maxProbabilityCategory == languageLabel) {
+ return emptyList()
+ }
+
+ // For C language.
+ // Consider line with language label being the one with high probability
+ // as not having library.
+ // Keep it while the number of libraries is small.
+ if (languageLabel == CExtractor.LANGUAGE_NAME &&
+ languageLabel in selectedCategories) {
+ return emptyList()
+ }
+
+ val lineLibraries = fileLibraries.filter { it in selectedCategories }
+ return lineLibraries
+ }
}
diff --git a/src/main/kotlin/app/extractors/GoExtractor.kt b/src/main/kotlin/app/extractors/GoExtractor.kt
index bd2862a4..c10a6184 100644
--- a/src/main/kotlin/app/extractors/GoExtractor.kt
+++ b/src/main/kotlin/app/extractors/GoExtractor.kt
@@ -11,6 +11,9 @@ class GoExtractor : ExtractorInterface {
companion object {
val LANGUAGE_NAME = "go"
val FILE_EXTS = listOf("go")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
}
override fun extract(files: List): List {
@@ -41,4 +44,18 @@ class GoExtractor : ExtractorInterface {
return imports.toList()
}
+
+ override fun tokenize(line: String): List {
+ val importRegex = Regex("""^(.*import)\s[^\n]*""")
+ val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+ var newLine = importRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+ }
}
diff --git a/src/main/kotlin/app/extractors/JavaExtractor.kt b/src/main/kotlin/app/extractors/JavaExtractor.kt
index f7a57e58..5b990421 100644
--- a/src/main/kotlin/app/extractors/JavaExtractor.kt
+++ b/src/main/kotlin/app/extractors/JavaExtractor.kt
@@ -20,10 +20,13 @@ class JavaExtractor : ExtractorInterface {
"extends", "int", "short", "try", "char", "final", "interface",
"static", "void", "class", "finally", "long", "strictfp",
"volatile", "const", "float", "native", "super", "while")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
}
override fun extract(files: List): List {
- files.map { file -> file.language = GoExtractor.LANGUAGE_NAME }
+ files.map { file -> file.language = LANGUAGE_NAME }
val stats = super.extract(files).toMutableList()
@@ -72,4 +75,18 @@ class JavaExtractor : ExtractorInterface {
return imports.toList()
}
+
+ override fun tokenize(line: String): List {
+ val importRegex = Regex("""^(.*import)\s[^\n]*""")
+ val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+ var newLine = importRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+ }
}
diff --git a/src/main/kotlin/app/extractors/JavascriptExtractor.kt b/src/main/kotlin/app/extractors/JavascriptExtractor.kt
index a5d3a2df..8743b9a9 100644
--- a/src/main/kotlin/app/extractors/JavascriptExtractor.kt
+++ b/src/main/kotlin/app/extractors/JavascriptExtractor.kt
@@ -9,9 +9,12 @@ import app.model.DiffFile
class JavascriptExtractor : ExtractorInterface {
companion object {
- val LANGUAGE_NAME = "js"
+ val LANGUAGE_NAME = "javascript"
val FILE_EXTS = listOf("js")
val LIBRARIES = ExtractorInterface.getLibraries("js")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
}
override fun extract(files: List): List {
@@ -23,11 +26,17 @@ class JavascriptExtractor : ExtractorInterface {
val imports = mutableSetOf()
val splitRegex =
- Regex("""\s+|,|;|:|\\*|\n|\(|\)|\\[|]|\{|}|\+|=|\.|>|<|#|@|\$""")
- val fileTokens = fileContent.joinToString(separator = " ")
+ Regex("""\s+|,|;|:|\*|\n|\(|\)|\\[|]|\{|}|\+|=|\.|>|<|#|@|\$""")
+ val fileTokens = fileContent.joinToString(separator = " ").toLowerCase()
.split(splitRegex)
imports.addAll(fileTokens.filter { token -> token in LIBRARIES })
return imports.toList()
}
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+ }
}
diff --git a/src/main/kotlin/app/extractors/ObjectiveCExtractor.kt b/src/main/kotlin/app/extractors/ObjectiveCExtractor.kt
index 4eda9282..62f934e5 100644
--- a/src/main/kotlin/app/extractors/ObjectiveCExtractor.kt
+++ b/src/main/kotlin/app/extractors/ObjectiveCExtractor.kt
@@ -11,6 +11,9 @@ class ObjectiveCExtractor : ExtractorInterface {
companion object {
val LANGUAGE_NAME = "objectivec"
val FILE_EXTS = listOf("m", "mm")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
}
override fun extract(files: List): List {
@@ -36,4 +39,18 @@ class ObjectiveCExtractor : ExtractorInterface {
return imports.toList()
}
+
+ override fun tokenize(line: String): List {
+ val importRegex = Regex("""^([^\n]*[#@](import|include))\s[^\n]*""")
+ val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+ var newLine = importRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+ }
}
diff --git a/src/main/kotlin/app/extractors/PhpExtractor.kt b/src/main/kotlin/app/extractors/PhpExtractor.kt
index 66832489..48fb5c51 100644
--- a/src/main/kotlin/app/extractors/PhpExtractor.kt
+++ b/src/main/kotlin/app/extractors/PhpExtractor.kt
@@ -11,6 +11,9 @@ class PhpExtractor : ExtractorInterface {
companion object {
val LANGUAGE_NAME = "php"
val FILE_EXTS = listOf("php", "phtml", "php4", "php3", "php5", "phps")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
}
override fun extract(files: List): List {
@@ -34,4 +37,18 @@ class PhpExtractor : ExtractorInterface {
return imports.toList()
}
+
+ override fun tokenize(line: String): List {
+ val importRegex = Regex("""^(.*require|require_once|include|include_once|use)\s[^\n]*""")
+ val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+ var newLine = importRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+ }
}
diff --git a/src/main/kotlin/app/extractors/PythonExtractor.kt b/src/main/kotlin/app/extractors/PythonExtractor.kt
index 139cc9f2..4b7fe0be 100644
--- a/src/main/kotlin/app/extractors/PythonExtractor.kt
+++ b/src/main/kotlin/app/extractors/PythonExtractor.kt
@@ -11,6 +11,11 @@ class PythonExtractor : ExtractorInterface {
companion object {
val LANGUAGE_NAME = "python"
val FILE_EXTS = listOf("py", "py3")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
+ val MULTI_IMPORT_TO_LIB =
+ ExtractorInterface.getMultipleImportsToLibraryMap(LANGUAGE_NAME)
}
override fun extract(files: List): List {
@@ -32,6 +37,26 @@ class PythonExtractor : ExtractorInterface {
}
}
- return imports.toList()
+ var libraries = imports.map { MULTI_IMPORT_TO_LIB.getOrDefault(it, it) }
+ .filter { !it.endsWith("pb")}.toMutableList()
+ if (libraries.size < imports.size) {
+ libraries.add("protobuf")
+ }
+ return libraries
+
+ }
+
+ override fun tokenize(line: String): List {
+ val docImportRegex = Regex("""^([^\n]*#|\s*\"\"\"|\s*import|\s*from)[^\n]*""")
+ val commentRegex = Regex("""^(.*#).*""")
+ var newLine = docImportRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
}
}
diff --git a/src/main/kotlin/app/extractors/RubyExtractor.kt b/src/main/kotlin/app/extractors/RubyExtractor.kt
index 95ec6a81..4166e52d 100644
--- a/src/main/kotlin/app/extractors/RubyExtractor.kt
+++ b/src/main/kotlin/app/extractors/RubyExtractor.kt
@@ -11,6 +11,9 @@ class RubyExtractor : ExtractorInterface {
companion object {
val LANGUAGE_NAME = "ruby"
val FILE_EXTS = listOf("rb", "rbw")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
}
override fun extract(files: List): List {
@@ -32,4 +35,18 @@ class RubyExtractor : ExtractorInterface {
return imports.toList()
}
+
+ override fun tokenize(line: String): List {
+ val importRegex = Regex("""(require\s+'(\w+)'|load\s+'(\w+)\.\w+')""")
+ val commentRegex = Regex("""^([^\n]*#)[^\n]*""")
+ var newLine = importRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+ }
}
diff --git a/src/main/kotlin/app/extractors/SwiftExtractor.kt b/src/main/kotlin/app/extractors/SwiftExtractor.kt
index 742f2c29..9c044a43 100644
--- a/src/main/kotlin/app/extractors/SwiftExtractor.kt
+++ b/src/main/kotlin/app/extractors/SwiftExtractor.kt
@@ -11,6 +11,9 @@ class SwiftExtractor : ExtractorInterface {
companion object {
val LANGUAGE_NAME = "swift"
val FILE_EXTS = listOf("swift")
+ val evaluator by lazy {
+ ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+ }
}
override fun extract(files: List): List {
@@ -32,4 +35,18 @@ class SwiftExtractor : ExtractorInterface {
return imports.toList()
}
+
+ override fun tokenize(line: String): List {
+ val importRegex = Regex("""^(.*import)\s[^\n]*""")
+ val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+ var newLine = importRegex.replace(line, "")
+ newLine = commentRegex.replace(newLine, "")
+ return super.tokenize(newLine)
+ }
+
+ override fun getLineLibraries(line: String,
+ fileLibraries: List): List {
+
+ return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+ }
}
diff --git a/src/main/kotlin/app/hashers/CodeLongevity.kt b/src/main/kotlin/app/hashers/CodeLongevity.kt
index 6553dab8..078ea535 100644
--- a/src/main/kotlin/app/hashers/CodeLongevity.kt
+++ b/src/main/kotlin/app/hashers/CodeLongevity.kt
@@ -9,6 +9,7 @@ import app.api.Api
import app.model.Author
import app.model.Repo
import app.model.Fact
+import app.utils.FileHelper
import app.utils.RepoHelper
import io.reactivex.Observable
import org.eclipse.jgit.diff.DiffFormatter
@@ -22,9 +23,9 @@ import org.eclipse.jgit.revwalk.RevWalk
import org.eclipse.jgit.treewalk.TreeWalk
import org.eclipse.jgit.util.io.DisabledOutputStream
-import java.io.InputStream
import java.io.File
import java.io.FileInputStream
+import java.io.FileNotFoundException
import java.io.FileOutputStream
import java.io.ObjectOutputStream
import java.io.ObjectInputStream
@@ -143,13 +144,16 @@ class CodeLineAges : Serializable {
*/
class CodeLongevity(private val serverRepo: Repo,
private val emails: HashSet,
- git: Git) {
+ git: Git,
+ private val onError: (Throwable) -> Unit) {
val repo: Repository = git.repository
+ val revWalk = RevWalk(repo)
val head: RevCommit =
- RevWalk(repo).parseCommit(repo.resolve(RepoHelper.MASTER_BRANCH))
+ try { revWalk.parseCommit(repo.resolve(RepoHelper.MASTER_BRANCH)) }
+ catch(e: Exception) { throw Exception("No branch") }
+
val df = DiffFormatter(DisabledOutputStream.INSTANCE)
- val storageDir = ".sourcerer/data/longevity"
- val storagePath = "$storageDir/${serverRepo.rehash}"
+ val dataPath = FileHelper.getPath(serverRepo.rehash, "longevity")
init {
df.setRepository(repo)
@@ -192,8 +196,9 @@ class CodeLongevity(private val serverRepo: Repo,
code = FactCodes.LINE_LONGEVITY_REPO,
value = repoAvg.toString()))
val repoAvgDays = repoAvg / secondsInDay
- Logger.info("Repo average code line age is $repoAvgDays days, "
- + "lines total: $repoTotal")
+ Logger.info {
+ "Repo average code line age is $repoAvgDays days, lines total: $repoTotal"
+ }
for (email in emails) {
val aggrAge = aggrAges[email] ?: CodeLineAges.AggrAge()
@@ -206,8 +211,8 @@ class CodeLongevity(private val serverRepo: Repo,
}
if (stats.size > 0) {
- api.postFacts(stats)
- Logger.debug("Sent ${stats.size} stats to server")
+ api.postFacts(stats).onErrorThrow()
+ Logger.info { "Sent ${stats.size} facts to server" }
}
}
@@ -215,19 +220,33 @@ class CodeLongevity(private val serverRepo: Repo,
* Scans the repo to extract code line ages.
*/
fun scan() : CodeLineAges? {
- // Load existing age data if any.
- val iStream = try { ObjectInputStream(FileInputStream(storagePath)) }
- catch(e : Exception ) { null }
- val storedHead = iStream?.readObject() as RevCommit?
- if (storedHead == head) {
- return null
+ var storedHead: RevCommit? = null
+ var ageData = CodeLineAges()
+
+ // Load existing age data if any. Expected format: commit id and
+ // CodeLineAges structure following it.
+ try {
+ val file = dataPath.toFile()
+ val iStream = ObjectInputStream(FileInputStream(file))
+ val storedHeadId = iStream.readUTF()
+ Logger.debug { "Stored repo head: $storedHeadId" }
+ storedHead = revWalk.parseCommit(repo.resolve(storedHeadId))
+ if (storedHead == head) {
+ return null
+ }
+ ageData = (iStream.readObject() ?: CodeLineAges()) as CodeLineAges
+ }
+ catch(e: FileNotFoundException) { }
+ catch(e: Exception) {
+ Logger.error(
+ e,
+ "Failed to read longevity data. CAUTION: data will be recomputed."
+ )
}
-
- val ageData = (iStream?.readObject() ?: CodeLineAges()) as CodeLineAges
// Update ages.
getLinesObservable(storedHead).blockingSubscribe { line ->
- Logger.debug("Scanning: ${line}")
+ Logger.trace { "Scanning: ${line}" }
if (line.to.isDeleted) {
var age = line.age
if (ageData.lastingLines.contains(line.oldId)) {
@@ -249,12 +268,16 @@ class CodeLongevity(private val serverRepo: Repo,
}
// Store ages for subsequent runs.
- if (Files.notExists(Paths.get(storageDir))) {
- Files.createDirectories(Paths.get(storageDir))
+ try {
+ val file = dataPath.toFile()
+ val oStream = ObjectOutputStream(FileOutputStream(file))
+ oStream.writeUTF(head.getName())
+ oStream.writeObject(ageData)
+ }
+ catch(e: Exception) {
+ Logger.error(e, "Failed to save longevity data. CAUTION: data " +
+ "will be recomputed on a next run.")
}
- val oStream = ObjectOutputStream(FileOutputStream(storagePath))
- oStream.writeObject(head)
- oStream.writeObject(ageData)
return ageData
}
@@ -262,7 +285,7 @@ class CodeLongevity(private val serverRepo: Repo,
* Clears the stored age data if any.
*/
fun dropSavedData() {
- File(storagePath).delete()
+ dataPath.toFile().delete()
}
/**
@@ -292,33 +315,44 @@ class CodeLongevity(private val serverRepo: Repo,
// Build a map of file names and their code lines.
while (headWalk.next()) {
- val path = headWalk.getPathString()
- val fileId = headWalk.getObjectId(0)
- val fileLoader = repo.open(fileId)
- if (!RawText.isBinary(fileLoader.openStream())) {
- val fileText = RawText(fileLoader.getBytes())
- val lines = ArrayList(fileText.size())
- for (idx in 0 .. fileText.size() - 1) {
- lines.add(RevCommitLine(head, fileId, path, idx, false))
+ try {
+ val path = headWalk.getPathString()
+ val fileId = headWalk.getObjectId(0)
+ val fileLoader = repo.open(fileId)
+ if (!RawText.isBinary(fileLoader.openStream())) {
+ val fileText = RawText(fileLoader.getBytes())
+ val lines = ArrayList(fileText.size())
+ for (idx in 0..fileText.size() - 1) {
+ lines.add(RevCommitLine(head, fileId, path, idx, false))
+ }
+ files.put(path, lines)
}
- files.put(path, lines)
+ } catch (e: Exception) {
+ // TODO(anatoly): better fix of exceptions.
}
}
- getDiffsObservable(tail).blockingSubscribe { (commit, diffs) ->
+ getDiffsObservable(tail).blockingSubscribe( { (commit, diffs) ->
// A step back in commits history. Update the files map according
- // to the diff.
- for (diff in diffs) {
+ // to the diff. Traverse the diffs backwards to handle double
+ // renames properly.
+ // TODO(alex): cover file renames by tests (see APP-132 issue).
+ for (diff in diffs.asReversed()) {
val oldPath = diff.getOldPath()
val oldId = diff.getOldId().toObjectId()
val newPath = diff.getNewPath()
val newId = diff.getNewId().toObjectId()
- Logger.debug("old: '$oldPath', new: '$newPath'")
+ Logger.trace { "old: '$oldPath', new: '$newPath'" }
// Skip binary files.
val fileId = if (newPath != DiffEntry.DEV_NULL) newId else oldId
- if (RawText.isBinary(repo.open(fileId).openStream())) {
+ try {
+ if (RawText.isBinary(repo.open(fileId).openStream())) {
+ continue
+ }
+ } catch (e: Exception) {
continue
+ //TODO(anatoly): better exception handling.
}
// TODO(alex): does it happen in the wilds?
@@ -341,6 +375,7 @@ class CodeLongevity(private val serverRepo: Repo,
}
val lines = files.get(path)!!
+
// Update the lines array according to diff insertions.
// Traverse the edit list backwards to keep indices of
// the edit list and the lines array in sync.
@@ -351,15 +386,22 @@ class CodeLongevity(private val serverRepo: Repo,
if (insCount > 0) {
val insStart = edit.getBeginB()
val insEnd = edit.getEndB()
- Logger.debug("ins ($insStart, $insEnd)")
+ Logger.trace { "ins ($insStart, $insEnd)" }
for (idx in insStart .. insEnd - 1) {
val from = RevCommitLine(commit, newId,
newPath, idx, false)
- val to = lines.get(idx)
- val cl = CodeLine(repo, from, to)
- Logger.debug("Collected: ${cl}")
- subscriber.onNext(cl)
+ try {
+ val to = lines.get(idx)
+ val cl = CodeLine(repo, from, to)
+ Logger.trace { "Collected: ${cl}" }
+ subscriber.onNext(cl)
+ }
+ catch(e: IndexOutOfBoundsException) {
+ Logger.error(e,
+ "No line at ${idx}; commit: ${commit.getName()}; '${commit.getShortMessage()}'")
+ throw e
+ }
}
lines.subList(insStart, insEnd).clear()
}
@@ -373,7 +415,7 @@ class CodeLongevity(private val serverRepo: Repo,
if (delCount > 0) {
val delStart = edit.getBeginA()
val delEnd = edit.getEndA()
- Logger.debug("del ($delStart, $delEnd)")
+ Logger.trace { "del ($delStart, $delEnd)" }
val tmpLines = ArrayList(delCount)
for (idx in delStart .. delEnd - 1) {
@@ -389,7 +431,7 @@ class CodeLongevity(private val serverRepo: Repo,
files.set(oldPath, files.remove(newPath)!!)
}
}
- }
+ }, onError)
// If a tail revision was given then the map has to contain unclaimed
// code lines, i.e. the lines added before the tail revision. Push
@@ -409,7 +451,7 @@ class CodeLongevity(private val serverRepo: Repo,
val from = RevCommitLine(tail, fileId,
filePath, idx, false)
val cl = CodeLine(repo, from, lines[idx])
- Logger.debug("Collected (tail): $cl")
+ Logger.trace { "Collected (tail): $cl" }
subscriber.onNext(cl)
}
}
@@ -426,21 +468,28 @@ class CodeLongevity(private val serverRepo: Repo,
Observable>> =
Observable.create { subscriber ->
- val revWalk = RevWalk(repo)
revWalk.markStart(head)
-
var commit: RevCommit? = revWalk.next() // Move the walker to the head.
while (commit != null && commit != tail) {
val parentCommit: RevCommit? = revWalk.next()
- Logger.debug("commit: ${commit.getName()}; " +
- "'${commit.getShortMessage()}'")
- if (parentCommit != null) {
- Logger.debug("parent commit: ${parentCommit.getName()}; "
- + "'${parentCommit.getShortMessage()}'")
- }
- else {
- Logger.debug("parent commit: null")
+ // Smart casts are not yet supported for a mutable variable captured
+ // in an inline lambda, see
+ // https://youtrack.jetbrains.com/issue/KT-7186.
+ if (Logger.isDebug) {
+ val commitName = commit.getName()
+ val commitMsg = commit.getShortMessage()
+ Logger.debug { "commit: $commitName; '$commitMsg'" }
+ if (parentCommit != null) {
+ val parentCommitName = parentCommit.getName()
+ val parentCommitMsg = parentCommit.getShortMessage()
+ Logger.debug {
+ "parent commit: ${parentCommitName}; '${parentCommitMsg}'"
+ }
+ }
+ else {
+ Logger.debug { "parent commit: null" }
+ }
}
subscriber.onNext(Pair(commit, df.scan(parentCommit, commit)))
diff --git a/src/main/kotlin/app/hashers/CommitCrawler.kt b/src/main/kotlin/app/hashers/CommitCrawler.kt
index db738c67..43a7cedf 100644
--- a/src/main/kotlin/app/hashers/CommitCrawler.kt
+++ b/src/main/kotlin/app/hashers/CommitCrawler.kt
@@ -22,34 +22,52 @@ import org.eclipse.jgit.revwalk.RevWalk
import org.eclipse.jgit.util.io.DisabledOutputStream
object CommitCrawler {
- fun getObservable(git: Git, repo: Repo): Observable = Observable
- .create { subscriber ->
- try {
- val revWalk = RevWalk(git.repository)
- val commitId = git.repository.resolve(RepoHelper.MASTER_BRANCH)
- revWalk.markStart(revWalk.parseCommit(commitId))
- for (revCommit in revWalk) {
- subscriber.onNext(Commit(revCommit))
+ fun getObservable(git: Git, repo: Repo,
+ numCommits: Int = 0): Observable {
+ var curNumCommits = 0
+ return Observable
+ .create { subscriber ->
+ try {
+ val revWalk = RevWalk(git.repository)
+ val commitId = git.repository.resolve(RepoHelper.MASTER_BRANCH)
+ revWalk.markStart(revWalk.parseCommit(commitId))
+ for (revCommit in revWalk) {
+ subscriber.onNext(Commit(revCommit))
+ }
+ // Commits are combined in pairs, an empty commit concatenated
+ // to calculate the diff of the initial commit.
+ subscriber.onNext(Commit())
+ } catch (e: Exception) {
+ Logger.error(e, "Commit producing error")
+ subscriber.onError(e)
+ }
+ subscriber.onComplete()
+ } // TODO(anatoly): Rewrite diff calculation in non-weird way.
+ .pairWithNext() // Pair commits to get diff.
+ .map { (new, old) ->
+ curNumCommits++
+ // Mapping and stats extraction.
+ val message = new.raw?.shortMessage ?: ""
+ val hash = new.raw?.name ?: ""
+ val perc = if (numCommits != 0) {
+ (curNumCommits.toDouble() / numCommits) * 100
+ } else 0.0
+ Logger.printCommit(message, hash, perc)
+ new.diffs = getDiffFiles(git, new, old)
+ Logger.debug { "Diff: ${new.diffs.size} entries" }
+ // Count lines on all non-binary files. This is additional
+ // statistics to CommitStats because not all file extensions
+ // may be supported.
+ new.numLinesAdded = new.diffs.fold(0) { total, file ->
+ total + file.getAllAdded().size
+ }
+ new.numLinesDeleted = new.diffs.fold(0) { total, file ->
+ total + file.getAllDeleted().size
}
- // Commits are combined in pairs, an empty commit concatenated
- // to calculate the diff of the initial commit.
- subscriber.onNext(Commit())
- } catch (e: Exception) {
- Logger.error("Commit producing error", e)
- subscriber.onError(e)
+ new.repo = repo
+ new
}
- subscriber.onComplete()
- } // TODO(anatoly): Rewrite diff calculation in non-weird way.
- .pairWithNext() // Pair commits to get diff.
- .map { (new, old) ->
- // Mapping and stats extraction.
- Logger.debug("Commit: ${new.raw?.name ?: ""}: "
- + new.raw?.shortMessage)
- new.diffs = getDiffFiles(git, new, old)
- Logger.debug("Diff: ${new.diffs.size} entries")
- new.repo = repo
- new
- }
+ }
private fun getDiffFiles(git: Git,
commitNew: Commit,
@@ -70,37 +88,70 @@ object CommitCrawler {
} else {
it.newId.toObjectId()
}
- !RawText.isBinary(git.repository.open(id).openStream())
+ val stream = try {
+ git.repository.open(id).openStream()
+ } catch (e: Exception) { null }
+ stream != null && !RawText.isBinary(stream)
}
.map { diff ->
- val new = getContentByObjectId(git, diff.newId.toObjectId())
- val old = getContentByObjectId(git, diff.oldId.toObjectId())
+ // TODO(anatoly): Can produce exception for large object.
+ // Investigate for size.
+ val new = try {
+ getContentByObjectId(git, diff.newId.toObjectId())
+ } catch (e: Exception) {
+ Logger.error(e)
+ null
+ }
+ val old = try {
+ getContentByObjectId(git, diff.oldId.toObjectId())
+ } catch (e: Exception) {
+ Logger.error(e)
+ null
+ }
- val edits = formatter.toFileHeader(diff).toEditList()
- val path = when (diff.changeType) {
- DiffEntry.ChangeType.DELETE -> diff.oldPath
- else -> diff.newPath
+ val diffFiles = mutableListOf()
+ if (new != null && old != null) {
+ val header = try {
+ formatter.toFileHeader(diff)
+ } catch (e: Exception) {
+ Logger.error(e)
+ null
+ }
+
+ if (header != null) {
+ val edits = header.toEditList()
+ val path = when (diff.changeType) {
+ DiffEntry.ChangeType.DELETE -> diff.oldPath
+ else -> diff.newPath
+ }
+ diffFiles.add(DiffFile(path = path,
+ changeType = diff.changeType,
+ old = DiffContent(old, edits.map { edit ->
+ DiffRange(edit.beginA, edit.endA)
+ }),
+ new = DiffContent(new, edits.map { edit ->
+ DiffRange(edit.beginB, edit.endB)
+ })))
+ }
}
- DiffFile(path = path,
- changeType = diff.changeType,
- old = DiffContent(old, edits.map { edit ->
- DiffRange(edit.beginA, edit.endA) }),
- new = DiffContent(new, edits.map { edit ->
- DiffRange(edit.beginB, edit.endB) }))
+
+ diffFiles
}
+ .flatten()
}
}
private fun getContentByObjectId(git: Git,
objectId: ObjectId): List {
return try {
- val rawText = RawText(git.repository.open(objectId).bytes)
+ val obj = git.repository.open(objectId)
+ val rawText = RawText(obj.bytes)
val content = ArrayList(rawText.size())
for (i in 0..(rawText.size() - 1)) {
content.add(rawText.getString(i))
}
return content
- } catch (e: MissingObjectException) {
+ } catch (e: Exception) {
listOf()
}
}
diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt
index a478c069..bc9ddff6 100644
--- a/src/main/kotlin/app/hashers/CommitHasher.kt
+++ b/src/main/kotlin/app/hashers/CommitHasher.kt
@@ -44,22 +44,22 @@ class CommitHasher(private val serverRepo: Repo = Repo(),
// Hash only commits made by authors with specified emails.
.filter { commit -> emails.contains(commit.author.email) }
.map { commit ->
+ Logger.info { "Extracting stats" }
+
// Mapping and stats extraction.
commit.stats = Extractor().extract(commit.diffs)
- Logger.debug("Stats: ${commit.stats.size} entries")
-
- // Count lines on all non-binary files. This is additional
- // statistics to CommitStats because not all file extensions
- // may be supported.
- commit.numLinesAdded = commit.diffs.fold(0) { total, file ->
- total + file.getAllAdded().size
- }
- commit.numLinesDeleted = commit.diffs.fold(0) { total, file ->
- total + file.getAllDeleted().size
+ if (commit.stats.isNotEmpty()) {
+ Logger.printCommitDetail("${commit.stats.size} " +
+ "technology stats found")
}
+ Logger.debug { commit.stats.toString() }
+
commit
}
- .buffer(20, TimeUnit.SECONDS) // Group ready commits by time.
+ // Group ready commits by time and count. Max payload 10 mb,
+ // one commit with stats takes around 1 kb, so pack by max 1000
+ // with 10x margin of safety.
+ .buffer(20, TimeUnit.SECONDS, 1000)
.subscribe({ commitsBundle -> // OnNext.
postCommitsToServer(commitsBundle) // Send ready commits.
}, onError)
@@ -76,15 +76,15 @@ class CommitHasher(private val serverRepo: Repo = Repo(),
private fun postCommitsToServer(commits: List) {
if (commits.isNotEmpty()) {
- api.postCommits(commits)
- Logger.debug("Sent ${commits.size} added commits to server")
+ api.postCommits(commits).onErrorThrow()
+ Logger.info { "Sent ${commits.size} added commits to server" }
}
}
private fun deleteCommitsOnServer(commits: List) {
if (commits.isNotEmpty()) {
- api.deleteCommits(commits)
- Logger.debug("Sent ${commits.size} deleted commits to server")
+ api.deleteCommits(commits).onErrorThrow()
+ Logger.info { "Sent ${commits.size} deleted commits to server" }
}
}
}
diff --git a/src/main/kotlin/app/hashers/FactHasher.kt b/src/main/kotlin/app/hashers/FactHasher.kt
index 430bb66d..213c9a19 100644
--- a/src/main/kotlin/app/hashers/FactHasher.kt
+++ b/src/main/kotlin/app/hashers/FactHasher.kt
@@ -19,12 +19,17 @@ import java.time.ZoneOffset
*/
class FactHasher(private val serverRepo: Repo = Repo(),
private val api: Api,
+ private val rehashes: List,
private val emails: HashSet) {
private val fsDayWeek = hashMapOf>()
private val fsDayTime = hashMapOf>()
private val fsRepoDateStart = hashMapOf()
private val fsRepoDateEnd = hashMapOf()
- private val fsRepoTeamSize = hashSetOf()
+ private val fsCommitLineNumAvg = hashMapOf()
+ private val fsCommitNum = hashMapOf()
+ private val fsLineLenAvg = hashMapOf()
+ private val fsLineNum = hashMapOf()
+ private val fsLinesPerCommits = hashMapOf>()
init {
for (author in emails) {
@@ -32,6 +37,12 @@ class FactHasher(private val serverRepo: Repo = Repo(),
fsDayTime.put(author, Array(24) { 0 })
fsRepoDateStart.put(author, -1)
fsRepoDateEnd.put(author, -1)
+ fsCommitLineNumAvg.put(author, 0.0)
+ fsCommitNum.put(author, 0)
+ fsLineLenAvg.put(author, 0.0)
+ fsLineNum.put(author, 0)
+ // TODO(anatoly): Do the bin computations on the go.
+ fsLinesPerCommits.put(author, Array(rehashes.size) {0})
}
}
@@ -39,36 +50,7 @@ class FactHasher(private val serverRepo: Repo = Repo(),
onError: (Throwable) -> Unit) {
observable
.filter { commit -> emails.contains(commit.author.email) }
- .subscribe({ commit -> // OnNext.
- // Calculate facts.
- val email = commit.author.email
- val timestamp = commit.dateTimestamp
- val dateTime = LocalDateTime.ofEpochSecond(timestamp, 0,
- ZoneOffset.ofTotalSeconds(commit.dateTimeZoneOffset * 60))
-
- // DayWeek.
- val factDayWeek = fsDayWeek[email] ?: Array(7) { 0 }
- // The value is numbered from 1 (Monday) to 7 (Sunday).
- factDayWeek[dateTime.dayOfWeek.value - 1] += 1
- fsDayWeek[email] = factDayWeek
-
- // DayTime.
- val factDayTime = fsDayTime[email] ?: Array(24) { 0 }
- // Hour from 0 to 23.
- factDayTime[dateTime.hour] += 1
- fsDayTime[email] = factDayTime
-
- // RepoDateStart.
- fsRepoDateStart[email] = timestamp
-
- // RepoDateEnd.
- if ((fsRepoDateEnd[email] ?: -1) == -1L) {
- fsRepoDateEnd[email] = timestamp
- }
-
- // RepoTeamSize.
- fsRepoTeamSize.add(email)
- }, onError, { // OnComplete.
+ .subscribe(onNext, onError, { // OnComplete.
try {
postFactsToServer(createFacts())
} catch (e: Throwable) {
@@ -77,32 +59,141 @@ class FactHasher(private val serverRepo: Repo = Repo(),
})
}
+ private val onNext: (Commit) -> Unit = { commit ->
+ // Calculate facts.
+ val email = commit.author.email
+ val timestamp = commit.dateTimestamp
+ val dateTime = LocalDateTime.ofEpochSecond(timestamp, 0,
+ ZoneOffset.ofTotalSeconds(commit.dateTimeZoneOffset * 60))
+
+ // DayWeek.
+ val factDayWeek = fsDayWeek[email] ?: Array(7) { 0 }
+ // The value is numbered from 1 (Monday) to 7 (Sunday).
+ factDayWeek[dateTime.dayOfWeek.value - 1] += 1
+ fsDayWeek[email] = factDayWeek
+
+ // DayTime.
+ val factDayTime = fsDayTime[email] ?: Array(24) { 0 }
+ // Hour from 0 to 23.
+ factDayTime[dateTime.hour] += 1
+ fsDayTime[email] = factDayTime
+
+ // RepoDateStart.
+ fsRepoDateStart[email] = timestamp
+
+ // RepoDateEnd.
+ if (fsRepoDateEnd[email]!! == -1L) {
+ fsRepoDateEnd[email] = timestamp
+ }
+
+ // Commits.
+ val numCommits = fsCommitNum[email]!! + 1
+ val numLinesCurrent = commit.numLinesAdded + commit.numLinesDeleted
+
+ fsCommitNum[email] = numCommits
+ fsCommitLineNumAvg[email] = calcIncAvg(fsCommitLineNumAvg[email]!!,
+ numLinesCurrent.toDouble(), numCommits.toLong())
+
+ val lines = commit.getAllAdded() + commit.getAllDeleted()
+ lines.forEachIndexed { index, line ->
+ fsLineLenAvg[email] = calcIncAvg(fsLineLenAvg[email]!!,
+ line.length.toDouble(), fsLineNum[email]!! + index + 1)
+ }
+ fsLineNum[email] = fsLineNum[email]!! + lines.size
+
+ fsLinesPerCommits[email]!![numCommits - 1] += lines.size
+ }
+
private fun createFacts(): List {
val fs = mutableListOf()
emails.forEach { email ->
val author = Author(email = email)
fsDayTime[email]?.forEachIndexed { hour, count -> if (count > 0) {
- fs.add(Fact(serverRepo, FactCodes.COMMITS_DAY_TIME, hour,
+ fs.add(Fact(serverRepo, FactCodes.COMMIT_DAY_TIME, hour,
count.toString(), author))
}}
fsDayWeek[email]?.forEachIndexed { day, count -> if (count > 0) {
- fs.add(Fact(serverRepo, FactCodes.COMMITS_DAY_WEEK, day,
+ fs.add(Fact(serverRepo, FactCodes.COMMIT_DAY_WEEK, day,
count.toString(), author))
}}
fs.add(Fact(serverRepo, FactCodes.REPO_DATE_START, 0,
fsRepoDateStart[email].toString(), author))
fs.add(Fact(serverRepo, FactCodes.REPO_DATE_END, 0,
fsRepoDateEnd[email].toString(), author))
+ fs.add(Fact(serverRepo, FactCodes.COMMIT_NUM, 0,
+ fsCommitNum[email].toString(), author))
+ fs.add(Fact(serverRepo, FactCodes.COMMIT_LINE_NUM_AVG, 0,
+ fsCommitLineNumAvg[email].toString(), author))
+ fs.add(Fact(serverRepo, FactCodes.LINE_NUM, 0,
+ fsLineNum[email].toString(), author))
+ fs.add(Fact(serverRepo, FactCodes.LINE_LEN_AVG, 0,
+ fsLineLenAvg[email].toString(), author))
+ val linesPerCommits = fsLinesPerCommits[email]!!
+ .sliceArray(IntRange(0, fsCommitNum[email]!! - 1))
+ addCommitsPerLinesFacts(fs, linesPerCommits, author)
}
fs.add(Fact(serverRepo, FactCodes.REPO_TEAM_SIZE, 0,
- fsRepoTeamSize.size.toString()))
+ emails.size.toString()))
return fs
}
private fun postFactsToServer(facts: List) {
if (facts.isNotEmpty()) {
- api.postFacts(facts)
- Logger.debug("Sent ${facts.size} facts to server")
+ api.postFacts(facts).onErrorThrow()
+ Logger.info { "Sent ${facts.size} facts to server" }
+ }
+ }
+
+ /**
+ * Computes the average of a numerical sequence.
+ * Calculated numbers is never bigger than maximum element of sequence.
+ * No overflow due to summing of elements.
+ * @param prev previous value of average
+ * @param element new element of sequence
+ * @param count number of element in sequence
+ * @return new value of average with considering of new element
+ */
+ private fun calcIncAvg(prev: Double, element: Double,
+ count: Long): Double {
+ return prev * (1 - 1.0 / count) + element / count
+ }
+
+ private fun addCommitsPerLinesFacts(fs: MutableList,
+ linesPerCommits: Array,
+ author: Author) {
+ if (linesPerCommits.isEmpty()) return
+
+ var max = linesPerCommits[0]
+ var min = linesPerCommits[0]
+ for (lines in linesPerCommits) {
+ if (lines > max) {
+ max = lines
+ }
+ if (lines < min) {
+ min = lines
+ }
+ }
+
+ val numBins = Math.min(10, max - min + 1)
+ val binSize = (max - min + 1) / numBins.toDouble()
+ val bins = Array(numBins) { 0 }
+ for (numLines in linesPerCommits) {
+ if (numLines == 0) {
+ continue
+ }
+
+ val binId = Math.floor((numLines - min) / binSize).toInt()
+ bins[binId]++
+ }
+
+ for ((binId, numCommits) in bins.withIndex()) {
+ if (numCommits == 0) {
+ continue
+ }
+
+ val numLines = Math.floor(min + binId * binSize).toInt()
+ fs.add(Fact(serverRepo, FactCodes.COMMIT_NUM_TO_LINE_NUM,
+ numLines, numCommits.toString(), author))
}
}
}
diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt
index 93da5055..768e67c2 100644
--- a/src/main/kotlin/app/hashers/RepoHasher.kt
+++ b/src/main/kotlin/app/hashers/RepoHasher.kt
@@ -3,10 +3,11 @@
package app.hashers
-import app.Analytics
+import app.BuildConfig
import app.Logger
import app.api.Api
import app.config.Configurator
+import app.model.Author
import app.model.LocalRepo
import app.model.Repo
import app.utils.HashingException
@@ -31,63 +32,61 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
}
fun update() {
- println("Hashing $localRepo...")
+ Logger.info { "Hashing of repo started" }
val git = loadGit(localRepo.path)
try {
val (rehashes, emails) = fetchRehashesAndEmails(git)
localRepo.parseGitConfig(git.repository.config)
- if (localRepo.author.email.isBlank()) {
- throw IllegalStateException("Can't load email from Git config")
- }
-
- val filteredEmails = filterEmails(emails)
-
initServerRepo(rehashes.last)
+ Logger.debug { "Local repo path: ${localRepo.path}" }
+ Logger.debug { "Repo remote: ${localRepo.remoteOrigin}" }
+ Logger.debug { "Repo rehash: ${serverRepo.rehash}" }
- if (!isKnownRepo()) {
- // Notify server about new contributor and his email.
- postRepoToServer()
- }
// Get repo setup (commits, emails to hash) from server.
- getRepoFromServer()
+ postRepoFromServer()
+
+ // Send all repo emails for invites.
+ postAuthorsToServer(emails)
+
+ val filteredEmails = filterEmails(emails)
// Common error handling for subscribers.
// Exceptions can't be thrown out of reactive chain.
val errors = mutableListOf()
val onError: (Throwable) -> Unit = {
e -> errors.add(e)
- Logger.error("Hashing error", e)
+ Logger.error(e, "Hashing error")
}
// Hash by all plugins.
- val observable = CommitCrawler.getObservable(git, serverRepo)
- .publish()
+ val observable = CommitCrawler.getObservable(git, serverRepo,
+ rehashes.size).publish()
CommitHasher(serverRepo, api, rehashes, filteredEmails)
.updateFromObservable(observable, onError)
- FactHasher(serverRepo, api, filteredEmails)
+ FactHasher(serverRepo, api, rehashes, filteredEmails)
.updateFromObservable(observable, onError)
// Start and synchronously wait until all subscribers complete.
observable.connect()
// TODO(anatoly): CodeLongevity hash from observable.
+ Logger.print("Code longevity calculation. May take a while...")
try {
- CodeLongevity(serverRepo, filteredEmails, git).updateStats(api)
+ CodeLongevity(serverRepo, filteredEmails, git, onError)
+ .updateStats(api)
}
catch (e: Throwable) {
onError(e)
}
-
- // Confirm hashing completion.
- postRepoToServer()
+ Logger.print("Finished.")
if (errors.isNotEmpty()) {
throw HashingException(errors)
}
- println("Hashing $localRepo successfully finished.")
- Analytics.trackHashingRepoSuccess()
+ Logger.info(Logger.Events.HASHING_REPO_SUCCESS)
+ { "Hashing repo completed" }
}
finally {
closeGit(git)
@@ -107,26 +106,25 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
git.close()
}
- private fun isKnownRepo(): Boolean {
- return configurator.getRepos()
- .find { it.rehash == serverRepo.rehash } != null
- }
-
- private fun getRepoFromServer() {
- serverRepo = api.getRepo(serverRepo.rehash)
- Logger.debug("Received repo from server with " +
- serverRepo.commits.size + " commits")
+ private fun postRepoFromServer() {
+ val repo = api.postRepo(serverRepo).getOrThrow()
+ serverRepo.commits = repo.commits
+ Logger.info {
+ "Received repo from server with ${serverRepo.commits.size} commits"
+ }
+ Logger.debug { serverRepo.toString() }
}
- private fun postRepoToServer() {
- api.postRepo(serverRepo)
+ private fun postAuthorsToServer(emails: HashSet) {
+ api.postAuthors(emails.map { email ->
+ Author(email = email, repo = serverRepo)
+ }).onErrorThrow()
}
private fun initServerRepo(initCommitRehash: String) {
- serverRepo = Repo(userEmail = localRepo.author.email)
- serverRepo.initialCommitRehash = initCommitRehash
- serverRepo.rehash = RepoHelper.calculateRepoRehash(
- serverRepo.initialCommitRehash, localRepo)
+ serverRepo = Repo(initialCommitRehash = initCommitRehash,
+ rehash = RepoHelper.calculateRepoRehash(
+ initCommitRehash, localRepo))
}
private fun fetchRehashesAndEmails(git: Git):
@@ -157,8 +155,8 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
return emails
}
- val knownEmails = mutableListOf()
- knownEmails.add(serverRepo.userEmail)
+ val knownEmails = hashSetOf()
+ knownEmails.addAll(configurator.getUser().emails.map { it.email })
knownEmails.addAll(serverRepo.emails)
return knownEmails.filter { emails.contains(it) }.toHashSet()
diff --git a/src/main/kotlin/app/model/Author.kt b/src/main/kotlin/app/model/Author.kt
index 6f690f53..f302cc11 100644
--- a/src/main/kotlin/app/model/Author.kt
+++ b/src/main/kotlin/app/model/Author.kt
@@ -3,10 +3,40 @@
package app.model
+import app.Protos
+import com.google.protobuf.InvalidProtocolBufferException
+import java.security.InvalidParameterException
+
/**
* Commit author.
*/
-data class Author(var name: String = "", var email: String = "") {
+data class Author(
+ var name: String = "",
+ var email: String = "",
+ var repo: Repo = Repo()
+) {
+ @Throws(InvalidParameterException::class)
+ constructor(proto: Protos.Author) : this() {
+ email = proto.email
+ repo = Repo(proto.repoRehash)
+ }
+
+ @Throws(InvalidProtocolBufferException::class)
+ constructor(bytes: ByteArray) : this(Protos.Author.parseFrom(bytes))
+
+ constructor(serialized: String) : this(serialized.toByteArray())
+
+ fun getProto(): Protos.Author {
+ return Protos.Author.newBuilder()
+ .setEmail(email)
+ .setRepoRehash(repo.rehash)
+ .build()
+ }
+
+ fun serialize(): ByteArray {
+ return getProto().toByteArray()
+ }
+
// Email defines user identity.
override fun equals(other: Any?): Boolean {
if (other is Author) {
diff --git a/src/main/kotlin/app/model/AuthorGroup.kt b/src/main/kotlin/app/model/AuthorGroup.kt
new file mode 100644
index 00000000..bf252d78
--- /dev/null
+++ b/src/main/kotlin/app/model/AuthorGroup.kt
@@ -0,0 +1,32 @@
+package app.model
+
+import app.Protos
+import com.google.protobuf.InvalidProtocolBufferException
+import java.security.InvalidParameterException
+
+/**
+ * Group of commit authors.
+ */
+data class AuthorGroup(
+ var authors: List = listOf()
+) {
+ @Throws(InvalidParameterException::class)
+ constructor(proto: Protos.AuthorGroup) : this() {
+ authors = proto.authorsList.map { it -> Author(it) }
+ }
+
+ @Throws(InvalidProtocolBufferException::class)
+ constructor(bytes: ByteArray) : this(Protos.AuthorGroup.parseFrom(bytes))
+
+ constructor(serialized: String) : this(serialized.toByteArray())
+
+ fun getProto(): Protos.AuthorGroup {
+ return Protos.AuthorGroup.newBuilder()
+ .addAllAuthors(authors.map { it -> it.getProto() })
+ .build()
+ }
+
+ fun serialize(): ByteArray {
+ return getProto().toByteArray()
+ }
+}
diff --git a/src/main/kotlin/app/model/Commit.kt b/src/main/kotlin/app/model/Commit.kt
index 9810a77a..32835788 100644
--- a/src/main/kotlin/app/model/Commit.kt
+++ b/src/main/kotlin/app/model/Commit.kt
@@ -86,4 +86,12 @@ data class Commit(
override fun hashCode(): Int {
return rehash.hashCode()
}
+
+ fun getAllAdded(): List {
+ return diffs.map { it.getAllAdded() }.flatten()
+ }
+
+ fun getAllDeleted(): List {
+ return diffs.map { it.getAllDeleted() }.flatten()
+ }
}
diff --git a/src/main/kotlin/app/model/DiffContent.kt b/src/main/kotlin/app/model/DiffContent.kt
index 82cb7d1e..212eb978 100644
--- a/src/main/kotlin/app/model/DiffContent.kt
+++ b/src/main/kotlin/app/model/DiffContent.kt
@@ -1,3 +1,6 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
package app.model
class DiffContent(
diff --git a/src/main/kotlin/app/model/Error.kt b/src/main/kotlin/app/model/Error.kt
new file mode 100644
index 00000000..0baf8997
--- /dev/null
+++ b/src/main/kotlin/app/model/Error.kt
@@ -0,0 +1,35 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
+package app.model
+
+import app.Protos
+import com.google.protobuf.InvalidProtocolBufferException
+import java.security.InvalidParameterException
+
+data class Error(
+ var code: Int = 0,
+ var message: String = ""
+) {
+ @Throws(InvalidParameterException::class)
+ constructor(proto: Protos.Error) : this() {
+ code = proto.code
+ message = proto.message
+ }
+
+ @Throws(InvalidProtocolBufferException::class)
+ constructor(bytes: ByteArray) : this(Protos.Error.parseFrom(bytes))
+
+ constructor(serialized: String) : this(serialized.toByteArray())
+
+ fun getProto(): Protos.Error {
+ return Protos.Error.newBuilder()
+ .setCode(code)
+ .setMessage(message)
+ .build()
+ }
+
+ fun serialize(): ByteArray {
+ return getProto().toByteArray()
+ }
+}
diff --git a/src/main/kotlin/app/model/Errors.kt b/src/main/kotlin/app/model/Errors.kt
new file mode 100644
index 00000000..59b9625a
--- /dev/null
+++ b/src/main/kotlin/app/model/Errors.kt
@@ -0,0 +1,32 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
+package app.model
+
+import app.Protos
+import com.google.protobuf.InvalidProtocolBufferException
+import java.security.InvalidParameterException
+
+data class Errors (
+ var errors: List = listOf()
+) {
+ @Throws(InvalidParameterException::class)
+ constructor(proto: Protos.Errors) : this() {
+ errors = proto.errorsList.map { error -> Error(error) }
+ }
+
+ @Throws(InvalidProtocolBufferException::class)
+ constructor(bytes: ByteArray) : this(Protos.Errors.parseFrom(bytes))
+
+ constructor(serialized: String) : this(serialized.toByteArray())
+
+ fun getProto(): Protos.Errors {
+ return Protos.Errors.newBuilder()
+ .addAllErrors(errors.map { error -> error.getProto() })
+ .build()
+ }
+
+ fun serialize(): ByteArray {
+ return getProto().toByteArray()
+ }
+}
diff --git a/src/main/kotlin/app/model/Fact.kt b/src/main/kotlin/app/model/Fact.kt
index 91daf5a8..e137f1d9 100644
--- a/src/main/kotlin/app/model/Fact.kt
+++ b/src/main/kotlin/app/model/Fact.kt
@@ -1,3 +1,6 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
package app.model
import app.Protos
diff --git a/src/main/kotlin/app/model/FactGroup.kt b/src/main/kotlin/app/model/FactGroup.kt
index 9c00871f..681196d1 100644
--- a/src/main/kotlin/app/model/FactGroup.kt
+++ b/src/main/kotlin/app/model/FactGroup.kt
@@ -1,3 +1,6 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
package app.model
import app.Protos
diff --git a/src/main/kotlin/app/model/LocalRepo.kt b/src/main/kotlin/app/model/LocalRepo.kt
index 66291679..8339d1b1 100644
--- a/src/main/kotlin/app/model/LocalRepo.kt
+++ b/src/main/kotlin/app/model/LocalRepo.kt
@@ -18,4 +18,8 @@ data class LocalRepo(var path: String = "") {
email = config.getString("user", null, "email") ?: "")
remoteOrigin = config.getString("remote", "origin", "url") ?: ""
}
+
+ override fun toString(): String {
+ return path
+ }
}
diff --git a/src/main/kotlin/app/model/Repo.kt b/src/main/kotlin/app/model/Repo.kt
index 62e35562..b609d23b 100644
--- a/src/main/kotlin/app/model/Repo.kt
+++ b/src/main/kotlin/app/model/Repo.kt
@@ -15,8 +15,6 @@ data class Repo(
var rehash: String = "",
var initialCommitRehash: String = "",
- var userEmail: String = "",
-
// Authors' email filter for hashed commits. If empty list then hash
// only commits that created by current user.
var emails: List = listOf(),
@@ -28,7 +26,6 @@ data class Repo(
constructor(proto: Protos.Repo) : this() {
rehash = proto.rehash
initialCommitRehash = proto.initialCommitRehash
- userEmail = proto.userEmail
emails = proto.emailsList
commits = proto.commitsList.map { Commit(it) }
}
@@ -42,7 +39,6 @@ data class Repo(
return Protos.Repo.newBuilder()
.setRehash(rehash)
.setInitialCommitRehash(rehash)
- .setUserEmail(userEmail)
.addAllEmails(emails)
.addAllCommits(commits.map { it.getProto() })
.build()
diff --git a/src/main/kotlin/app/model/User.kt b/src/main/kotlin/app/model/User.kt
index c9181490..78f731bc 100644
--- a/src/main/kotlin/app/model/User.kt
+++ b/src/main/kotlin/app/model/User.kt
@@ -11,12 +11,14 @@ import java.security.InvalidParameterException
* User information.
*/
data class User (
- var repos: MutableList = mutableListOf()
+ var repos: MutableList = mutableListOf(),
+ var emails: HashSet = hashSetOf()
) {
@Throws(InvalidParameterException::class)
constructor(proto: Protos.User) : this() {
repos = proto.reposList.map { repo -> Repo(repo) }
- .toMutableList()
+ .toMutableList()
+ emails = proto.emailsList.map { email -> UserEmail(email) }.toHashSet()
}
@Throws(InvalidProtocolBufferException::class)
@@ -26,8 +28,9 @@ data class User (
fun getProto(): Protos.User {
return Protos.User.newBuilder()
- .addAllRepos(repos.map { repo -> repo.getProto() })
- .build()
+ .addAllRepos(repos.map { repo -> repo.getProto() })
+ .addAllEmails(emails.map { email -> email.getProto() })
+ .build()
}
fun serialize(): ByteArray {
diff --git a/src/main/kotlin/app/model/UserEmail.kt b/src/main/kotlin/app/model/UserEmail.kt
new file mode 100644
index 00000000..d03219e0
--- /dev/null
+++ b/src/main/kotlin/app/model/UserEmail.kt
@@ -0,0 +1,58 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
+package app.model
+
+import app.Protos
+import com.google.protobuf.InvalidProtocolBufferException
+import java.security.InvalidParameterException
+
+/**
+ * User information.
+ */
+class UserEmail(
+ var email: String = "",
+ var primary: Boolean = false,
+ var verified: Boolean = false
+) {
+ @Throws(InvalidParameterException::class)
+ constructor(proto: Protos.UserEmail) : this() {
+ email = proto.email
+ primary = proto.primary
+ verified = proto.verified
+ }
+
+ @Throws(InvalidProtocolBufferException::class)
+ constructor(bytes: ByteArray) : this(Protos.UserEmail.parseFrom(bytes))
+
+ constructor(serialized: String) : this(serialized.toByteArray())
+
+ fun getProto(): Protos.UserEmail {
+ return Protos.UserEmail.newBuilder()
+ .setEmail(email)
+ .setPrimary(primary)
+ .setVerified(verified)
+ .build()
+ }
+
+ fun serialize(): ByteArray {
+ return getProto().toByteArray()
+ }
+
+ override fun toString(): String {
+ val primary = if (this.primary) " (Primary)" else ""
+ val verified = if (this.verified) "Confirmed" else "Not confirmed"
+ return "${this.email}$primary — $verified"
+ }
+
+ override fun equals(other: Any?): Boolean {
+ if (other is UserEmail) {
+ return email == other.email
+ }
+ return false
+ }
+
+ override fun hashCode(): Int {
+ return email.hashCode()
+ }
+}
diff --git a/src/main/kotlin/app/ui/AddRepoState.kt b/src/main/kotlin/app/ui/AddRepoState.kt
index e36156db..1ebc3cee 100644
--- a/src/main/kotlin/app/ui/AddRepoState.kt
+++ b/src/main/kotlin/app/ui/AddRepoState.kt
@@ -3,7 +3,7 @@
package app.ui
-import app.Analytics
+import app.Logger
import app.api.Api
import app.config.Configurator
import app.model.LocalRepo
@@ -21,19 +21,18 @@ class AddRepoState constructor(private val context: Context,
if (configurator.getLocalRepos().isNotEmpty()) return
while (true) {
- println("Type a path to repository, or hit Enter to start "
- + "hashing.")
+ Logger.print("Type a path to repository, or hit Enter to continue.")
val pathString = readLine() ?: ""
if (pathString.isEmpty()) {
if (configurator.getLocalRepos().isEmpty()) {
- println("Add at least one valid repository.")
+ Logger.print("Add at least one valid repository.")
} else {
break // User finished to add repos.
}
} else {
if (RepoHelper.isValidRepo(pathString)) {
- println("Added git repository at $pathString.")
+ Logger.print("Added git repository at $pathString.")
val localRepo = LocalRepo(pathString)
localRepo.hashAllContributors = UiHelper.confirm("Do you "
+ "want to hash commits of all contributors?",
@@ -41,15 +40,18 @@ class AddRepoState constructor(private val context: Context,
configurator.addLocalRepoPersistent(localRepo)
configurator.saveToFile()
} else {
- println("No valid git repository found at $pathString.")
+ Logger.print("Directory should contain a valid git " +
+ "repository.")
+ Logger.print("Make sure that master branch with at least " +
+ "one commit exists.")
}
}
}
- Analytics.trackConfigSetup()
+ Logger.info(Logger.Events.CONFIG_SETUP) { "Config setup" }
}
override fun next() {
- context.changeState(UpdateRepoState(context, api, configurator))
+ context.changeState(EmailState(context, api, configurator))
}
}
diff --git a/src/main/kotlin/app/ui/AuthState.kt b/src/main/kotlin/app/ui/AuthState.kt
index 627b63e1..44208bb4 100644
--- a/src/main/kotlin/app/ui/AuthState.kt
+++ b/src/main/kotlin/app/ui/AuthState.kt
@@ -3,12 +3,14 @@
package app.ui
-import app.Analytics
import app.BuildConfig
+import app.Logger
import app.api.Api
import app.config.Configurator
import app.utils.PasswordHelper
-import app.utils.RequestException
+import app.api.ApiError
+import app.api.ifNotNullThrow
+import app.api.isWithServerCode
/**
* Authorization console UI state.
@@ -19,7 +21,8 @@ class AuthState constructor(private val context: Context,
: ConsoleState {
var username = ""
var password = ""
- var connectionError = false
+ var retry = true
+ var authorized = false
override fun doAction() {
if (!configurator.isValidCredentials()) {
@@ -27,13 +30,15 @@ class AuthState constructor(private val context: Context,
getPassword()
}
- while (!tryAuth() && !connectionError) {
+ authorized = tryAuth()
+ while (!authorized && retry) {
getPassword()
+ authorized = tryAuth()
}
}
override fun next() {
- if (!connectionError) {
+ if (authorized) {
context.changeState(ListRepoState(context, api, configurator))
} else {
context.changeState(CloseState())
@@ -41,13 +46,13 @@ class AuthState constructor(private val context: Context,
}
fun getUsername() {
- println("Enter username:")
+ Logger.print("Enter username:")
username = readLine() ?: ""
configurator.setUsernameCurrent(username)
}
fun getPassword() {
- println("Enter password:")
+ Logger.print("Enter password:")
password = PasswordHelper.readPassword()
configurator.setPasswordCurrent(password)
}
@@ -66,32 +71,42 @@ class AuthState constructor(private val context: Context,
fun tryAuth(): Boolean {
try {
- println("Authenticating...")
- api.authorize()
+ Logger.print("Signing in...")
+ val (_, error) = api.authorize()
+ if (error.isWithServerCode(Api.OUT_OF_DATE)) {
+ Logger.print("App is out of date. Please get new version at " +
+ "https://sourcerer.io")
+ retry = false
+ return false
+ }
+ // Other request errors should be processed by try/catch.
+ error.ifNotNullThrow()
- val user = api.getUser()
- configurator.setRepos(user.repos)
+ val user = api.getUser().getOrThrow()
+ configurator.setUser(user)
- println("You are successfully authenticated. Your profile page is "
- + BuildConfig.PROFILE_URL + configurator.getUsername())
- saveCredentialsIfChanged()
+ Logger.print("Signed in successfully. Your profile page is " +
+ BuildConfig.PROFILE_URL + configurator.getUsername())
- Analytics.username = configurator.getUsername()
- Analytics.trackAuth()
+ saveCredentialsIfChanged()
+ Logger.username = configurator.getUsername()
+ Logger.info(Logger.Events.AUTH) { "Auth success" }
return true
- } catch (e: RequestException) {
+ } catch (e: ApiError) {
if (e.isAuthError) {
if(e.httpBodyMessage.isNotBlank()) {
- println(e.httpBodyMessage)
+ Logger.print(e.httpBodyMessage)
} else {
- println("Authentication error. Try again.")
+ Logger.print("Authentication error. Try again.")
}
} else {
- connectionError = true
- println("Connection problems. Try again later.")
+ Logger.print("Connection problems. Try again later.")
+ Logger.error(e)
+ retry = false
}
}
+
return false
}
}
diff --git a/src/main/kotlin/app/ui/CloseState.kt b/src/main/kotlin/app/ui/CloseState.kt
index 6d91f4f7..2b6403bb 100644
--- a/src/main/kotlin/app/ui/CloseState.kt
+++ b/src/main/kotlin/app/ui/CloseState.kt
@@ -3,17 +3,19 @@
package app.ui
+import app.Logger
+
/**
* On application close console UI state.
*/
class CloseState : ConsoleState {
override fun doAction() {
- println("You could use console commands to control repositories. To "
- + "setup again run application with flag --setup. For more "
- + "info run application with flag --help.")
+ Logger.print("You could use console commands to control repositories.",
+ indentLine = true)
+ Logger.print("For more info run application with flag --help.")
// TODO(anatoly): Check for problems for display support message.
- println("Feel free to contact us on any problem by "
- + "support@sourcerer.io.")
+ Logger.print("Feel free to contact us on any problem by " +
+ "support@sourcerer.io.")
}
override fun next() {
diff --git a/src/main/kotlin/app/ui/EmailState.kt b/src/main/kotlin/app/ui/EmailState.kt
new file mode 100644
index 00000000..2f7a0530
--- /dev/null
+++ b/src/main/kotlin/app/ui/EmailState.kt
@@ -0,0 +1,96 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Anatoly Kislov (anatoly@sourcerer.io)
+
+package app.ui
+
+import app.Logger
+import app.api.Api
+import app.config.Configurator
+import app.model.User
+import app.model.UserEmail
+import app.utils.UiHelper
+import org.eclipse.jgit.api.Git
+import java.io.File
+
+/**
+ * Update repositories console UI state.
+ */
+class EmailState constructor(private val context: Context,
+ private val api: Api,
+ private val configurator: Configurator)
+ : ConsoleState {
+ override fun doAction() {
+ val user = configurator.getUser()
+
+ if (user.emails.isNotEmpty()) {
+ Logger.print("List of your emails:", indentLine = true)
+ user.emails.forEach { email -> println(email) }
+ } else {
+ // Shouldn't really happen. User always have primary email.
+ Logger.print("Add at least one email to build your profile.",
+ indentLine = true)
+ }
+
+ val knownEmails = user.emails.map { it.email }
+ val newEmails = hashSetOf()
+ val configEmails = hashSetOf()
+ // TODO(anatoly): Tell about web editing emails, when it's ready.
+ // TODO(anatoly): Add global config parsing.
+ // TODO(anatoly): Add user config parsing.
+
+ // Add emails from git configs.
+ for (repo in configurator.getLocalRepos()) {
+ try {
+ val git = Git.open(File(repo.path))
+ val email = git.repository
+ .config.getString("user", null, "email") ?: ""
+ if (!knownEmails.contains(email)) {
+ configEmails.add(email)
+ }
+ } catch (e: Exception) {
+ Logger.error(e, "Error while parsing git config")
+ }
+ }
+
+ if (configEmails.isNotEmpty()) {
+ Logger.print("Your git config contains untracked emails:")
+ configEmails.forEach { email -> println(email) }
+ if (UiHelper.confirm("Do you want to add this emails to your " +
+ "account?", defaultIsYes = true)) {
+ newEmails.addAll(configEmails)
+ }
+ }
+
+ // Ask user to enter his emails.
+ if (UiHelper.confirm("Do you want to specify additional emails " +
+ "that you use in repositories?", defaultIsYes = false)) {
+ while (true) {
+ Logger.print("Type a email, or hit Enter to continue.")
+ val email = readLine() ?: ""
+ if (email.isBlank()) break
+ if (!knownEmails.contains(email)) newEmails.add(email)
+ }
+ }
+
+ if (newEmails.isNotEmpty()) {
+ val newUserEmails = newEmails.map { UserEmail(email = it) }
+ // We will need new emails during hashing.
+ user.emails.addAll(newUserEmails)
+
+ // Send new emails to server.
+ val userNewEmails = User(emails = newUserEmails.toHashSet())
+ api.postUser(userNewEmails)
+ }
+
+ // Warn user about need of confirmation.
+ if (user.emails.filter { email -> !email.verified }.isNotEmpty() ||
+ newEmails.isNotEmpty()) {
+ Logger.print("Confirm your emails to show all statistics in " +
+ "profile.")
+ }
+ }
+
+ override fun next() {
+ context.changeState(UpdateRepoState(context, api, configurator))
+ }
+}
diff --git a/src/main/kotlin/app/ui/OpenState.kt b/src/main/kotlin/app/ui/OpenState.kt
index 7c04d729..44165dff 100644
--- a/src/main/kotlin/app/ui/OpenState.kt
+++ b/src/main/kotlin/app/ui/OpenState.kt
@@ -3,6 +3,7 @@
package app.ui
+import app.Logger
import app.api.Api
import app.config.Configurator
@@ -15,11 +16,13 @@ class OpenState constructor(private val context: Context,
: ConsoleState {
override fun doAction() {
if (!configurator.isValidCredentials()) {
- println("Sourcerer hashes your git repositories into intelligent "
- + "engineering profiles. If you don't have an account, "
- + "please, proceed to http://sourcerer.io/register.")
+ Logger.print("Sourcerer hashes your git repositories into " +
+ "intelligent engineering profiles.")
+ Logger.print("If you don't have an account, please, sign up at " +
+ "https://sourcerer.io/join")
} else {
- println("Sourcerer. Use flag --help to list available commands.")
+ Logger.print("Sourcerer. Use flag --help to list available " +
+ "commands.")
}
}
diff --git a/src/main/kotlin/app/ui/UpdateRepoState.kt b/src/main/kotlin/app/ui/UpdateRepoState.kt
index 925a4c23..4e4b52bf 100644
--- a/src/main/kotlin/app/ui/UpdateRepoState.kt
+++ b/src/main/kotlin/app/ui/UpdateRepoState.kt
@@ -3,13 +3,12 @@
package app.ui
-import app.Analytics
+import app.BuildConfig
import app.hashers.RepoHasher
import app.Logger
import app.api.Api
import app.config.Configurator
import app.utils.HashingException
-import app.utils.RequestException
/**
* Update repositories console UI state.
@@ -19,23 +18,28 @@ class UpdateRepoState constructor(private val context: Context,
private val configurator: Configurator)
: ConsoleState {
override fun doAction() {
- println("Hashing your git repositories.")
+ Logger.info { "Hashing started" }
+
for (repo in configurator.getLocalRepos()) {
try {
+ Logger.print("Hashing $repo repository...", indentLine = true)
RepoHasher(repo, api, configurator).update()
+ Logger.print("Hashing $repo completed.")
} catch (e: HashingException) {
- Logger.error("During hashing ${e.errors.size} errors occurred:")
e.errors.forEach { error ->
- Logger.error("", error)
+ Logger.error(error, "Error while hashing")
}
} catch (e: Exception) {
- Logger.error("Error while hashing $repo", e)
+ Logger.error(e, "Error while hashing")
}
}
- println("The repositories have been hashed. See result online on your "
- + "Sourcerer profile.")
- Analytics.trackHashingSuccess()
+ api.postComplete().onErrorThrow()
+ Logger.print("The repositories have been hashed.")
+ Logger.print("Take a look at the updates in your profile at " +
+ BuildConfig.PROFILE_URL + configurator.getUsername(),
+ indentLine = true)
+ Logger.info(Logger.Events.HASHING_SUCCESS) { "Hashing success" }
}
override fun next() {
diff --git a/src/main/kotlin/app/utils/FileHelper.kt b/src/main/kotlin/app/utils/FileHelper.kt
index 0ec2a3e8..668348f1 100644
--- a/src/main/kotlin/app/utils/FileHelper.kt
+++ b/src/main/kotlin/app/utils/FileHelper.kt
@@ -3,12 +3,48 @@
package app.utils
+import java.io.File
+import java.net.URLDecoder
+import java.nio.file.Files
import java.nio.file.Paths
+import java.nio.file.Path
+/*
+ * Wrapper around Java Path and File classes to work the sourcerer's files.
+ */
object FileHelper {
+ private val dirName = "data"
+ private val jarPath = getJarPath()
+ private val settingsPath = jarPath.resolve(dirName)
+
+ fun getPath(name: String, vararg parts: String): Path {
+ val path = settingsPath.resolve(Paths.get("", *parts))
+ if (Files.notExists(path)) {
+ Files.createDirectories(path)
+ }
+ return path.resolve(name)
+ }
+
+ fun getFile(name: String, vararg parts: String): File {
+ return getPath(name, *parts).toFile()
+ }
+
+ fun notExists(name:String, vararg parts: String): Boolean {
+ return Files.notExists(getPath(name, *parts))
+ }
+
fun getFileExtension(path: String): String {
val fileName = Paths.get(path).fileName.toString()
- return fileName.substringAfterLast(
- delimiter = '.', missingDelimiterValue = "")
+ return fileName.substringAfterLast(delimiter = '.',
+ missingDelimiterValue = "")
+ }
+
+ fun getJarPath(): Path {
+ val fullPathString = URLDecoder.decode(FileHelper::class.java
+ .protectionDomain.codeSource.location.toURI().path, "UTF-8")
+ val fullPath = Paths.get(fullPathString)
+ val root = fullPath.root
+ // Removing jar filename.
+ return root.resolve(fullPath.subpath(0, fullPath.nameCount - 1))
}
}
diff --git a/src/main/kotlin/app/utils/Metrics.kt b/src/main/kotlin/app/utils/Metrics.kt
new file mode 100644
index 00000000..7476ccaa
--- /dev/null
+++ b/src/main/kotlin/app/utils/Metrics.kt
@@ -0,0 +1,82 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Alexander Surkov (alex@sourcerer.io)
+
+package app.utils
+
+import kotlin.system.*
+
+object Metrics {
+ override fun toString(): String {
+ var str: String = ""
+
+ var total: Long = 0
+ for ((_, item) in timing.items) {
+ total += item.time
+ }
+
+ str += timing.toStringHelper(total)
+
+ for ((tag, value) in numerical) {
+ str += "$tag: $value\n"
+ }
+
+ return str
+ }
+
+ fun recordTime(tag: String, block: () -> T) : T {
+ val parentItem = timingStack.get(timingStack.lastIndex)
+ var item = parentItem.items.getOrPut(tag, { TimingData() })
+
+ timingStack.add(item)
+
+ val start = System.currentTimeMillis()
+ val result = block()
+ item.time += System.currentTimeMillis() - start
+
+ timingStack.subList(timingStack.lastIndex, timingStack.lastIndex + 1).clear()
+
+ return result
+ }
+
+ fun recordMetric(tag: String, metric: Long)
+ {
+ var v = numerical.getOrDefault(tag, 0)
+ numerical.put(tag, v + metric);
+ }
+
+ class TimingData {
+ var time: Long = 0
+ var items = mutableMapOf()
+
+ fun toStringHelper(total: Long, offset: String = ""): String {
+ var str = ""
+ for ((tag, item) in items) {
+ str += "$offset$tag: ${item.time} ms, ${item.time * 100 / total}%\n"
+ str += item.toStringHelper(total, "$offset ")
+ }
+ return str
+ }
+ }
+
+ private val timing = TimingData()
+ private val timingStack = mutableListOf(timing)
+ private var nestedTiming: Boolean = true
+
+ private val numerical = mutableMapOf()
+}
+
+/**
+ * Records a block's execution time in milliseconds and assigns it a tag in
+ * calls hierarchy.
+ */
+fun recordTime(tag: String, block: () -> T) : T {
+ return Metrics.recordTime(tag, block)
+}
+
+/**
+ * Records a numeric metric of given tag.
+ */
+fun recordMetric(tag: String, metric: Long)
+{
+ return Metrics.recordMetric(tag, metric)
+}
diff --git a/src/main/kotlin/app/utils/RepoHelper.kt b/src/main/kotlin/app/utils/RepoHelper.kt
index 2cb5f3de..411eba84 100644
--- a/src/main/kotlin/app/utils/RepoHelper.kt
+++ b/src/main/kotlin/app/utils/RepoHelper.kt
@@ -32,7 +32,7 @@ object RepoHelper {
repository = git.repository
commitId = repository.resolve(MASTER_BRANCH)
} catch (e: Exception) {
- Logger.error("Cannot access repository at path $path", e)
+ Logger.error(e, "Cannot access repository at specified path")
return false
} finally {
repository?.close()
@@ -49,13 +49,13 @@ object RepoHelper {
return try {
Paths.get(path).toFile().isDirectory
} catch (e: InvalidPathException) {
- Logger.error("Invalid path $path", e)
+ Logger.error(e, "Invalid path")
false
} catch (e: UnsupportedOperationException) {
- Logger.error("Invalid path $path", e)
+ Logger.error(e, "Invalid path")
false
} catch (e: SecurityException) {
- Logger.error("Cannot access repository at path $path", e)
+ Logger.error(e, "Cannot access repository at specified path")
false
}
}
@@ -86,23 +86,23 @@ object RepoHelper {
fun printRepos(localRepos: List)
{
for (repo in localRepos) {
- println(repo)
+ Logger.print(repo)
}
}
fun printRepos(localRepos: List, title: String) {
if (localRepos.isNotEmpty()) {
- println(title)
+ Logger.print(title, indentLine = true)
printRepos(localRepos)
}
}
fun printRepos(localRepos: List, title: String, empty: String) {
if (localRepos.isNotEmpty()) {
- println(title)
+ Logger.print(title, indentLine = true)
printRepos(localRepos)
} else {
- println(empty)
+ Logger.print(empty, indentLine = true)
}
}
}
diff --git a/src/main/kotlin/app/utils/RequestException.kt b/src/main/kotlin/app/utils/RequestException.kt
index 7a6f57db..fd56e7ef 100644
--- a/src/main/kotlin/app/utils/RequestException.kt
+++ b/src/main/kotlin/app/utils/RequestException.kt
@@ -21,8 +21,8 @@ class RequestException(exception: Exception) : Exception(exception.message) {
var isParseError = false
constructor(fuelError: FuelError) : this(fuelError as Exception) {
- httpStatusCode = fuelError.response.httpStatusCode
- httpResponseMessage = fuelError.response.httpResponseMessage
+ httpStatusCode = fuelError.response.statusCode
+ httpResponseMessage = fuelError.response.responseMessage
httpBodyMessage = fuelError.response.data
.toString(Charset.defaultCharset())
}
diff --git a/src/main/kotlin/app/utils/UiHelper.kt b/src/main/kotlin/app/utils/UiHelper.kt
index c7d9671d..a7b067d6 100644
--- a/src/main/kotlin/app/utils/UiHelper.kt
+++ b/src/main/kotlin/app/utils/UiHelper.kt
@@ -3,11 +3,13 @@
package app.utils
+import app.Logger
+
object UiHelper {
fun confirm(message: String, defaultIsYes: Boolean): Boolean {
val yes = if (defaultIsYes) "Y" else "y"
val no = if (!defaultIsYes) "N" else "n"
- println("$message [$yes/$no]")
+ Logger.print("$message [$yes/$no]")
val oppositeDefaultValue = if (defaultIsYes) no else yes
if ((readLine() ?: "").toLowerCase() == oppositeDefaultValue) {
return !defaultIsYes
diff --git a/src/main/proto/classifier.proto b/src/main/proto/classifier.proto
new file mode 100644
index 00000000..dcefcaf0
--- /dev/null
+++ b/src/main/proto/classifier.proto
@@ -0,0 +1,21 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
+
+syntax = "proto3";
+
+// Java compiler settings.
+package app;
+option java_package = "app";
+option java_outer_classname = "ModelsProtos";
+
+message Classifier {
+ repeated string tokens = 1;
+
+ repeated string libraries = 2;
+
+ repeated float idf = 3;
+
+ repeated float weights = 4;
+
+ repeated float biases = 5;
+}
diff --git a/src/main/proto/sourcerer.proto b/src/main/proto/sourcerer.proto
index efa1ca90..153ea403 100644
--- a/src/main/proto/sourcerer.proto
+++ b/src/main/proto/sourcerer.proto
@@ -72,10 +72,29 @@ message FactGroup {
repeated Fact facts = 1;
}
+// Used to authors for invitational procedures.
+message Author {
+ string email = 1;
+ string repo_rehash = 2;
+}
+
+message AuthorGroup {
+ repeated Author authors = 1;
+}
+
// User properties and indentity information about repos.
message User {
// List of known repos containing basic information for indentifying repo.
repeated Repo repos = 1;
+ // List of known emails.
+ repeated UserEmail emails = 2;
+}
+
+// Email of user with its status information.
+message UserEmail {
+ string email = 1;
+ bool primary = 2;
+ bool verified = 3;
}
// Repository parameters for hashing.
@@ -87,9 +106,6 @@ message Repo {
// Rehash of first commit in repo. Used for indentifying forks.
string initial_commit_rehash = 2;
- // Email of app user from git config of repo.
- string user_email = 3;
-
// Authors' email filter for hashed commits. If empty list then hash only
// commits that created by current user.
repeated string emails = 4;
@@ -98,3 +114,13 @@ message Repo {
// starts after last commit from the overlap.
repeated Commit commits = 5;
}
+
+// Errors from server.
+message Error {
+ uint32 code = 1;
+ string message = 2;
+}
+
+message Errors {
+ repeated Error errors = 1;
+}
diff --git a/src/main/resources/data/imports/cpp.txt b/src/main/resources/data/imports/cpp.txt
new file mode 100644
index 00000000..f8b23502
--- /dev/null
+++ b/src/main/resources/data/imports/cpp.txt
@@ -0,0 +1,4 @@
+opencv:opencv
+opencv2:opencv
+protobuf:protobuf
+pb:protobuf
\ No newline at end of file
diff --git a/src/main/resources/data/imports/python.txt b/src/main/resources/data/imports/python.txt
new file mode 100644
index 00000000..e8a45a71
--- /dev/null
+++ b/src/main/resources/data/imports/python.txt
@@ -0,0 +1,2 @@
+cv2:opencv
+cv:opencv
\ No newline at end of file
diff --git a/src/test/kotlin/test/tests/extractors/ExtractorTest.kt b/src/test/kotlin/test/tests/extractors/ExtractorTest.kt
new file mode 100644
index 00000000..ccc80a88
--- /dev/null
+++ b/src/test/kotlin/test/tests/extractors/ExtractorTest.kt
@@ -0,0 +1,191 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
+
+package test.tests.extractors
+
+import app.extractors.*
+import junit.framework.TestCase.assertTrue
+import org.jetbrains.spek.api.Spek
+import org.jetbrains.spek.api.dsl.given
+import org.jetbrains.spek.api.dsl.it
+import kotlin.test.assertEquals
+
+fun assertExtractsLineLibraries(expectedLibrary: String, actualLine: String,
+ extractor: ExtractorInterface) {
+ val actualLineLibraries =
+ extractor.getLineLibraries(actualLine, listOf(expectedLibrary))
+ assertTrue(expectedLibrary in actualLineLibraries)
+}
+
+fun assertExtractsNoLibraries(actualLine: String,
+ extractor: ExtractorInterface) {
+ val actualLineLibraries =
+ extractor.getLineLibraries(actualLine, listOf())
+ assertEquals(listOf(), actualLineLibraries)
+}
+
+fun assertExtractsImport(expectedImport: String, actualLine: String,
+ extractor: ExtractorInterface) {
+ val actualLineImport = extractor.extractImports(listOf(actualLine))
+ assertTrue(expectedImport in actualLineImport)
+}
+
+class ExtractorTest : Spek({
+
+ given(" code line contains library code" ) {
+ it("python extractor extracts the library") {
+ val line = "with tf.Session() as sess"
+ assertExtractsLineLibraries("tensorflow",
+ line, PythonExtractor())
+ }
+
+ it("java extractor extracts the library") {
+ val line = "private JdbcTemplate jdbcTemplate=new JdbcTemplate();"
+ assertExtractsLineLibraries("org.springframework",
+ line, JavaExtractor())
+ }
+
+ it("javascript extractor extracts the library") {
+ val line = "new Vue({"
+ assertExtractsLineLibraries("vue",
+ line, JavascriptExtractor())
+ }
+
+ it("ruby extractor extracts the library") {
+ val line1 = "img = Magick::Image.read_inline(Base64.encode64(image)).first"
+ assertExtractsLineLibraries("RMagick",
+ line1, RubyExtractor())
+ val line2 = "fximages << {image: img.adaptive_threshold(3, 3, 0), name: \"Adaptive Threshold\"}"
+ assertExtractsLineLibraries("RMagick",
+ line2, RubyExtractor())
+ }
+
+ it("go extractor extracts the library") {
+ val line = "if DB, found = revel.Config.String(\"bloggo.db\"); !found {"
+ assertExtractsLineLibraries("revel",
+ line, GoExtractor())
+ }
+
+ it("objectiveC extractor extracts the library") {
+ val line = "[[NSFileManager defaultManager] removeItemAtURL:[RLMRealmConfiguration defaultConfiguration].fileURL error:nil];"
+ assertExtractsLineLibraries("Realm",
+ line, ObjectiveCExtractor())
+ }
+
+ it("swift extractor extracts the library") {
+ val line = "class City: RLMObject {"
+ assertExtractsLineLibraries("Realm",
+ line, SwiftExtractor())
+ }
+
+ it("cpp extractor extracts the library") {
+ val line1 = "leveldb::Options options;"
+ assertExtractsLineLibraries("leveldb",
+ line1, CppExtractor())
+ val line2 = "leveldb::Status status = leveldb::DB::Open(options, \"./testdb\", &tmp);"
+ assertExtractsLineLibraries("leveldb",
+ line2, CppExtractor())
+ }
+
+ it("csharp extractor extracts the library") {
+ val line = "Algorithm = (h, v, i) => new ContrastiveDivergenceLearning(h, v)"
+ assertExtractsLineLibraries("Accord",
+ line, CSharpExtractor())
+ }
+
+ it("php extractor extracts the library") {
+ val line = "public function listRepos(string \$user, int \$limit): Call;"
+ assertExtractsLineLibraries("Tebru\\Retrofit",
+ line, PhpExtractor())
+ }
+
+ it("c extractor extracts the library") {
+ val line = "grpc_test_init(argc, argv);"
+ assertExtractsLineLibraries("grpc",
+ line, CExtractor())
+ }
+ }
+
+ given("code line doesn't use libraries" ) {
+ it("python extractor returns empty list") {
+ val line = "from collections import Counter"
+ assertExtractsNoLibraries(line, PythonExtractor())
+ }
+
+ it("java extractor returns empty list") {
+ val line = "throw new RuntimeException(e);"
+ assertExtractsNoLibraries(line, JavaExtractor())
+ }
+
+ it("javascript extractor returns empty list") {
+ val line = "console.log(self.commits[0].html_url)"
+ assertExtractsNoLibraries(line, JavascriptExtractor())
+ }
+
+ it("ruby extractor returns empty list") {
+ val line = "require \"RMagick\""
+ assertExtractsNoLibraries(line, RubyExtractor())
+ }
+
+ it("go extractor returns empty list") {
+ val line = "var found bool"
+ assertExtractsNoLibraries(line, GoExtractor())
+ }
+
+ it("objectivec extractor returns empty list") {
+ val line = "@end"
+ assertExtractsNoLibraries(line, ObjectiveCExtractor())
+ }
+
+ it("php extractor returns empty list") {
+ val line = " fail("exception") }).getLinesList()
it("'t1: initial insertion'") {
assertEquals(2, lines1.size)
@@ -85,7 +89,8 @@ class CodeLongevityTest : Spek({
// t2: subsequent insertion
testRepo.insertLines(fileName, 1, listOf("line in the middle"))
val rev2 = testRepo.commit("insert line")
- val lines2 = CodeLongevity(Repo(), emails, testRepo.git).getLinesList()
+ val lines2 = CodeLongevity(Repo(), emails, testRepo.git,
+ { _ -> fail("exception") }).getLinesList()
it("'t2: subsequent insertion'") {
assertEquals(3, lines2.size)
@@ -106,7 +111,8 @@ class CodeLongevityTest : Spek({
// t3: subsequent deletion
testRepo.deleteLines(fileName, 2, 2)
val rev3 = testRepo.commit("delete line")
- val lines3 = CodeLongevity(Repo(), emails, testRepo.git).getLinesList()
+ val lines3 = CodeLongevity(Repo(), emails, testRepo.git,
+ { _ -> fail("exception") }).getLinesList()
it("'t3: subsequent deletion'") {
assertEquals(3, lines3.size)
@@ -127,7 +133,8 @@ class CodeLongevityTest : Spek({
// t4: file deletion
testRepo.deleteFile(fileName)
val rev4 = testRepo.commit("delete file")
- val lines4 = CodeLongevity(Repo(), emails, testRepo.git).getLinesList()
+ val lines4 = CodeLongevity(Repo(), emails, testRepo.git,
+ { _ -> fail("exception") }).getLinesList()
it("'t4: file deletion'") {
assertEquals(3, lines4.size)
@@ -181,7 +188,8 @@ class CodeLongevityTest : Spek({
)
testRepo.createFile(fileName, fileContent)
val rev1 = testRepo.commit("initial commit")
- val lines1 = CodeLongevity(Repo(), emails, testRepo.git).getLinesList()
+ val lines1 = CodeLongevity(Repo(), emails, testRepo.git,
+ { _ -> fail("exception") }).getLinesList()
it("'t2.1: initial insertion'") {
assertEquals(fileContent.size, lines1.size)
@@ -227,7 +235,8 @@ class CodeLongevityTest : Spek({
testRepo.insertLines(fileName, 11, listOf("Proof addition 3"))
val rev2 = testRepo.commit("insert+delete")
- val lines2 = CodeLongevity(Repo(), emails, testRepo.git).getLinesList()
+ val lines2 = CodeLongevity(Repo(), emails, testRepo.git,
+ { _ -> fail("exception") }).getLinesList()
it("'t2.2: ins+del'") {
assertEquals(22, lines2.size)
@@ -299,7 +308,8 @@ class CodeLongevityTest : Spek({
testRepo.deleteLines(fileName, 2, 2)
val rev3 = testRepo.commit("delete line2")
- val lines1 = CodeLongevity(Repo(), emails, testRepo.git).getLinesList()
+ val lines1 = CodeLongevity(Repo(), emails, testRepo.git,
+ { _ -> fail("exception") }).getLinesList()
val lines1_line15 = lines1[0]
val lines1_line1 = lines1[1]
val lines1_line2 = lines1[2]
@@ -323,8 +333,8 @@ class CodeLongevityTest : Spek({
testRepo.deleteLines(fileName, 0, 0)
val rev4 = testRepo.commit("delete line1")
- val lines2 =
- CodeLongevity(Repo(), emails, testRepo.git).getLinesList(rev3)
+ val lines2 = CodeLongevity(Repo(), emails, testRepo.git,
+ { _ -> fail("exception") }).getLinesList(rev3)
val lines2_line1 = lines2[0]
val lines2_line15 = lines2[1]
@@ -367,10 +377,12 @@ class CodeLongevityTest : Spek({
val t1_rev = testRepo.commit(message = "insert line",
date = Calendar.Builder().setTimeOfDay(0, 1, 0).build().time)
- val t1_ages = CodeLongevity(Repo(rehash = testRehash),
- emails, testRepo.git).scan()!!
- val t1_lines = CodeLongevity(Repo(rehash = testRehash),
- emails, testRepo.git).getLinesList()
+ val t1_ages = CodeLongevity(
+ Repo(rehash = testRehash), emails, testRepo.git,
+ { _ -> fail("exception") }).scan()!!
+ val t1_lines = CodeLongevity(
+ Repo(rehash = testRehash), emails, testRepo.git,
+ { _ -> fail("exception") }).getLinesList()
it("'t1'") {
assertTrue(t1_ages.aggrAges.isEmpty(),
@@ -387,10 +399,12 @@ class CodeLongevityTest : Spek({
testRepo.commit(message = "delete line2",
date = Calendar.Builder().setTimeOfDay(0, 3, 0).build().time)
- val t2_ages = CodeLongevity(Repo(rehash = testRehash),
- emails, testRepo.git).scan()!!
- val t2_lines = CodeLongevity(Repo(rehash = testRehash),
- emails, testRepo.git).getLinesList(t1_rev)
+ val t2_ages = CodeLongevity(
+ Repo(rehash = testRehash), emails, testRepo.git,
+ { _ -> fail("exception") }).scan()!!
+ val t2_lines = CodeLongevity(
+ Repo(rehash = testRehash), emails, testRepo.git,
+ { _ -> fail("exception") }).getLinesList(t1_rev)
it("'t2'") {
assertEquals(1, t2_ages.aggrAges[email]!!.count,
@@ -408,8 +422,9 @@ class CodeLongevityTest : Spek({
}
afterGroup {
- CodeLongevity(Repo(rehash = testRehash),
- emails, testRepo.git).dropSavedData()
+ CodeLongevity(
+ Repo(rehash = testRehash), emails, testRepo.git,
+ { _ -> fail("exception") }).dropSavedData()
testRepo.destroy()
}
}
@@ -446,7 +461,8 @@ class CodeLongevityTest : Spek({
author = author1,
date = Calendar.Builder().setTimeOfDay(0, 4, 0).build().time)
- CodeLongevity(serverRepo, emails, testRepo.git).updateStats(mockApi)
+ CodeLongevity(serverRepo, emails, testRepo.git,
+ { _ -> fail("exception") }).updateStats(mockApi)
it("'t1'") {
assertTrue(mockApi.receivedFacts.contains(
@@ -464,8 +480,9 @@ class CodeLongevityTest : Spek({
}
afterGroup {
- CodeLongevity(Repo(rehash = testRehash),
- emails, testRepo.git).dropSavedData()
+ CodeLongevity(
+ Repo(rehash = testRehash), emails, testRepo.git,
+ { _ -> fail("exception") }).dropSavedData()
testRepo.destroy()
}
}
diff --git a/src/test/kotlin/test/tests/hashers/FactHasherTest.kt b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt
index 3a39b6c2..f6302188 100644
--- a/src/test/kotlin/test/tests/hashers/FactHasherTest.kt
+++ b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt
@@ -16,6 +16,7 @@ import org.jetbrains.spek.api.dsl.it
import test.utils.TestRepo
import java.util.*
import kotlin.test.assertEquals
+import kotlin.test.assertNotNull
import kotlin.test.assertTrue
class FactHasherTest : Spek({
@@ -35,6 +36,27 @@ class FactHasherTest : Spek({
return cal.time
}
+ fun getFact(code: Int, key: Int, author: Author, facts: List): Fact {
+ val fact = facts.find { fact ->
+ fact.code == code && fact.key == key && fact.author == author
+ }
+ assertNotNull(fact)
+ return fact!!
+ }
+
+ fun assertFactInt(code: Int, key: Int, value: Int, author: Author,
+ facts: List) {
+ val fact = getFact(code, key, author, facts)
+ assertEquals(value, fact.value.toInt())
+ }
+
+ fun assertFactDouble(code: Int, key: Int, value: Double, author: Author,
+ facts: List) {
+ val fact = getFact(code, key, author, facts)
+ assertTrue(Math.abs(value - fact.value.toDouble()) < 0.1,
+ "Expected approximately <$value>, actual <${fact.value}>")
+ }
+
given("commits for date facts") {
val testRepo = TestRepo(repoPath + "date-facts")
val emails = hashSetOf(authorEmail1, authorEmail2)
@@ -54,13 +76,13 @@ class FactHasherTest : Spek({
val errors = mutableListOf()
val observable = CommitCrawler.getObservable(testRepo.git, repo)
- FactHasher(repo, mockApi, emails)
+ FactHasher(repo, mockApi, listOf("r1"), emails)
.updateFromObservable(observable, { e -> errors.add(e) })
assertEquals(0, errors.size)
- assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_TIME, 13,
+ assertTrue(facts.contains(Fact(repo, FactCodes.COMMIT_DAY_TIME, 13,
"1", author1)))
- assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_WEEK, 6,
+ assertTrue(facts.contains(Fact(repo, FactCodes.COMMIT_DAY_WEEK, 6,
"1", author1)))
}
@@ -79,17 +101,17 @@ class FactHasherTest : Spek({
val errors = mutableListOf()
val observable = CommitCrawler.getObservable(testRepo.git, repo)
- FactHasher(repo, mockApi, emails)
+ FactHasher(repo, mockApi, listOf("r1", "r2"), emails)
.updateFromObservable(observable, { e -> errors.add(e) })
assertEquals(0, errors.size)
- assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_TIME, 18,
+ assertTrue(facts.contains(Fact(repo, FactCodes.COMMIT_DAY_TIME, 18,
"1", author2)))
- assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_WEEK, 0,
+ assertTrue(facts.contains(Fact(repo, FactCodes.COMMIT_DAY_WEEK, 0,
"1", author2)))
- assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_TIME, 13,
+ assertTrue(facts.contains(Fact(repo, FactCodes.COMMIT_DAY_TIME, 13,
"2", author1)))
- assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_WEEK, 0,
+ assertTrue(facts.contains(Fact(repo, FactCodes.COMMIT_DAY_WEEK, 0,
"1", author1)))
}
@@ -154,9 +176,11 @@ class FactHasherTest : Spek({
val errors = mutableListOf()
val observable = CommitCrawler.getObservable(testRepo.git, repo)
- FactHasher(repo, mockApi, emails)
+ FactHasher(repo, mockApi,
+ listOf("r1", "r2", "r3", "r4", "r5", "r6"), emails)
.updateFromObservable(observable, { e -> errors.add(e) })
+ assertEquals(0, errors.size)
assertTrue(facts.contains(Fact(repo, FactCodes.REPO_DATE_START, 0,
(startAuthor1.time/1000).toString(), author1)))
assertTrue(facts.contains(Fact(repo, FactCodes.REPO_DATE_START, 0,
@@ -169,6 +193,90 @@ class FactHasherTest : Spek({
"2")))
}
+ afterGroup {
+ testRepo.destroy()
+ }
+ }
+
+ given("test of commit facts") {
+ val testRepo = TestRepo(repoPath + "commit-facts")
+ val emails = hashSetOf(authorEmail1, authorEmail2)
+ val mockApi = MockApi(mockRepo = repo)
+ val facts = mockApi.receivedFacts
+ val lines = listOf(
+ "All my rap, if shortly, is about the thing that",
+ "For so many years so many cities have been under hoof",
+ "To go uphill when gets lucky. Then downhill when feels sick",
+ "I'm not really a Gulliver, but still the city is under hoof",
+ "City under hoof, city under hoof",
+ "Traffic lights, state duties, charges and customs",
+ "I don't know whether this path is wade or to the bottom",
+ "You live under a thumb, I have a city under my hoof",
+ "All my rap, if shortly, is about the thing that",
+ "For so many years so many cities have been under hoof",
+ "To go uphill when gets lucky. Then downhill when feels sick",
+ "I'm not really a Gulliver, but still the city is under hoof",
+ "City under hoof, city under hoof",
+ "Traffic lights, state duties, charges and customs",
+ "I don't know whether this path is wade or to the bottom",
+ "You live under a thumb, I have a city under my hoof",
+ "All my rap, if shortly, is about the thing that",
+ "For so many years so many cities have been under hoof",
+ "To go uphill when gets lucky. Then downhill when feels sick",
+ "I'm not really a Gulliver, but still the city is under hoof",
+ "City under hoof, city under hoof",
+ "Traffic lights, state duties, charges and customs",
+ "I don't know whether this path is wade or to the bottom",
+ "You live under a thumb, I have a city under my hoof"
+ )
+ val linesLenAvg = lines.fold (0) { acc, s -> acc + s.length } /
+ lines.size.toDouble()
+
+ afterEachTest {
+ facts.clear()
+ }
+
+ it("sends facts") {
+ testRepo.createFile("test1.txt", listOf())
+
+ testRepo.insertLines("test1.txt", 0, lines.subList(0, 3))
+ testRepo.commit(message = "Commit 1", author = author1)
+
+ testRepo.insertLines("test1.txt", 0, lines.subList(3, 6))
+ testRepo.commit(message = "Commit 2", author = author1)
+
+ testRepo.insertLines("test1.txt", 0, lines.subList(6, 9))
+ testRepo.commit(message = "Commit 3", author = author1)
+
+ testRepo.insertLines("test1.txt", 0, lines.subList(9, 12))
+ testRepo.commit(message = "Commit 4", author = author1)
+
+ testRepo.insertLines("test1.txt", 0, lines.subList(12, 16))
+ testRepo.commit(message = "Commit 5", author = author1)
+
+ testRepo.insertLines("test1.txt", 0, lines.subList(16, 24))
+ testRepo.commit(message = "Commit 6", author = author1)
+
+ val errors = mutableListOf()
+ val observable = CommitCrawler.getObservable(testRepo.git, repo)
+ FactHasher(repo, mockApi,
+ listOf("r1", "r2", "r3", "r4", "r5", "r6"), emails)
+ .updateFromObservable(observable, { e -> errors.add(e) })
+
+ assertEquals(0, errors.size)
+ assertFactInt(FactCodes.COMMIT_NUM, 0, 6, author1, facts)
+ assertFactDouble(FactCodes.COMMIT_LINE_NUM_AVG, 0, 4.0, author1,
+ facts)
+ assertFactInt(FactCodes.LINE_NUM, 0, 24, author1, facts)
+ assertFactDouble(FactCodes.LINE_LEN_AVG, 0, linesLenAvg, author1,
+ facts)
+ assertFactInt(FactCodes.COMMIT_NUM_TO_LINE_NUM, 3, 4, author1,
+ facts)
+ assertFactInt(FactCodes.COMMIT_NUM_TO_LINE_NUM, 4, 1, author1,
+ facts)
+ assertFactInt(FactCodes.COMMIT_NUM_TO_LINE_NUM, 8, 1, author1,
+ facts)
+ }
afterGroup {
testRepo.destroy()
diff --git a/src/test/kotlin/test/utils/TestRepo.kt b/src/test/kotlin/test/utils/TestRepo.kt
index 448fddcd..d5d123df 100644
--- a/src/test/kotlin/test/utils/TestRepo.kt
+++ b/src/test/kotlin/test/utils/TestRepo.kt
@@ -24,7 +24,7 @@ class TestRepo(val repoPath: String) {
val userName = "Contributor"
val userEmail = "test@sourcerer.com"
- val git = Git.init().setDirectory(File(repoPath)).call()
+ val git = initGit()
init {
val config = git.repository.config
@@ -33,6 +33,11 @@ class TestRepo(val repoPath: String) {
config.save()
}
+ private fun initGit(): Git {
+ destroy() // Remove repo directory if exists.
+ return Git.init().setDirectory(File(repoPath)).call()
+ }
+
fun createFile(fileName: String, content: List) {
val file = File("$repoPath/$fileName")
val writer = BufferedWriter(FileWriter(file))