diff --git a/src/main/kotlin/app/FactKey.kt b/src/main/kotlin/app/FactKey.kt new file mode 100644 index 00000000..60ef9a80 --- /dev/null +++ b/src/main/kotlin/app/FactKey.kt @@ -0,0 +1,8 @@ +package app + +object FactKey { + val COMMITS_DAY_WEEK = "commits-day-week-" + val COMMITS_DAY_TIME = "commits-day-time-" + val LINE_LONGEVITY = "line-longevity-avg" + val LINE_LONGEVITY_REPO = "line-longevity-repo-avg" +} diff --git a/src/main/kotlin/app/hashers/CodeLongevity.kt b/src/main/kotlin/app/hashers/CodeLongevity.kt index b23410cd..75f9c979 100644 --- a/src/main/kotlin/app/hashers/CodeLongevity.kt +++ b/src/main/kotlin/app/hashers/CodeLongevity.kt @@ -3,6 +3,7 @@ package app.hashers +import app.FactKey import app.Logger import app.api.Api import app.config.Configurator @@ -80,9 +81,6 @@ class CodeLongevity(private val localRepo: LocalRepo, private val serverRepo: Repo, private val api: Api, private val git: Git, tailRev: String = "") { - val CODE_LONGEVITY_KEY = "line-longevity-avg" - val CODE_LONGEVITY_REPO_KEY = "line-longevity-repo-avg" - val repo: Repository = git.repository val head: RevCommit = RevWalk(repo).parseCommit(repo.resolve(RepoHelper.MASTER_BRANCH)) @@ -121,7 +119,7 @@ class CodeLongevity(private val localRepo: LocalRepo, val repoAvg = if (repoTotal > 0) { repoSum / repoTotal } else 0 val stats = mutableListOf() stats.add(Fact(repo = serverRepo, - key = CODE_LONGEVITY_REPO_KEY, + key = FactKey.LINE_LONGEVITY_REPO, value = repoAvg.toDouble())) val repoAvgDays = repoAvg / secondsInDay Logger.info("Repo average code line age is $repoAvgDays days, " @@ -131,7 +129,7 @@ class CodeLongevity(private val localRepo: LocalRepo, val total = totals[email] ?: 0 val avg = if (total > 0) { sums[email]!! / total } else 0 stats.add(Fact(repo = serverRepo, - key = CODE_LONGEVITY_KEY, + key = FactKey.LINE_LONGEVITY, value = avg.toDouble(), author = Author(email = email))) if (email == localRepo.author.email) { diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index 323698a2..326cd2bc 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -3,13 +3,16 @@ package app.hashers +import app.FactKey import app.Logger import app.api.Api import app.extractors.Extractor +import app.model.Author import app.model.Commit import app.model.DiffContent import app.model.DiffFile import app.model.DiffRange +import app.model.Fact import app.model.LocalRepo import app.model.Repo import app.utils.RepoHelper @@ -26,6 +29,8 @@ import org.eclipse.jgit.lib.ObjectId import org.eclipse.jgit.errors.MissingObjectException import org.eclipse.jgit.revwalk.RevCommit import org.eclipse.jgit.util.io.DisabledOutputStream +import java.time.LocalDateTime +import java.time.ZoneOffset import java.util.concurrent.TimeUnit /** @@ -35,11 +40,12 @@ class CommitHasher(private val localRepo: LocalRepo, private val repo: Repo = Repo(), private val api: Api, private val git: Git) { + private val gitRepo: Repository = git.repository private fun findFirstOverlappingCommit(): Commit? { val serverHistoryCommits = repo.commits.toHashSet() - return getObservableCommits() + return getCommitsAsObservable() .skipWhile { commit -> !serverHistoryCommits.contains(commit) } .blockingFirst(null) } @@ -47,9 +53,32 @@ class CommitHasher(private val localRepo: LocalRepo, private fun hashAndSendCommits() { val lastKnownCommit = repo.commits.lastOrNull() val knownCommits = repo.commits.toHashSet() + + val factsDayWeek = hashMapOf>() + val factsDayTime = hashMapOf>() + // Commits are combined in pairs, an empty commit concatenated to // calculate the diff of the initial commit. - Observable.concat(getObservableCommits(), Observable.just(Commit())) + Observable.concat(getCommitsAsObservable() + .doOnNext { commit -> + Logger.debug("Commit: ${commit.raw?.name ?: ""}: " + + commit.raw?.shortMessage) + commit.repo = repo + + // Calculate facts. + val author = commit.author + val factDayWeek = factsDayWeek[author] ?: Array(7) { 0 } + val factDayTime = factsDayTime[author] ?: Array(24) { 0 } + val timestamp = commit.dateTimestamp + val dateTime = LocalDateTime.ofEpochSecond(timestamp, 0, + ZoneOffset.ofTotalSeconds(commit.dateTimeZoneOffset * 60)) + // The value is numbered from 1 (Monday) to 7 (Sunday). + factDayWeek[dateTime.dayOfWeek.value - 1] += 1 + // Hour from 0 to 23. + factDayTime[dateTime.hour] += 1 + factsDayWeek[author] = factDayWeek + factsDayTime[author] = factDayTime + }, Observable.just(Commit())) .pairWithNext() // Pair commits to get diff. .takeWhile { (new, _) -> // Hash until last known commit. new.rehash != lastKnownCommit?.rehash } @@ -57,10 +86,6 @@ class CommitHasher(private val localRepo: LocalRepo, || !knownCommits.contains(new) } .filter { (new, _) -> emailFilter(new) } // Email filtering. .map { (new, old) -> // Mapping and stats extraction. - Logger.debug("Commit: ${new.raw?.name ?: ""}: " - + new.raw?.shortMessage) - new.repo = repo - val diffFiles = getDiffFiles(new, old) Logger.debug("Diff: ${diffFiles.size} entries") new.stats = Extractor().extract(diffFiles) @@ -73,18 +98,33 @@ class CommitHasher(private val localRepo: LocalRepo, total + file.getAllAdded().size } new.numLinesDeleted = diffFiles.fold(0) { total, file -> total + file.getAllDeleted().size } - new } .observeOn(Schedulers.io()) // Different thread for data sending. .buffer(20, TimeUnit.SECONDS) // Group ready commits by time. - .doOnNext { commitsBundle -> // Send ready commits. - postCommitsToServer(commitsBundle) } - .blockingSubscribe({ - // OnNext - }, { t -> // OnError - Logger.error("Error while hashing: $t") - t.printStackTrace() + .blockingSubscribe({ commitsBundle -> // OnNext. + postCommitsToServer(commitsBundle) // Send ready commits. + }, { e -> // OnError. + Logger.error("Error while hashing: $e") + }, { // OnComplete. + val facts = mutableListOf() + factsDayTime.map { (author, list) -> + list.forEachIndexed { hour, count -> + if (count > 0) { + facts.add(Fact(repo, FactKey.COMMITS_DAY_TIME + + hour, count.toDouble(), author)) + } + } + } + factsDayWeek.map { (author, list) -> + list.forEachIndexed { day, count -> + if (count > 0) { + facts.add(Fact(repo, FactKey.COMMITS_DAY_WEEK + + day, count.toDouble(), author)) + } + } + } + postFactsToServer(facts) }) } @@ -142,6 +182,13 @@ class CommitHasher(private val localRepo: LocalRepo, } } + private fun postFactsToServer(facts: List) { + if (facts.isNotEmpty()) { + api.postFacts(facts) + Logger.debug("Sent ${facts.size} facts to server") + } + } + private fun deleteCommitsOnServer(commits: List) { if (commits.isNotEmpty()) { api.deleteCommits(commits) @@ -149,7 +196,7 @@ class CommitHasher(private val localRepo: LocalRepo, } } - private fun getObservableCommits(): Observable = + private fun getCommitsAsObservable(): Observable = Observable.create { subscriber -> try { val revWalk = RevWalk(gitRepo) diff --git a/src/main/kotlin/app/model/Commit.kt b/src/main/kotlin/app/model/Commit.kt index 4853b4c4..33da1dc3 100644 --- a/src/main/kotlin/app/model/Commit.kt +++ b/src/main/kotlin/app/model/Commit.kt @@ -18,7 +18,8 @@ data class Commit( // Tree rehash used for adjustments of stats due to rebase and fraud. var treeRehash: String = "", var author: Author = Author(), - var dateTimestamp: Int = 0, + var dateTimestamp: Long = 0, + var dateTimeZoneOffset: Int = 0, var isQommit: Boolean = false, var numLinesAdded: Int = 0, var numLinesDeleted: Int = 0, @@ -33,7 +34,8 @@ data class Commit( rehash = DigestUtils.sha256Hex(revCommit.id.name) author = Author(revCommit.authorIdent.name, revCommit.authorIdent.emailAddress) - dateTimestamp = revCommit.commitTime + dateTimestamp = revCommit.authorIdent.getWhen().time / 1000 + dateTimeZoneOffset = revCommit.authorIdent.timeZoneOffset treeRehash = DigestUtils.sha256Hex(revCommit.tree.name) } diff --git a/src/main/proto/sourcerer.proto b/src/main/proto/sourcerer.proto index cc9c26ec..e85dbc85 100644 --- a/src/main/proto/sourcerer.proto +++ b/src/main/proto/sourcerer.proto @@ -24,7 +24,7 @@ message Commit { string author_email = 5; // Timestamp of a commit creation in seconds UTC+0. - uint32 date = 6; + uint64 date = 6; // Is quality commit. bool is_qommit = 7; diff --git a/src/test/kotlin/test/tests/hashers/CodeLongevityTest.kt b/src/test/kotlin/test/tests/hashers/CodeLongevityTest.kt index bd4a1b69..6651a6d6 100644 --- a/src/test/kotlin/test/tests/hashers/CodeLongevityTest.kt +++ b/src/test/kotlin/test/tests/hashers/CodeLongevityTest.kt @@ -12,7 +12,6 @@ import app.model.* import test.utils.TestRepo import kotlin.test.assertEquals -import kotlin.test.assertNotEquals import org.eclipse.jgit.revwalk.RevCommit @@ -60,7 +59,7 @@ class CodeLongevityTest : Spek({ val fileName = "test1.txt" // t1: initial insertion - testRepo.newFile(fileName, listOf("line1", "line2")) + testRepo.createFile(fileName, listOf("line1", "line2")) val rev1 = testRepo.commit("inital commit") val lines1 = CodeLongevity( LocalRepo(testRepoPath), Repo(), MockApi(), testRepo.git).compute() @@ -177,7 +176,7 @@ class CodeLongevityTest : Spek({ "line17", "line18" ) - testRepo.newFile(fileName, fileContent) + testRepo.createFile(fileName, fileContent) val rev1 = testRepo.commit("inital commit") val lines1 = CodeLongevity( LocalRepo(testRepoPath), Repo(), MockApi(), testRepo.git).compute() diff --git a/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt b/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt index 2de293c7..837f68e5 100644 --- a/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt +++ b/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt @@ -4,6 +4,7 @@ package test.tests.hashers +import app.FactKey import app.api.MockApi import app.hashers.CommitHasher import app.model.* @@ -12,11 +13,14 @@ import org.eclipse.jgit.api.Git import org.jetbrains.spek.api.Spek import org.jetbrains.spek.api.dsl.given import org.jetbrains.spek.api.dsl.it +import test.utils.TestRepo import java.io.File +import java.util.* import java.util.stream.StreamSupport.stream import kotlin.streams.toList import kotlin.test.assertEquals import kotlin.test.assertNotEquals +import kotlin.test.assertTrue class CommitHasherTest : Spek({ val userName = "Contributor" @@ -56,6 +60,14 @@ class CommitHasherTest : Spek({ return lastCommit } + fun createDate(year: Int = 2017, month: Int = 1, day: Int = 1, + hour: Int = 0, minute: Int = 0, seconds: Int = 0): Date { + val cal = Calendar.getInstance() + // Month in calendar is 0-based. + cal.set(year, month - 1, day, hour, minute, seconds) + return cal.time + } + given("repo with initial commit and no history") { repo.commits = listOf() @@ -199,5 +211,66 @@ class CommitHasherTest : Spek({ } } + given("test of commits date facts") { + val testRepoPath = "../testrepo1" + val testRepo = TestRepo(testRepoPath) + val author1 = Author("Test1", "test@domain.com") + val author2 = Author("Test2", "test@domain.com") + + val repo = Repo(rehash = "rehash", commits = listOf()) + val mockApi = MockApi(mockRepo = repo) + val facts = mockApi.receivedFacts + + afterEachTest { + facts.clear() + } + + it("send two facts") { + testRepo.createFile("test1.txt", listOf("line1", "line2")) + testRepo.commit(message = "initial commit", + author = author1, + date = createDate(month = 1, day = 1, // Sunday. + hour = 13, minute = 0, seconds = 0)) + + CommitHasher(localRepo, repo, mockApi, testRepo.git).update() + + assertEquals(2, facts.size) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 13, + 1.0, author1))) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 6, + 1.0, author1))) + } + + it("send more facts") { + testRepo.createFile("test2.txt", listOf("line1", "line2")) + testRepo.commit(message = "second commit", + author = author2, + date = createDate(month = 1, day = 2, // Monday. + hour = 18, minute = 0, seconds = 0)) + + testRepo.createFile("test3.txt", listOf("line1", "line2")) + testRepo.commit(message = "third commit", + author = author1, + date = createDate(month = 1, day = 2, // Monday. + hour = 13, minute = 0, seconds = 0)) + + CommitHasher(localRepo, repo, mockApi, testRepo.git).update() + + assertEquals(5, facts.size) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 18, + 1.0, author2))) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 0, + 1.0, author2))) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 13, + 2.0, author1))) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 0, + 1.0, author1))) + } + + afterGroup { + testRepo.destroy() + } + } + Runtime.getRuntime().exec("src/test/delete_repo.sh").waitFor() }) diff --git a/src/test/kotlin/test/utils/TestRepo.kt b/src/test/kotlin/test/utils/TestRepo.kt index 6a825bd1..448fddcd 100644 --- a/src/test/kotlin/test/utils/TestRepo.kt +++ b/src/test/kotlin/test/utils/TestRepo.kt @@ -1,8 +1,10 @@ // Copyright 2017 Sourcerer Inc. All Rights Reserved. // Author: Alexander Surkov (alex@sourcerer.io) +// Author: Anatoly Kislov (anatoly@sourcerer.io) package test.utils +import app.model.Author import java.io.BufferedReader import java.io.BufferedWriter import java.io.File @@ -11,7 +13,9 @@ import java.io.FileWriter import java.io.StringWriter import org.eclipse.jgit.api.Git +import org.eclipse.jgit.lib.PersonIdent import org.eclipse.jgit.revwalk.RevCommit +import java.util.* /** * A wrapper around Git repo allowing to add/remove/edit files and make commits. @@ -29,7 +33,7 @@ class TestRepo(val repoPath: String) { config.save() } - fun newFile(fileName: String, content: List) { + fun createFile(fileName: String, content: List) { val file = File("$repoPath/$fileName") val writer = BufferedWriter(FileWriter(file)) for (line in content) { @@ -103,8 +107,13 @@ class TestRepo(val repoPath: String) { writer.close() } - fun commit(message: String): RevCommit { - return git.commit().setMessage(message).setAll(true).call() + fun commit(message: String, + author: Author = Author(userName, userEmail), + date: Date = Date(), + timeZone: TimeZone = TimeZone.getDefault()): RevCommit { + val personIdent = PersonIdent(author.name, author.email, date, timeZone) + return git.commit().setMessage(message).setAll(true) + .setAuthor(personIdent).setCommitter(personIdent).call() } fun destroy() {