From 582d2af287d6d4a2f1b4978913a60843ad607979 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Wed, 27 Sep 2017 23:53:15 +0300 Subject: [PATCH 1/3] feat: add repo facts, add email filtering for all hashers --- src/main/kotlin/app/FactCodes.kt | 3 + src/main/kotlin/app/hashers/CodeLongevity.kt | 17 +-- src/main/kotlin/app/hashers/CommitHasher.kt | 62 ++++---- src/main/kotlin/app/hashers/FactHasher.kt | 102 +++++++++----- src/main/kotlin/app/hashers/RepoHasher.kt | 38 +++-- src/main/kotlin/app/model/Author.kt | 14 +- src/main/kotlin/app/model/Fact.kt | 6 +- .../test/tests/hashers/CodeLongevityTest.kt | 41 +++--- .../test/tests/hashers/CommitHasherTest.kt | 15 +- .../test/tests/hashers/FactHasherTest.kt | 133 +++++++++++++----- 10 files changed, 268 insertions(+), 163 deletions(-) diff --git a/src/main/kotlin/app/FactCodes.kt b/src/main/kotlin/app/FactCodes.kt index b212f5a1..ccc5834a 100644 --- a/src/main/kotlin/app/FactCodes.kt +++ b/src/main/kotlin/app/FactCodes.kt @@ -5,4 +5,7 @@ object FactCodes { val COMMITS_DAY_TIME = 2 val LINE_LONGEVITY = 3 val LINE_LONGEVITY_REPO = 4 + val REPO_DATE_START = 5 + val REPO_DATE_END = 6 + val REPO_TEAM_SIZE = 7 } diff --git a/src/main/kotlin/app/hashers/CodeLongevity.kt b/src/main/kotlin/app/hashers/CodeLongevity.kt index f1753eac..984ebbe0 100644 --- a/src/main/kotlin/app/hashers/CodeLongevity.kt +++ b/src/main/kotlin/app/hashers/CodeLongevity.kt @@ -7,7 +7,6 @@ import app.FactCodes import app.Logger import app.api.Api import app.model.Author -import app.model.LocalRepo import app.model.Repo import app.model.Fact import app.utils.RepoHelper @@ -106,9 +105,9 @@ class CodeLine(val repo: Repository, /** * Used to compute age of code lines in the repo. */ -class CodeLongevity(private val localRepo: LocalRepo, - private val serverRepo: Repo, +class CodeLongevity(private val serverRepo: Repo, private val api: Api, + private val emails: HashSet, git: Git) { val repo: Repository = git.repository val head: RevCommit = @@ -121,9 +120,6 @@ class CodeLongevity(private val localRepo: LocalRepo, } fun update() { - // TODO(anatoly): Add emails from server or hashAll. - val emails = hashSetOf(localRepo.author.email) - val sums: MutableMap = emails.associate { Pair(it, 0L) } .toMutableMap() val totals: MutableMap = emails.associate { Pair(it, 0) } @@ -150,7 +146,7 @@ class CodeLongevity(private val localRepo: LocalRepo, val stats = mutableListOf() stats.add(Fact(repo = serverRepo, code = FactCodes.LINE_LONGEVITY_REPO, - value = repoAvg.toDouble())) + value = repoAvg.toString())) val repoAvgDays = repoAvg / secondsInDay Logger.info("Repo average code line age is $repoAvgDays days, " + "lines total: $repoTotal") @@ -160,13 +156,8 @@ class CodeLongevity(private val localRepo: LocalRepo, val avg = if (total > 0) { sums[email]!! / total } else 0 stats.add(Fact(repo = serverRepo, code = FactCodes.LINE_LONGEVITY, - value = avg.toDouble(), + value = avg.toString(), author = Author(email = email))) - if (email == localRepo.author.email) { - val avgDays = avg / secondsInDay - Logger.info("Your average code line age is $avgDays days, " - + "lines total: $total") - } } if (stats.size > 0) { diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index e9cc1d81..58232b1d 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -7,7 +7,6 @@ import app.Logger import app.api.Api import app.extractors.Extractor import app.model.Commit -import app.model.LocalRepo import app.model.Repo import io.reactivex.Observable import java.util.concurrent.TimeUnit @@ -15,10 +14,10 @@ import java.util.concurrent.TimeUnit /** * CommitHasher hashes repository and uploads stats to server. */ -class CommitHasher(private val localRepo: LocalRepo, - private val serverRepo: Repo = Repo(), +class CommitHasher(private val serverRepo: Repo = Repo(), private val api: Api, - private val rehashes: List) { + private val rehashes: List, + private val emails: HashSet) { init { // Delete locally missing commits from server. If found at least one @@ -30,36 +29,6 @@ class CommitHasher(private val localRepo: LocalRepo, deleteCommitsOnServer(deletedCommits) } - private fun findFirstOverlappingCommitRehash(): String? { - - val serverHistoryRehashes = serverRepo.commits - .map { commit -> commit.rehash } - .toHashSet() - return rehashes.firstOrNull { rehash -> - serverHistoryRehashes.contains(rehash) - } - } - - private fun postCommitsToServer(commits: List) { - if (commits.isNotEmpty()) { - api.postCommits(commits) - Logger.debug("Sent ${commits.size} added commits to server") - } - } - - private fun deleteCommitsOnServer(commits: List) { - if (commits.isNotEmpty()) { - api.deleteCommits(commits) - Logger.debug("Sent ${commits.size} deleted commits to server") - } - } - - private val emailFilter: (Commit) -> Boolean = { - val email = it.author.email - localRepo.hashAllContributors || (email == localRepo.author.email || - serverRepo.emails.contains(email)) - } - // Hash added and missing server commits and send them to server. fun updateFromObservable(observable: Observable) { val lastKnownCommit = serverRepo.commits.lastOrNull() @@ -74,7 +43,7 @@ class CommitHasher(private val localRepo: LocalRepo, .filter { commit -> // Don't hash known. knownCommits.isEmpty() || !knownCommits.contains(commit) } - .filter { commit -> emailFilter(commit) } // Email filtering. + .filter { commit -> emails.contains(commit.author.email) } .map { commit -> // Mapping and stats extraction. commit.stats = Extractor().extract(commit.diffs) @@ -99,4 +68,27 @@ class CommitHasher(private val localRepo: LocalRepo, throwables.add(e) // TODO(anatoly): Top-class handling errors. }) } + + private fun findFirstOverlappingCommitRehash(): String? { + val serverHistoryRehashes = serverRepo.commits + .map { commit -> commit.rehash } + .toHashSet() + return rehashes.firstOrNull { rehash -> + serverHistoryRehashes.contains(rehash) + } + } + + private fun postCommitsToServer(commits: List) { + if (commits.isNotEmpty()) { + api.postCommits(commits) + Logger.debug("Sent ${commits.size} added commits to server") + } + } + + private fun deleteCommitsOnServer(commits: List) { + if (commits.isNotEmpty()) { + api.deleteCommits(commits) + Logger.debug("Sent ${commits.size} deleted commits to server") + } + } } diff --git a/src/main/kotlin/app/hashers/FactHasher.kt b/src/main/kotlin/app/hashers/FactHasher.kt index 08de2450..69545d3f 100644 --- a/src/main/kotlin/app/hashers/FactHasher.kt +++ b/src/main/kotlin/app/hashers/FactHasher.kt @@ -9,7 +9,6 @@ import app.api.Api import app.model.Author import app.model.Commit import app.model.Fact -import app.model.LocalRepo import app.model.Repo import io.reactivex.Observable import java.time.LocalDateTime @@ -18,66 +17,95 @@ import java.time.ZoneOffset /** * CommitHasher hashes repository and uploads stats to server. */ -class FactHasher(private val localRepo: LocalRepo, - private val serverRepo: Repo = Repo(), - private val api: Api) { +class FactHasher(private val serverRepo: Repo = Repo(), + private val api: Api, + private val emails: HashSet) { + private val fsDayWeek = hashMapOf>() + private val fsDayTime = hashMapOf>() + private val fsRepoDateStart = hashMapOf() + private val fsRepoDateEnd = hashMapOf() + private val fsRepoTeamSize = hashSetOf() - private fun postFactsToServer(facts: List) { - if (facts.isNotEmpty()) { - api.postFacts(facts) - Logger.debug("Sent ${facts.size} facts to server") + init { + for (author in emails) { + fsDayWeek.put(author, Array(7) { 0 }) + fsDayTime.put(author, Array(24) { 0 }) + fsRepoDateStart.put(author, -1) + fsRepoDateEnd.put(author, -1) } } fun updateFromObservable(observable: Observable) { - val factsDayWeek = hashMapOf>() - val factsDayTime = hashMapOf>() - val throwables = mutableListOf() - // TODO(anatoly): Filter hashing by email as in CommitHasher. observable + .filter { commit -> emails.contains(commit.author.email) } .subscribe({ commit -> // OnNext. // Calculate facts. - val author = commit.author - val factDayWeek = factsDayWeek[author] ?: Array(7) { 0 } - val factDayTime = factsDayTime[author] ?: Array(24) { 0 } + val email = commit.author.email val timestamp = commit.dateTimestamp val dateTime = LocalDateTime.ofEpochSecond(timestamp, 0, ZoneOffset.ofTotalSeconds(commit.dateTimeZoneOffset * 60)) + + // DayWeek. + val factDayWeek = fsDayWeek[email] ?: Array(7) { 0 } // The value is numbered from 1 (Monday) to 7 (Sunday). factDayWeek[dateTime.dayOfWeek.value - 1] += 1 + fsDayWeek[email] = factDayWeek + + // DayTime. + val factDayTime = fsDayTime[email] ?: Array(24) { 0 } // Hour from 0 to 23. factDayTime[dateTime.hour] += 1 - factsDayWeek[author] = factDayWeek - factsDayTime[author] = factDayTime + fsDayTime[email] = factDayTime + + // RepoDateStart. + fsRepoDateStart[email] = timestamp + + // RepoDateEnd. + if ((fsRepoDateEnd[email] ?: -1) == -1L) { + fsRepoDateEnd[email] = timestamp + } + + // RepoTeamSize. + fsRepoTeamSize.add(email) }, { e -> // OnError. throwables.add(e) // TODO(anatoly): Top-class handling errors. }, { // OnComplete. try { - val facts = mutableListOf() - factsDayTime.map { (author, list) -> - list.forEachIndexed { hour, count -> - if (count > 0) { - facts.add(Fact(serverRepo, - FactCodes.COMMITS_DAY_TIME, hour, - count.toDouble(), author)) - } - } - } - factsDayWeek.map { (author, list) -> - list.forEachIndexed { day, count -> - if (count > 0) { - facts.add(Fact(serverRepo, - FactCodes.COMMITS_DAY_WEEK, day, - count.toDouble(), author)) - } - } - } - postFactsToServer(facts) + postFactsToServer(createFacts()) } catch (e: Throwable) { throwables.add(e) } }) } + + private fun createFacts(): List { + val fs = mutableListOf() + emails.forEach { email -> + val author = Author(email = email) + fsDayTime[email]?.forEachIndexed { hour, count -> if (count > 0) { + fs.add(Fact(serverRepo, FactCodes.COMMITS_DAY_TIME, hour, + count.toString(), author)) + }} + fsDayWeek[email]?.forEachIndexed { day, count -> if (count > 0) { + fs.add(Fact(serverRepo, FactCodes.COMMITS_DAY_WEEK, day, + count.toString(), author)) + }} + fs.add(Fact(serverRepo, FactCodes.REPO_DATE_START, 0, + fsRepoDateStart[email].toString(), author)) + fs.add(Fact(serverRepo, FactCodes.REPO_DATE_END, 0, + fsRepoDateEnd[email].toString(), author)) + } + fs.add(Fact(serverRepo, FactCodes.REPO_TEAM_SIZE, 0, + fsRepoTeamSize.size.toString())) + return fs + } + + private fun postFactsToServer(facts: List) { + if (facts.isNotEmpty()) { + api.postFacts(facts) + Logger.debug("Sent ${facts.size} facts to server") + } + } } diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt index 19dc2e18..fefce47e 100644 --- a/src/main/kotlin/app/hashers/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -6,7 +6,6 @@ package app.hashers import app.Logger import app.api.Api import app.config.Configurator -import app.model.Author import app.model.LocalRepo import app.model.Repo import app.utils.RepoHelper @@ -17,6 +16,7 @@ import org.eclipse.jgit.revwalk.RevWalk import java.io.File import java.io.IOException import java.util.* +import kotlin.collections.HashSet class RepoHasher(private val localRepo: LocalRepo, private val api: Api, private val configurator: Configurator) { @@ -30,13 +30,15 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, println("Hashing $localRepo...") val git = loadGit(localRepo.path) try { - val (rehashes, _) = fetchRehashesAndAuthors(git) + val (rehashes, emails) = fetchRehashesAndEmails(git) localRepo.parseGitConfig(git.repository.config) if (localRepo.author.email.isBlank()) { throw IllegalStateException("Can't load email from Git config") } + val filteredEmails = filterEmails(emails) + initServerRepo(rehashes.last) if (!isKnownRepo()) { @@ -49,15 +51,15 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, // Hash by all plugins. val observable = CommitCrawler.getObservable(git, serverRepo) .publish() - CommitHasher(localRepo, serverRepo, api, rehashes) + CommitHasher(serverRepo, api, rehashes, filteredEmails) .updateFromObservable(observable) - FactHasher(localRepo, serverRepo, api) + FactHasher(serverRepo, api, filteredEmails) .updateFromObservable(observable) // Start and synchronously wait until all subscribers complete. observable.connect() // TODO(anatoly): CodeLongevity hash from observable. - CodeLongevity(localRepo, serverRepo, api, git).update() + CodeLongevity(serverRepo, api, filteredEmails, git).update() // Confirm hashing completion. postRepoToServer() @@ -103,8 +105,8 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, serverRepo.initialCommitRehash, localRepo) } - private fun fetchRehashesAndAuthors(git: Git): - Pair, HashMap> { + private fun fetchRehashesAndEmails(git: Git): + Pair, HashSet> { val head: RevCommit = RevWalk(git.repository) .parseCommit(git.repository.resolve(RepoHelper.MASTER_BRANCH)) @@ -112,21 +114,29 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, revWalk.markStart(head) val commitsRehashes = LinkedList() - val contributors = hashMapOf() + val emails = hashSetOf() var commit: RevCommit? = revWalk.next() while (commit != null) { commitsRehashes.add(DigestUtils.sha256Hex(commit.name)) - if (!contributors.containsKey(commit.authorIdent.emailAddress)) { - val author = Author(commit.authorIdent.name, - commit.authorIdent.emailAddress) - contributors.put(commit.authorIdent.emailAddress, author) - } + emails.add(commit.authorIdent.emailAddress) commit.disposeBody() commit = revWalk.next() } revWalk.dispose() - return Pair(commitsRehashes, contributors) + return Pair(commitsRehashes, emails) + } + + private fun filterEmails(emails: HashSet): HashSet { + if (localRepo.hashAllContributors) { + return emails + } + + val knownEmails = mutableListOf() + knownEmails.add(serverRepo.userEmail) + knownEmails.addAll(serverRepo.emails) + + return knownEmails.filter { emails.contains(it) }.toHashSet() } } diff --git a/src/main/kotlin/app/model/Author.kt b/src/main/kotlin/app/model/Author.kt index 219ece10..6f690f53 100644 --- a/src/main/kotlin/app/model/Author.kt +++ b/src/main/kotlin/app/model/Author.kt @@ -6,4 +6,16 @@ package app.model /** * Commit author. */ -data class Author(var name: String = "", var email: String = "") +data class Author(var name: String = "", var email: String = "") { + // Email defines user identity. + override fun equals(other: Any?): Boolean { + if (other is Author) { + return email == other.email + } + return false + } + + override fun hashCode(): Int { + return email.hashCode() + } +} diff --git a/src/main/kotlin/app/model/Fact.kt b/src/main/kotlin/app/model/Fact.kt index 43e1e92b..91daf5a8 100644 --- a/src/main/kotlin/app/model/Fact.kt +++ b/src/main/kotlin/app/model/Fact.kt @@ -11,7 +11,7 @@ data class Fact( var repo: Repo = Repo(), var code: Int = 0, var key: Int = 0, - var value: Double = 0.0, + var value: String = "", var author: Author = Author() ) { @Throws(InvalidParameterException::class) @@ -20,7 +20,7 @@ data class Fact( author = Author("", proto.email) code = proto.code key = proto.key - value = proto.value1.toDoubleOrNull() ?: 0.0 + value = "" } @Throws(InvalidProtocolBufferException::class) @@ -34,7 +34,7 @@ data class Fact( .setEmail(author.email) .setCode(code) .setKey(key) - .setValue1(value.toString()) + .setValue1(value) .build() } diff --git a/src/test/kotlin/test/tests/hashers/CodeLongevityTest.kt b/src/test/kotlin/test/tests/hashers/CodeLongevityTest.kt index dee6e32d..d60a9df0 100644 --- a/src/test/kotlin/test/tests/hashers/CodeLongevityTest.kt +++ b/src/test/kotlin/test/tests/hashers/CodeLongevityTest.kt @@ -47,9 +47,9 @@ class CodeLongevityTest : Spek({ actualLine: CodeLine) { assertRevCommitLine(fromCommit, fromFile, fromLineNum, actualLine.from, - "'$lineText' from_commit"); + "'$lineText' from_commit") assertRevCommitLine(toCommit, toFile, toLineNum, actualLine.to, - "'$lineText' to_commit"); + "'$lineText' to_commit") assertEquals(lineText, actualLine.text, "line text") assertEquals(isDeleted, actualLine.isDeleted, "'$lineText' line is deleted") @@ -59,12 +59,13 @@ class CodeLongevityTest : Spek({ val testRepoPath = "../CodeLongevity_lc1" val testRepo = TestRepo(testRepoPath) val fileName = "test1.txt" + val emails = hashSetOf(testRepo.userEmail) // t1: initial insertion testRepo.createFile(fileName, listOf("line1", "line2")) - val rev1 = testRepo.commit("inital commit") - val lines1 = CodeLongevity( - LocalRepo(testRepoPath), Repo(), MockApi(), testRepo.git).getLinesList() + val rev1 = testRepo.commit("initial commit") + val lines1 = CodeLongevity(Repo(), MockApi(), emails, + testRepo.git).getLinesList() it("'t1: initial insertion'") { assertEquals(2, lines1.size) @@ -81,8 +82,8 @@ class CodeLongevityTest : Spek({ // t2: subsequent insertion testRepo.insertLines(fileName, 1, listOf("line in the middle")) val rev2 = testRepo.commit("insert line") - val lines2 = CodeLongevity( - LocalRepo(testRepoPath), Repo(), MockApi(), testRepo.git).getLinesList() + val lines2 = CodeLongevity(Repo(), MockApi(), emails, + testRepo.git).getLinesList() it("'t2: subsequent insertion'") { assertEquals(3, lines2.size) @@ -103,8 +104,8 @@ class CodeLongevityTest : Spek({ // t3: subsequent deletion testRepo.deleteLines(fileName, 2, 2) val rev3 = testRepo.commit("delete line") - val lines3 = CodeLongevity(LocalRepo(testRepoPath), Repo(), - MockApi(), testRepo.git).getLinesList() + val lines3 = CodeLongevity(Repo(), MockApi(), emails, + testRepo.git).getLinesList() it("'t3: subsequent deletion'") { assertEquals(3, lines3.size) @@ -125,8 +126,8 @@ class CodeLongevityTest : Spek({ // t4: file deletion testRepo.deleteFile(fileName) val rev4 = testRepo.commit("delete file") - val lines4 = CodeLongevity(LocalRepo(testRepoPath), Repo(), - MockApi(), testRepo.git).getLinesList() + val lines4 = CodeLongevity(Repo(), MockApi(), emails, + testRepo.git).getLinesList() it("'t4: file deletion'") { assertEquals(3, lines4.size) @@ -154,6 +155,7 @@ class CodeLongevityTest : Spek({ val testRepoPath = "../CodeLongevity_lc2" val testRepo = TestRepo(testRepoPath) val fileName = "test1.txt" + val emails = hashSetOf(testRepo.userEmail) // t2.1: initial insertion val fileContent = listOf( @@ -179,8 +181,8 @@ class CodeLongevityTest : Spek({ ) testRepo.createFile(fileName, fileContent) val rev1 = testRepo.commit("inital commit") - val lines1 = CodeLongevity( - LocalRepo(testRepoPath), Repo(), MockApi(), testRepo.git).getLinesList() + val lines1 = CodeLongevity(Repo(), MockApi(), emails, + testRepo.git).getLinesList() it("'t2.1: initial insertion'") { assertEquals(fileContent.size, lines1.size) @@ -226,8 +228,8 @@ class CodeLongevityTest : Spek({ testRepo.insertLines(fileName, 11, listOf("Proof addition 3")) val rev2 = testRepo.commit("insert+delete") - val lines2 = CodeLongevity( - LocalRepo(testRepoPath), Repo(), MockApi(), testRepo.git).getLinesList() + val lines2 = CodeLongevity(Repo(), MockApi(), emails, + testRepo.git).getLinesList() it("'t2.2: ins+del'") { assertEquals(22, lines2.size) @@ -290,6 +292,7 @@ class CodeLongevityTest : Spek({ val testRepoPath = "../CodeLongevity_lc3" val testRepo = TestRepo(testRepoPath) val fileName = "test1.txt" + val emails = hashSetOf(testRepo.userEmail) testRepo.createFile(fileName, listOf("line1", "line2")) val rev1 = testRepo.commit("inital commit") @@ -298,8 +301,8 @@ class CodeLongevityTest : Spek({ testRepo.deleteLines(fileName, 2, 2) val rev3 = testRepo.commit("delete line2") - val lines1 = CodeLongevity(LocalRepo(testRepoPath), Repo(), - MockApi(), testRepo.git).getLinesList() + val lines1 = CodeLongevity(Repo(), MockApi(), emails, + testRepo.git).getLinesList() val lines1_line15 = lines1[0] val lines1_line1 = lines1[1] val lines1_line2 = lines1[2] @@ -323,8 +326,8 @@ class CodeLongevityTest : Spek({ testRepo.deleteLines(fileName, 0, 0) val rev4 = testRepo.commit("delete line1") - val lines2 = CodeLongevity(LocalRepo(testRepoPath), Repo(), - MockApi(), testRepo.git).getLinesList(rev3) + val lines2 = CodeLongevity(Repo(), MockApi(), emails, + testRepo.git).getLinesList(rev3) val lines2_line1 = lines2[0] val lines2_line15 = lines2[1] diff --git a/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt b/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt index 4405500d..8682e21b 100644 --- a/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt +++ b/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt @@ -52,20 +52,19 @@ class CommitHasherTest : Spek({ // Common parameters for CommitHasher. val gitHasher = Git.open(File(repoPath)) - val localRepo = LocalRepo(repoPath) - localRepo.author = Author(userName, userEmail) val initialCommit = Commit(git.commit().setMessage("Initial commit").call()) val repoRehash = RepoHelper.calculateRepoRehash(initialCommit.rehash, - localRepo) + LocalRepo(repoPath).also { it.author = Author(userName, userEmail) }) val repo = Repo(rehash = repoRehash, initialCommitRehash = initialCommit.rehash) + val emails = hashSetOf(userEmail) given("repo with initial commit and no history") { repo.commits = listOf() val mockApi = MockApi(mockRepo = repo) val observable = CommitCrawler.getObservable(gitHasher, repo) - CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + CommitHasher(repo, mockApi, repo.commits.map {it.rehash}, emails) .updateFromObservable(observable) it("send added commits") { @@ -82,7 +81,7 @@ class CommitHasherTest : Spek({ val mockApi = MockApi(mockRepo = repo) val observable = CommitCrawler.getObservable(gitHasher, repo) - CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + CommitHasher(repo, mockApi, repo.commits.map {it.rehash}, emails) .updateFromObservable(observable) it("doesn't send added commits") { @@ -102,7 +101,7 @@ class CommitHasherTest : Spek({ val revCommit = git.commit().setMessage("Second commit.").call() val addedCommit = Commit(revCommit) val observable = CommitCrawler.getObservable(gitHasher, repo) - CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + CommitHasher(repo, mockApi, repo.commits.map {it.rehash}, emails) .updateFromObservable(observable) it("doesn't send deleted commits") { @@ -138,7 +137,7 @@ class CommitHasherTest : Spek({ authorCommits.add(Commit(revCommit)) } val observable = CommitCrawler.getObservable(gitHasher, repo) - CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + CommitHasher(repo, mockApi, repo.commits.map {it.rehash}, emails) .updateFromObservable(observable) it("posts five commits as added") { @@ -200,7 +199,7 @@ class CommitHasherTest : Spek({ repo.commits = addedCommits.toList().asReversed() val observable = CommitCrawler.getObservable(gitHasher, repo) - CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + CommitHasher(repo, mockApi, repo.commits.map {it.rehash}, emails) .updateFromObservable(observable) it("adds posts one commit as added and received commit is lost one") { diff --git a/src/test/kotlin/test/tests/hashers/FactHasherTest.kt b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt index ba0de54e..169c53f2 100644 --- a/src/test/kotlin/test/tests/hashers/FactHasherTest.kt +++ b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt @@ -9,46 +9,42 @@ import app.hashers.CommitCrawler import app.hashers.FactHasher import app.model.Author import app.model.Fact -import app.model.LocalRepo import app.model.Repo import org.jetbrains.spek.api.Spek import org.jetbrains.spek.api.dsl.given import org.jetbrains.spek.api.dsl.it import test.utils.TestRepo import java.util.* -import kotlin.test.assertEquals import kotlin.test.assertTrue class FactHasherTest : Spek({ - val userName = "Contributor" - val userEmail = "test@domain.com" + val authorEmail1 = "test1@domain.com" + val authorEmail2 = "test2@domain.com" + val author1 = Author("Test1", authorEmail1) + val author2 = Author("Test2", authorEmail2) val repoPath = "../testrepo-fact-hasher-" - val localRepo = LocalRepo(repoPath) - localRepo.author = Author(userName, userEmail) + val repo = Repo(rehash = "rehash", commits = listOf()) + + fun createDate(year: Int = 2017, month: Int = 1, day: Int = 1, + hour: Int = 0, minute: Int = 0, seconds: Int = 0): Date { + val cal = Calendar.getInstance() + // Month in calendar is 0-based. + cal.set(year, month - 1, day, hour, minute, seconds) + return cal.time + } - given("test of commits date facts") { + given("commits for date facts") { val testRepo = TestRepo(repoPath + "date-facts") - val author1 = Author("Test1", "test@domain.com") - val author2 = Author("Test2", "test@domain.com") - - val repo = Repo(rehash = "rehash", commits = listOf()) + val emails = hashSetOf(authorEmail1, authorEmail2) val mockApi = MockApi(mockRepo = repo) val facts = mockApi.receivedFacts - fun createDate(year: Int = 2017, month: Int = 1, day: Int = 1, - hour: Int = 0, minute: Int = 0, seconds: Int = 0): Date { - val cal = Calendar.getInstance() - // Month in calendar is 0-based. - cal.set(year, month - 1, day, hour, minute, seconds) - return cal.time - } - afterEachTest { facts.clear() } - it("send two facts") { + it("sends initial facts") { testRepo.createFile("test1.txt", listOf("line1", "line2")) testRepo.commit(message = "initial commit", author = author1, @@ -56,17 +52,15 @@ class FactHasherTest : Spek({ hour = 13, minute = 0, seconds = 0)) val observable = CommitCrawler.getObservable(testRepo.git, repo) - FactHasher(localRepo, repo, mockApi) - .updateFromObservable(observable) + FactHasher(repo, mockApi, emails).updateFromObservable(observable) - assertEquals(2, facts.size) assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_TIME, 13, - 1.0, author1))) + "1", author1))) assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_WEEK, 6, - 1.0, author1))) + "1", author1))) } - it("send more facts") { + it("sends more facts") { testRepo.createFile("test2.txt", listOf("line1", "line2")) testRepo.commit(message = "second commit", author = author2, @@ -80,19 +74,92 @@ class FactHasherTest : Spek({ hour = 13, minute = 0, seconds = 0)) val observable = CommitCrawler.getObservable(testRepo.git, repo) - FactHasher(localRepo, repo, mockApi) - .updateFromObservable(observable) + FactHasher(repo, mockApi, emails).updateFromObservable(observable) - assertEquals(5, facts.size) assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_TIME, 18, - 1.0, author2))) + "1", author2))) assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_WEEK, 0, - 1.0, author2))) + "1", author2))) assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_TIME, 13, - 2.0, author1))) + "2", author1))) assertTrue(facts.contains(Fact(repo, FactCodes.COMMITS_DAY_WEEK, 0, - 1.0, author1))) + "1", author1))) + } + + afterGroup { + testRepo.destroy() } + } + + given("test of repo facts") { + val testRepo = TestRepo(repoPath + "repo-facts") + val emails = hashSetOf(authorEmail1, authorEmail2) + val mockApi = MockApi(mockRepo = repo) + val facts = mockApi.receivedFacts + + afterEachTest { + facts.clear() + } + + it("sends facts") { + val startAuthor1 = createDate(year = 2016, month = 2, day = 10, + hour = 13, minute = 0, seconds = 0) + val midAuthor1 = createDate(year = 2016, month = 10, day = 5, + hour = 10, minute = 0, seconds = 0) + val startAuthor2 = createDate(year = 2017, month = 1, day = 1, + hour = 12, minute = 10, seconds = 10) + val midAuthor2 = createDate(year = 2017, month = 2, day = 2, + hour = 15, minute = 0, seconds = 0) + val endAuthor1 = createDate(year = 2017, month = 4, day = 5, + hour = 13, minute = 0, seconds = 0) + val endAuthor2 = createDate(year = 2017, month = 5, day = 15, + hour = 15, minute = 0, seconds = 0) + + testRepo.createFile("test1.txt", listOf("line1", "line2")) + testRepo.commit(message = "author1 -> start", + author = author1, + date = startAuthor1) + + testRepo.insertLines("test1.txt", 0, listOf("line3")) + testRepo.commit(message = "author1 -> mid", + author = author1, + date = midAuthor1) + + testRepo.createFile("test2.txt", listOf("line4")) + testRepo.commit(message = "author2 -> start", + author = author2, + date = startAuthor2) + + testRepo.insertLines("test2.txt", 0, listOf("line5")) + testRepo.commit(message = "author2 -> mid", + author = author2, + date = midAuthor2) + + testRepo.insertLines("test1.txt", 0, listOf("line6")) + testRepo.commit(message = "author1 -> end", + author = author1, + date = endAuthor1) + + testRepo.insertLines("test2.txt", 0, listOf("line7")) + testRepo.commit(message = "author2 -> end", + author = author2, + date = endAuthor2) + + val observable = CommitCrawler.getObservable(testRepo.git, repo) + FactHasher(repo, mockApi, emails).updateFromObservable(observable) + + assertTrue(facts.contains(Fact(repo, FactCodes.REPO_DATE_START, 0, + (startAuthor1.time/1000).toString(), author1))) + assertTrue(facts.contains(Fact(repo, FactCodes.REPO_DATE_START, 0, + (startAuthor2.time/1000).toString(), author2))) + assertTrue(facts.contains(Fact(repo, FactCodes.REPO_DATE_END, 0, + (endAuthor1.time/1000).toString(), author1))) + assertTrue(facts.contains(Fact(repo, FactCodes.REPO_DATE_END, 0, + (endAuthor2.time/1000).toString(), author2))) + assertTrue(facts.contains(Fact(repo, FactCodes.REPO_TEAM_SIZE, 0, + "2"))) + } + afterGroup { testRepo.destroy() From 2c48e86fffdf6067f2b9ea8696df4224d3a6eb53 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Mon, 2 Oct 2017 14:42:19 +0300 Subject: [PATCH 2/3] chore(CommitHasher): update filters --- src/main/kotlin/app/hashers/CommitHasher.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index 58232b1d..29c11f6d 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -40,9 +40,9 @@ class CommitHasher(private val serverRepo: Repo = Repo(), .takeWhile { new -> // Hash until last known commit. new.rehash != lastKnownCommit?.rehash } - .filter { commit -> // Don't hash known. - knownCommits.isEmpty() || !knownCommits.contains(commit) - } + // Don't hash known to server commits. + .filter { commit -> !knownCommits.contains(commit) } + // Hash only commits made by authors with specified emails. .filter { commit -> emails.contains(commit.author.email) } .map { commit -> // Mapping and stats extraction. From b27ad790166a14ed23895989978f29125379dc23 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Mon, 2 Oct 2017 15:15:20 +0300 Subject: [PATCH 3/3] fix: merge --- src/main/kotlin/app/hashers/RepoHasher.kt | 4 ++-- src/test/kotlin/test/tests/hashers/FactHasherTest.kt | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt index 3722ba9b..cd51c272 100644 --- a/src/main/kotlin/app/hashers/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -63,9 +63,9 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, // Hash by all plugins. val observable = CommitCrawler.getObservable(git, serverRepo) .publish() - CommitHasher(localRepo, serverRepo, api, rehashes, filteredEmails) + CommitHasher(serverRepo, api, rehashes, filteredEmails) .updateFromObservable(observable, onError) - FactHasher(localRepo, serverRepo, api, filteredEmails) + FactHasher(serverRepo, api, filteredEmails) .updateFromObservable(observable, onError) // Start and synchronously wait until all subscribers complete. diff --git a/src/test/kotlin/test/tests/hashers/FactHasherTest.kt b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt index 092b471c..3a39b6c2 100644 --- a/src/test/kotlin/test/tests/hashers/FactHasherTest.kt +++ b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt @@ -15,6 +15,7 @@ import org.jetbrains.spek.api.dsl.given import org.jetbrains.spek.api.dsl.it import test.utils.TestRepo import java.util.* +import kotlin.test.assertEquals import kotlin.test.assertTrue class FactHasherTest : Spek({ @@ -151,8 +152,10 @@ class FactHasherTest : Spek({ author = author2, date = endAuthor2) + val errors = mutableListOf() val observable = CommitCrawler.getObservable(testRepo.git, repo) - FactHasher(repo, mockApi, emails).updateFromObservable(observable) + FactHasher(repo, mockApi, emails) + .updateFromObservable(observable, { e -> errors.add(e) }) assertTrue(facts.contains(Fact(repo, FactCodes.REPO_DATE_START, 0, (startAuthor1.time/1000).toString(), author1)))